Add comprehensive SSH terminal drawer feature with embedded and popout modes, integrate playbook lint results API with local cache fallback, and enhance host management UI with terminal access buttons

app/services/terminal_service.py

@@ -0,0 +1,368 @@
+"""
+Terminal Service - Manages SSH terminal sessions via ttyd.
+
+This service handles:
+- Creating terminal sessions with unique tokens
+- Spawning ttyd processes for SSH connections
+- Managing session lifecycle (creation, expiration, cleanup)
+- Port allocation for ttyd instances
+"""
+import asyncio
+import hashlib
+import logging
+import os
+import secrets
+import shutil
+import signal
+import socket
+import subprocess
+from datetime import datetime, timedelta, timezone
+from typing import Dict, Optional, Tuple
+
+from app.core.config import settings
+
+logger = logging.getLogger(__name__)
+
+
+# Configuration
+TERMINAL_SESSION_TTL_MINUTES = int(os.environ.get("TERMINAL_SESSION_TTL_MINUTES", "30"))
+TERMINAL_MAX_SESSIONS_PER_USER = int(os.environ.get("TERMINAL_MAX_SESSIONS_PER_USER", "3"))
+TERMINAL_PORT_RANGE_START = int(os.environ.get("TERMINAL_PORT_RANGE_START", "7680"))
+TERMINAL_PORT_RANGE_END = int(os.environ.get("TERMINAL_PORT_RANGE_END", "7700"))
+SSH_USER = os.environ.get("TERMINAL_SSH_USER", "automation")
+TTYD_PATH = os.environ.get("TTYD_PATH", "ttyd")
+
+
+class TerminalServiceError(Exception):
+    """Base exception for terminal service errors."""
+    pass
+
+
+class HostNotReadyError(TerminalServiceError):
+    """Host is not ready for terminal connection."""
+    pass
+
+
+class SessionLimitExceededError(TerminalServiceError):
+    """User has too many active sessions."""
+    pass
+
+
+class TtydNotAvailableError(TerminalServiceError):
+    """ttyd binary is not available."""
+    pass
+
+
+class TerminalService:
+    """
+    Manages terminal sessions and ttyd processes.
+    
+    This service is responsible for:
+    - Generating secure session tokens
+    - Allocating ports for ttyd instances
+    - Spawning and managing ttyd processes
+    - Cleaning up expired sessions
+    """
+    
+    def __init__(self):
+        # Track active ttyd processes: session_id -> subprocess.Popen
+        self._processes: Dict[str, subprocess.Popen] = {}
+        # Track allocated ports: port -> session_id
+        self._allocated_ports: Dict[int, str] = {}
+        # Lock for thread-safe operations
+        self._lock = asyncio.Lock()
+        # Check if ttyd is available
+        self._ttyd_available: Optional[bool] = None
+    
+    def check_ttyd_available(self) -> bool:
+        """Check if ttyd binary is available."""
+        if self._ttyd_available is not None:
+            return self._ttyd_available
+        
+        self._ttyd_available = shutil.which(TTYD_PATH) is not None
+        if not self._ttyd_available:
+            logger.warning(f"ttyd not found at '{TTYD_PATH}'. Terminal feature will be unavailable.")
+        else:
+            logger.info(f"ttyd found at: {shutil.which(TTYD_PATH)}")
+        
+        return self._ttyd_available
+    
+    def generate_session_id(self) -> str:
+        """Generate a unique session ID."""
+        return secrets.token_hex(32)
+    
+    def generate_session_token(self) -> Tuple[str, str]:
+        """
+        Generate a session token and its hash.
+        
+        Returns:
+            Tuple of (plain_token, token_hash)
+        """
+        token = secrets.token_urlsafe(48)
+        token_hash = hashlib.sha256(token.encode()).hexdigest()
+        return token, token_hash
+    
+    def verify_token(self, token: str, token_hash: str) -> bool:
+        """Verify a token against its hash."""
+        computed_hash = hashlib.sha256(token.encode()).hexdigest()
+        return secrets.compare_digest(computed_hash, token_hash)
+
+    def _is_port_available(self, port: int) -> bool:
+        """Check if a TCP port is available for binding."""
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        try:
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 0)
+            sock.bind(("0.0.0.0", port))
+            return True
+        except OSError:
+            return False
+        finally:
+            try:
+                sock.close()
+            except Exception:
+                pass
+    
+    async def allocate_port(self, session_id: str) -> int:
+        """
+        Allocate an available port for a ttyd instance.
+        
+        Returns:
+            Available port number
+            
+        Raises:
+            TerminalServiceError if no ports available
+        """
+        async with self._lock:
+            for port in range(TERMINAL_PORT_RANGE_START, TERMINAL_PORT_RANGE_END + 1):
+                if port in self._allocated_ports:
+                    continue
+                if not self._is_port_available(port):
+                    continue
+
+                if port not in self._allocated_ports:
+                    self._allocated_ports[port] = session_id
+                    logger.debug(f"Allocated port {port} for session {session_id[:8]}...")
+                    return port
+            
+            raise TerminalServiceError("No available ports for terminal session")
+    
+    async def release_port(self, port: int) -> None:
+        """Release an allocated port."""
+        async with self._lock:
+            if port in self._allocated_ports:
+                session_id = self._allocated_ports.pop(port)
+                logger.debug(f"Released port {port} from session {session_id[:8]}...")
+    
+    async def spawn_ttyd(
+        self,
+        session_id: str,
+        host_ip: str,
+        port: int,
+        token: str,
+    ) -> Optional[int]:
+        """
+        Spawn a ttyd process for SSH connection.
+        
+        Args:
+            session_id: Unique session identifier
+            host_ip: Target host IP address
+            port: Port to run ttyd on
+            token: Session token for authentication
+            
+        Returns:
+            Process ID of the spawned ttyd, or None if failed
+        """
+        if not self.check_ttyd_available():
+            raise TtydNotAvailableError("ttyd is not installed or not in PATH")
+        
+        key_candidates = []
+        env_key = os.environ.get("TERMINAL_SSH_KEY_PATH")
+        if env_key:
+            key_candidates.append(env_key)
+        if getattr(settings, "ssh_key_path", None):
+            key_candidates.append(settings.ssh_key_path)
+        key_candidates.append(str(os.path.expanduser("~/.ssh/id_automation_ansible")))
+        key_candidates.append(str(os.path.expanduser("~/.ssh/id_rsa")))
+
+        ssh_key_path = next((p for p in key_candidates if p and os.path.exists(p)), None)
+        if not ssh_key_path:
+            raise TerminalServiceError(
+                "No SSH key found for terminal sessions. Set SSH_KEY_PATH or TERMINAL_SSH_KEY_PATH (expected ~/.ssh/id_automation_ansible)."
+            )
+
+        # Build ttyd command
+        # --once: Exit after single client disconnects
+        # --credential: Basic auth with session token
+        # --port: Listen port
+        # -W: Write-only mode disabled (allow input)
+        # The command executed is SSH to the target host
+        ssh_cmd = [
+            "ssh",
+            "-i",
+            ssh_key_path,
+            "-o",
+            "BatchMode=no",
+            "-o",
+            "PreferredAuthentications=publickey,keyboard-interactive,password",
+            "-o",
+            "ConnectTimeout=10",
+            "-o",
+            "ServerAliveInterval=30",
+            "-o",
+            "ServerAliveCountMax=2",
+            "-o",
+            "StrictHostKeyChecking=accept-new",
+            "-o",
+            "UserKnownHostsFile=/dev/null",
+            f"{SSH_USER}@{host_ip}",
+        ]
+
+        # Note: --credential removed to avoid HTTP basic auth popup
+        # Security is handled by session token validation in the connect/popout routes
+        cmd = [
+            TTYD_PATH,
+            "--once",
+            f"--port={port}",
+            "--writable",
+            *ssh_cmd,
+        ]
+
+        logger.info(
+            f"Spawning ttyd for session {session_id[:8]}... on port {port} -> {SSH_USER}@{host_ip} (key={ssh_key_path})"
+        )
+        
+        try:
+            # Spawn the process
+            process = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                start_new_session=True,  # Detach from parent process group
+            )
+            
+            # Store the process
+            async with self._lock:
+                self._processes[session_id] = process
+            
+            # Wait briefly to check if process started successfully
+            await asyncio.sleep(0.5)
+            
+            if process.poll() is not None:
+                # Process exited immediately - likely an error
+                stdout, stderr = process.communicate(timeout=1)
+                error_msg = stderr.decode() if stderr else stdout.decode() if stdout else "Unknown error"
+                logger.error(f"ttyd failed to start: {error_msg}")
+                await self.release_port(port)
+                raise TerminalServiceError(f"ttyd failed to start: {error_msg.strip()}")
+            
+            logger.info(f"ttyd started with PID {process.pid} for session {session_id[:8]}...")
+            return process.pid
+            
+        except TerminalServiceError:
+            # Preserve detailed error message for API layer
+            raise
+        except Exception as e:
+            logger.exception(f"Failed to spawn ttyd: {e}")
+            await self.release_port(port)
+            raise TerminalServiceError(str(e))
+    
+    async def terminate_session(self, session_id: str) -> bool:
+        """
+        Terminate a terminal session and its ttyd process.
+        
+        Returns:
+            True if session was terminated, False if not found
+        """
+        async with self._lock:
+            process = self._processes.pop(session_id, None)
+        
+        if process is None:
+            return False
+        
+        try:
+            # Try graceful termination first
+            if process.poll() is None:
+                process.terminate()
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    # Force kill if graceful termination failed
+                    process.kill()
+                    process.wait(timeout=2)
+            
+            logger.info(f"Terminated ttyd process for session {session_id[:8]}...")
+            return True
+            
+        except Exception as e:
+            logger.exception(f"Error terminating ttyd process: {e}")
+            return False
+    
+    async def cleanup_expired_sessions(
+        self,
+        get_expired_sessions_func,
+        mark_expired_func,
+        db_session
+    ) -> int:
+        """
+        Clean up expired terminal sessions.
+        
+        Args:
+            get_expired_sessions_func: Async function to get expired sessions from DB
+            mark_expired_func: Async function to mark session as expired in DB
+            db_session: Database session
+            
+        Returns:
+            Number of sessions cleaned up
+        """
+        from app.crud.terminal_session import TerminalSessionRepository
+        
+        repo = TerminalSessionRepository(db_session)
+        expired = await repo.list_expired()
+        
+        count = 0
+        for session in expired:
+            # Terminate the ttyd process
+            await self.terminate_session(session.id)
+            # Release the port
+            await self.release_port(session.ttyd_port)
+            # Mark as expired in DB
+            await repo.mark_expired(session.id)
+            count += 1
+            logger.info(f"Cleaned up expired session {session.id[:8]}... for host {session.host_name}")
+        
+        return count
+    
+    def get_active_session_count(self) -> int:
+        """Get the number of active ttyd processes."""
+        return len(self._processes)
+    
+    def get_session_url(self, port: int, token: str, base_url: str = "") -> str:
+        """
+        Get the URL to access a terminal session.
+        
+        Args:
+            port: ttyd port
+            token: Session token
+            base_url: Optional base URL prefix
+            
+        Returns:
+            Full URL to access the terminal
+        """
+        # If we have a reverse proxy, use the proxied URL
+        # Otherwise, direct access to ttyd port
+        if base_url:
+            return f"{base_url}/terminal/proxy/{port}?token={token}"
+        return f"http://localhost:{port}/"
+    
+    def get_websocket_url(self, port: int) -> str:
+        """Get the WebSocket URL for terminal connection."""
+        return f"ws://localhost:{port}/ws"
+
+
+# Global service instance
+terminal_service = TerminalService()
+
+
+def get_terminal_service() -> TerminalService:
+    """Get the global terminal service instance."""
+    return terminal_service