Bruno Charest
493668f746
Some checks failed
Tests / Backend Tests (Python) (3.10) (push) Has been cancelled
Tests / Backend Tests (Python) (3.11) (push) Has been cancelled
Tests / Backend Tests (Python) (3.12) (push) Has been cancelled
Tests / Frontend Tests (JS) (push) Has been cancelled
Tests / Integration Tests (push) Has been cancelled
Tests / All Tests Passed (push) Has been cancelled
369 lines
12 KiB
Python
369 lines
12 KiB
Python
"""
|
|
Terminal Service - Manages SSH terminal sessions via ttyd.
|
|
|
|
This service handles:
|
|
- Creating terminal sessions with unique tokens
|
|
- Spawning ttyd processes for SSH connections
|
|
- Managing session lifecycle (creation, expiration, cleanup)
|
|
- Port allocation for ttyd instances
|
|
"""
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Dict, Optional, Tuple
|
|
|
|
from app.core.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Configuration
|
|
TERMINAL_SESSION_TTL_MINUTES = int(os.environ.get("TERMINAL_SESSION_TTL_MINUTES", "30"))
|
|
TERMINAL_MAX_SESSIONS_PER_USER = int(os.environ.get("TERMINAL_MAX_SESSIONS_PER_USER", "3"))
|
|
TERMINAL_PORT_RANGE_START = int(os.environ.get("TERMINAL_PORT_RANGE_START", "7680"))
|
|
TERMINAL_PORT_RANGE_END = int(os.environ.get("TERMINAL_PORT_RANGE_END", "7700"))
|
|
SSH_USER = os.environ.get("TERMINAL_SSH_USER", "automation")
|
|
TTYD_PATH = os.environ.get("TTYD_PATH", "ttyd")
|
|
|
|
|
|
class TerminalServiceError(Exception):
|
|
"""Base exception for terminal service errors."""
|
|
pass
|
|
|
|
|
|
class HostNotReadyError(TerminalServiceError):
|
|
"""Host is not ready for terminal connection."""
|
|
pass
|
|
|
|
|
|
class SessionLimitExceededError(TerminalServiceError):
|
|
"""User has too many active sessions."""
|
|
pass
|
|
|
|
|
|
class TtydNotAvailableError(TerminalServiceError):
|
|
"""ttyd binary is not available."""
|
|
pass
|
|
|
|
|
|
class TerminalService:
|
|
"""
|
|
Manages terminal sessions and ttyd processes.
|
|
|
|
This service is responsible for:
|
|
- Generating secure session tokens
|
|
- Allocating ports for ttyd instances
|
|
- Spawning and managing ttyd processes
|
|
- Cleaning up expired sessions
|
|
"""
|
|
|
|
def __init__(self):
|
|
# Track active ttyd processes: session_id -> subprocess.Popen
|
|
self._processes: Dict[str, subprocess.Popen] = {}
|
|
# Track allocated ports: port -> session_id
|
|
self._allocated_ports: Dict[int, str] = {}
|
|
# Lock for thread-safe operations
|
|
self._lock = asyncio.Lock()
|
|
# Check if ttyd is available
|
|
self._ttyd_available: Optional[bool] = None
|
|
|
|
def check_ttyd_available(self) -> bool:
|
|
"""Check if ttyd binary is available."""
|
|
if self._ttyd_available is not None:
|
|
return self._ttyd_available
|
|
|
|
self._ttyd_available = shutil.which(TTYD_PATH) is not None
|
|
if not self._ttyd_available:
|
|
logger.warning(f"ttyd not found at '{TTYD_PATH}'. Terminal feature will be unavailable.")
|
|
else:
|
|
logger.info(f"ttyd found at: {shutil.which(TTYD_PATH)}")
|
|
|
|
return self._ttyd_available
|
|
|
|
def generate_session_id(self) -> str:
|
|
"""Generate a unique session ID."""
|
|
return secrets.token_hex(32)
|
|
|
|
def generate_session_token(self) -> Tuple[str, str]:
|
|
"""
|
|
Generate a session token and its hash.
|
|
|
|
Returns:
|
|
Tuple of (plain_token, token_hash)
|
|
"""
|
|
token = secrets.token_urlsafe(48)
|
|
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
|
return token, token_hash
|
|
|
|
def verify_token(self, token: str, token_hash: str) -> bool:
|
|
"""Verify a token against its hash."""
|
|
computed_hash = hashlib.sha256(token.encode()).hexdigest()
|
|
return secrets.compare_digest(computed_hash, token_hash)
|
|
|
|
def _is_port_available(self, port: int) -> bool:
|
|
"""Check if a TCP port is available for binding."""
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
try:
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 0)
|
|
sock.bind(("0.0.0.0", port))
|
|
return True
|
|
except OSError:
|
|
return False
|
|
finally:
|
|
try:
|
|
sock.close()
|
|
except Exception:
|
|
pass
|
|
|
|
async def allocate_port(self, session_id: str) -> int:
|
|
"""
|
|
Allocate an available port for a ttyd instance.
|
|
|
|
Returns:
|
|
Available port number
|
|
|
|
Raises:
|
|
TerminalServiceError if no ports available
|
|
"""
|
|
async with self._lock:
|
|
for port in range(TERMINAL_PORT_RANGE_START, TERMINAL_PORT_RANGE_END + 1):
|
|
if port in self._allocated_ports:
|
|
continue
|
|
if not self._is_port_available(port):
|
|
continue
|
|
|
|
if port not in self._allocated_ports:
|
|
self._allocated_ports[port] = session_id
|
|
logger.debug(f"Allocated port {port} for session {session_id[:8]}...")
|
|
return port
|
|
|
|
raise TerminalServiceError("No available ports for terminal session")
|
|
|
|
async def release_port(self, port: int) -> None:
|
|
"""Release an allocated port."""
|
|
async with self._lock:
|
|
if port in self._allocated_ports:
|
|
session_id = self._allocated_ports.pop(port)
|
|
logger.debug(f"Released port {port} from session {session_id[:8]}...")
|
|
|
|
async def spawn_ttyd(
|
|
self,
|
|
session_id: str,
|
|
host_ip: str,
|
|
port: int,
|
|
token: str,
|
|
) -> Optional[int]:
|
|
"""
|
|
Spawn a ttyd process for SSH connection.
|
|
|
|
Args:
|
|
session_id: Unique session identifier
|
|
host_ip: Target host IP address
|
|
port: Port to run ttyd on
|
|
token: Session token for authentication
|
|
|
|
Returns:
|
|
Process ID of the spawned ttyd, or None if failed
|
|
"""
|
|
if not self.check_ttyd_available():
|
|
raise TtydNotAvailableError("ttyd is not installed or not in PATH")
|
|
|
|
key_candidates = []
|
|
env_key = os.environ.get("TERMINAL_SSH_KEY_PATH")
|
|
if env_key:
|
|
key_candidates.append(env_key)
|
|
if getattr(settings, "ssh_key_path", None):
|
|
key_candidates.append(settings.ssh_key_path)
|
|
key_candidates.append(str(os.path.expanduser("~/.ssh/id_automation_ansible")))
|
|
key_candidates.append(str(os.path.expanduser("~/.ssh/id_rsa")))
|
|
|
|
ssh_key_path = next((p for p in key_candidates if p and os.path.exists(p)), None)
|
|
if not ssh_key_path:
|
|
raise TerminalServiceError(
|
|
"No SSH key found for terminal sessions. Set SSH_KEY_PATH or TERMINAL_SSH_KEY_PATH (expected ~/.ssh/id_automation_ansible)."
|
|
)
|
|
|
|
# Build ttyd command
|
|
# --once: Exit after single client disconnects
|
|
# --credential: Basic auth with session token
|
|
# --port: Listen port
|
|
# -W: Write-only mode disabled (allow input)
|
|
# The command executed is SSH to the target host
|
|
ssh_cmd = [
|
|
"ssh",
|
|
"-i",
|
|
ssh_key_path,
|
|
"-o",
|
|
"BatchMode=no",
|
|
"-o",
|
|
"PreferredAuthentications=publickey,keyboard-interactive,password",
|
|
"-o",
|
|
"ConnectTimeout=10",
|
|
"-o",
|
|
"ServerAliveInterval=30",
|
|
"-o",
|
|
"ServerAliveCountMax=2",
|
|
"-o",
|
|
"StrictHostKeyChecking=accept-new",
|
|
"-o",
|
|
"UserKnownHostsFile=/dev/null",
|
|
f"{SSH_USER}@{host_ip}",
|
|
]
|
|
|
|
# Note: --credential removed to avoid HTTP basic auth popup
|
|
# Security is handled by session token validation in the connect/popout routes
|
|
cmd = [
|
|
TTYD_PATH,
|
|
"--once",
|
|
f"--port={port}",
|
|
"--writable",
|
|
*ssh_cmd,
|
|
]
|
|
|
|
logger.info(
|
|
f"Spawning ttyd for session {session_id[:8]}... on port {port} -> {SSH_USER}@{host_ip} (key={ssh_key_path})"
|
|
)
|
|
|
|
try:
|
|
# Spawn the process
|
|
process = subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
start_new_session=True, # Detach from parent process group
|
|
)
|
|
|
|
# Store the process
|
|
async with self._lock:
|
|
self._processes[session_id] = process
|
|
|
|
# Wait briefly to check if process started successfully
|
|
await asyncio.sleep(0.5)
|
|
|
|
if process.poll() is not None:
|
|
# Process exited immediately - likely an error
|
|
stdout, stderr = process.communicate(timeout=1)
|
|
error_msg = stderr.decode() if stderr else stdout.decode() if stdout else "Unknown error"
|
|
logger.error(f"ttyd failed to start: {error_msg}")
|
|
await self.release_port(port)
|
|
raise TerminalServiceError(f"ttyd failed to start: {error_msg.strip()}")
|
|
|
|
logger.info(f"ttyd started with PID {process.pid} for session {session_id[:8]}...")
|
|
return process.pid
|
|
|
|
except TerminalServiceError:
|
|
# Preserve detailed error message for API layer
|
|
raise
|
|
except Exception as e:
|
|
logger.exception(f"Failed to spawn ttyd: {e}")
|
|
await self.release_port(port)
|
|
raise TerminalServiceError(str(e))
|
|
|
|
async def terminate_session(self, session_id: str) -> bool:
|
|
"""
|
|
Terminate a terminal session and its ttyd process.
|
|
|
|
Returns:
|
|
True if session was terminated, False if not found
|
|
"""
|
|
async with self._lock:
|
|
process = self._processes.pop(session_id, None)
|
|
|
|
if process is None:
|
|
return False
|
|
|
|
try:
|
|
# Try graceful termination first
|
|
if process.poll() is None:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
# Force kill if graceful termination failed
|
|
process.kill()
|
|
process.wait(timeout=2)
|
|
|
|
logger.info(f"Terminated ttyd process for session {session_id[:8]}...")
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.exception(f"Error terminating ttyd process: {e}")
|
|
return False
|
|
|
|
async def cleanup_expired_sessions(
|
|
self,
|
|
get_expired_sessions_func,
|
|
mark_expired_func,
|
|
db_session
|
|
) -> int:
|
|
"""
|
|
Clean up expired terminal sessions.
|
|
|
|
Args:
|
|
get_expired_sessions_func: Async function to get expired sessions from DB
|
|
mark_expired_func: Async function to mark session as expired in DB
|
|
db_session: Database session
|
|
|
|
Returns:
|
|
Number of sessions cleaned up
|
|
"""
|
|
from app.crud.terminal_session import TerminalSessionRepository
|
|
|
|
repo = TerminalSessionRepository(db_session)
|
|
expired = await repo.list_expired()
|
|
|
|
count = 0
|
|
for session in expired:
|
|
# Terminate the ttyd process
|
|
await self.terminate_session(session.id)
|
|
# Release the port
|
|
await self.release_port(session.ttyd_port)
|
|
# Mark as expired in DB
|
|
await repo.mark_expired(session.id)
|
|
count += 1
|
|
logger.info(f"Cleaned up expired session {session.id[:8]}... for host {session.host_name}")
|
|
|
|
return count
|
|
|
|
def get_active_session_count(self) -> int:
|
|
"""Get the number of active ttyd processes."""
|
|
return len(self._processes)
|
|
|
|
def get_session_url(self, port: int, token: str, base_url: str = "") -> str:
|
|
"""
|
|
Get the URL to access a terminal session.
|
|
|
|
Args:
|
|
port: ttyd port
|
|
token: Session token
|
|
base_url: Optional base URL prefix
|
|
|
|
Returns:
|
|
Full URL to access the terminal
|
|
"""
|
|
# If we have a reverse proxy, use the proxied URL
|
|
# Otherwise, direct access to ttyd port
|
|
if base_url:
|
|
return f"{base_url}/terminal/proxy/{port}?token={token}"
|
|
return f"http://localhost:{port}/"
|
|
|
|
def get_websocket_url(self, port: int) -> str:
|
|
"""Get the WebSocket URL for terminal connection."""
|
|
return f"ws://localhost:{port}/ws"
|
|
|
|
|
|
# Global service instance
|
|
terminal_service = TerminalService()
|
|
|
|
|
|
def get_terminal_service() -> TerminalService:
|
|
"""Get the global terminal service instance."""
|
|
return terminal_service
|