283 lines
11 KiB
Python
283 lines
11 KiB
Python
"""
|
||
Secret redactor: masks sensitive patterns in rendered text.
|
||
|
||
Scans for common secret patterns and replaces them with a French mask
|
||
label (``[CLÉ API MASQUÉE]``, ``[MOT DE PASSE MASQUÉ]``, …) before content
|
||
is served to the frontend. Prevents accidental exposure of API keys,
|
||
tokens, and passwords in previews.
|
||
|
||
Patterns detected:
|
||
- Generic API keys (``api_key=…``, ``token: …`` — values of 8+ chars)
|
||
- Passwords (``password=…``, ``"passwd": "…"`` — any length)
|
||
- JWT tokens (eyJ... base64url)
|
||
- Provider key formats: OpenAI/Anthropic/OpenRouter (``sk-``), Stripe,
|
||
GitLab, Google (``AIza…`` / ``ya29.``), AWS, GitHub, Slack, SendGrid,
|
||
Hugging Face, npm, Docker, Resend, Square, Atlassian, Discord,
|
||
Telegram, ``Bearer …`` tokens
|
||
- Private key blocks (-----BEGIN ... PRIVATE KEY-----)
|
||
- Connection strings with passwords
|
||
- Bare hex secrets next to a secret keyword (BUG-035)
|
||
|
||
Interactive masking (feature #188): :func:`redact_with_placeholders`
|
||
returns the text with every mask replaced by an opaque placeholder plus
|
||
the list of ``(label, secret)`` entries; :func:`restore_masks` turns the
|
||
placeholders back into plain labels (public shares, PDF exports, AI
|
||
context) or into clickable ``<span class="secret-mask" data-secret="…">``
|
||
badges (authenticated app preview) so a click copies the real value to
|
||
the clipboard.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import html as _html
|
||
import logging
|
||
import re
|
||
|
||
logger = logging.getLogger("obsigate.redactor")
|
||
|
||
# --- Patterns ---
|
||
# Each entry is ``(pattern, replacement, secret_group)``:
|
||
# * ``replacement``: a literal label, a ``\\1``-style template, or a
|
||
# callable receiving the match and returning the visible label;
|
||
# * ``secret_group``: index of the group holding the value that a click
|
||
# copies to the clipboard (feature #188).
|
||
_PATTERNS: list[tuple[re.Pattern[str], object, int]] = [
|
||
# Private key blocks
|
||
(re.compile(r'-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----.*?-----END (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----', re.DOTALL), '[CLÉ PRIVÉE MASQUÉE]', 0),
|
||
|
||
# JWT tokens (base64url encoded, starts with eyJ)
|
||
(re.compile(r'eyJ[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}\.[a-zA-Z0-9_-]{20,}'), '[JWT MASQUÉ]', 0),
|
||
|
||
# Connection strings with passwords
|
||
(re.compile(r'(?:mongodb|mysql|postgres(?:ql)?|redis|sqlite)://[^:]+:[^@\s]+@'), '[CONNECTION_STRING MASQUÉE]', 0),
|
||
|
||
# Passwords — any length, bare or quoted (``password=…``,
|
||
# ``"passwd": "…"``). The left side may carry a qualifier
|
||
# (``db_password``, ``DATABASE.PASSWORD``, ``user_pwd``); a *bare*
|
||
# ``PWD=`` (shell working directory) must NOT match, hence ``pwd``
|
||
# only in its qualified branch.
|
||
(re.compile(
|
||
r'(?i)((?:[A-Za-z0-9_.-]*(?:password|passwd|passphrase|mot\s+de\s+passe)'
|
||
r'|[A-Za-z0-9_.-]+pwd)["\']?\s*[:=]\s*["\']?)([^\s"\',;]{4,})'),
|
||
r'\1[MOT DE PASSE MASQUÉ]', 2),
|
||
|
||
# Generic API key assignments: api_key=…, token=…, secret=… — values
|
||
# of 8+ characters (short enough to catch real keys, long enough to
|
||
# skip plain words).
|
||
(re.compile(r'(?i)([A-Za-z0-9_.-]*(?:api[_-]?key|apikey|secret|token|auth[_-]?token)["\']?\s*[:=]\s*["\']?)([^\s\'"]{8,})'),
|
||
lambda m: f'{m.group(1)}[MASQUÉ]' if ("=" in m.group(0) or ":" in m.group(0)) else '[MASQUÉ]', 2),
|
||
|
||
# GitHub tokens (ghp_, gho_, ghu_, ghs_, ghr_)
|
||
(re.compile(r'gh[pousr]_[a-zA-Z0-9]{36,}'), '[GITHUB_TOKEN MASQUÉ]', 0),
|
||
|
||
# AWS access keys
|
||
(re.compile(r'(?:AKIA|ASIA)[0-9A-Z]{16}'), '[AWS_KEY MASQUÉ]', 0),
|
||
|
||
# Provider key formats (feature #188) — one alternation covering the
|
||
# large majority of token shapes in the wild.
|
||
(re.compile(
|
||
r'(?<![A-Za-z0-9])(?:'
|
||
r'sk-[A-Za-z0-9_\-]{16,}' # OpenAI / Anthropic / OpenRouter
|
||
r'|sk_(?:live|test)_[A-Za-z0-9]{10,}' # Stripe secret key
|
||
r'|pk_(?:live|test)_[A-Za-z0-9]{10,}' # Stripe publishable key
|
||
r'|whsec_[A-Za-z0-9]{16,}' # Stripe / Svix webhook secret
|
||
r'|glpat-[A-Za-z0-9_\-]{20,}' # GitLab personal access token
|
||
r'|github_pat_[A-Za-z0-9_]{22,}' # GitHub fine-grained PAT
|
||
r'|npm_[A-Za-z0-9]{36}' # npm automation token
|
||
r'|dckr_pat_[A-Za-z0-9_\-]{20,}' # Docker Hub token
|
||
r'|hf_[A-Za-z0-9]{30,}' # Hugging Face token
|
||
r'|AIza[0-9A-Za-z_\-]{35}' # Google API key
|
||
r'|ya29\.[0-9A-Za-z_\-]{20,}' # Google OAuth access token
|
||
r'|xox[baprs]-[0-9A-Za-z\-]{10,}' # Slack token
|
||
r'|SG\.[A-Za-z0-9_\-]{16,}' # SendGrid API key
|
||
r'|re_[A-Za-z0-9]{40}' # Resend API key
|
||
r'|sq0[a-z]{3}-[A-Za-z0-9_\-]{16,}' # Square access token
|
||
r'|ATATT[A-Za-z0-9_\-]{20,}' # Atlassian access token
|
||
r'|[NOP][A-Za-z0-9_\-]{23,28}\.[A-Za-z0-9_\-]{6}\.[A-Za-z0-9_\-]{27,}' # Discord bot token
|
||
r'|\d{8,10}:[A-Za-z0-9_\-]{35}' # Telegram bot token
|
||
r'|Bearer\s+[A-Za-z0-9._~+/=\-]{20,}' # Authorization: Bearer …
|
||
r')'),
|
||
'[CLÉ API MASQUÉE]', 0),
|
||
|
||
]
|
||
|
||
# BUG-035: bare 40–64 char hex strings used to be redacted unconditionally,
|
||
# which mangled legitimate git commit SHAs, checksums and hashes in notes.
|
||
# They are now only redacted when a secret-ish keyword sits in the immediate
|
||
# context; hash/commit keywords explicitly exempt them.
|
||
_HEX_RE = re.compile(r'\b[a-fA-F0-9]{40,64}\b')
|
||
_SECRET_CONTEXT_RE = re.compile(
|
||
r'(?i)\b(?:secret|token|key|apikey|api[_-]?key|password|passwd|auth|bearer|'
|
||
r'credential|x-api-key|x-auth-token)\b'
|
||
)
|
||
_HASH_CONTEXT_RE = re.compile(
|
||
r'(?i)\b(?:commit|sha\d*|hash|md5|blob|git|checksum|digest|integrity|'
|
||
r'revision|rev|etag|fingerprint)\b'
|
||
)
|
||
#: How far before the hex string a keyword may appear to count as context.
|
||
_HEX_CONTEXT_WINDOW = 60
|
||
|
||
# --- Interactive masking (feature #188) ---
|
||
# Private-use-area sentinels: they survive markdown rendering (mistune
|
||
# treats them as plain text, fenced code blocks included) and are
|
||
# stripped by ``backend.render._heading_slugify``.
|
||
_PLACEHOLDER_OPEN = "\uE000"
|
||
_PLACEHOLDER_CLOSE = "\uE001"
|
||
_PLACEHOLDER_RE = re.compile("\uE000(\\d+)\uE001")
|
||
|
||
|
||
def _redact_bare_hex_secrets(text: str, mask) -> tuple[str, int]:
|
||
"""Redact 40–64 char hex strings only when a secret keyword is nearby.
|
||
|
||
Git/SHA/checksum contexts are left untouched (BUG-035).
|
||
|
||
Args:
|
||
text: Text to scan.
|
||
mask: ``mask(original, label) -> str`` replacement builder.
|
||
|
||
Returns:
|
||
(redacted_text, redaction_count) tuple.
|
||
"""
|
||
count = 0
|
||
|
||
def _replace(match: re.Match) -> str:
|
||
nonlocal count
|
||
window = text[max(0, match.start() - _HEX_CONTEXT_WINDOW):match.start()]
|
||
if _HASH_CONTEXT_RE.search(window):
|
||
return match.group(0)
|
||
if _SECRET_CONTEXT_RE.search(window):
|
||
count += 1
|
||
return mask(match.group(0), '[HEX_KEY MASQUÉ]')
|
||
return match.group(0)
|
||
|
||
return _HEX_RE.sub(_replace, text), count
|
||
|
||
|
||
def _redact(text: str, mask) -> tuple[str, int]:
|
||
"""Apply every pattern; ``mask(original, label) -> str`` builds the
|
||
replacement (plain label, or placeholder for the interactive mode)."""
|
||
count = 0
|
||
result = text
|
||
for pattern, replacement, secret_group in _PATTERNS:
|
||
|
||
def _sub(match: re.Match, replacement=replacement, secret_group=secret_group) -> str:
|
||
if callable(replacement):
|
||
label = replacement(match)
|
||
else:
|
||
label = match.expand(str(replacement))
|
||
return mask(match.group(secret_group), label)
|
||
|
||
new_result, n = pattern.subn(_sub, result)
|
||
count += n
|
||
result = new_result
|
||
result, hex_count = _redact_bare_hex_secrets(result, mask)
|
||
return result, count + hex_count
|
||
|
||
|
||
def _plain_mask(original: str, label: str) -> str:
|
||
"""Plain masking: only the visible label survives."""
|
||
return label
|
||
|
||
|
||
def redact(text: str) -> tuple:
|
||
"""Redact sensitive patterns from text.
|
||
|
||
Args:
|
||
text: The raw text content to scan.
|
||
|
||
Returns:
|
||
(redacted_text, redaction_count) tuple.
|
||
"""
|
||
result, count = _redact(text, _plain_mask)
|
||
if count > 0:
|
||
logger.info(f"Redacted {count} secret(s) from content")
|
||
return result, count
|
||
|
||
|
||
def redact_with_placeholders(text: str, file_path: str = "") -> tuple[str, list[tuple[str, str]]]:
|
||
"""Redact *text*, replacing every mask with an opaque placeholder.
|
||
|
||
Used by the markdown rendering pipeline: placeholders survive the
|
||
markdown → HTML conversion (fenced code blocks included, where a
|
||
literal ``<span>`` would be shown as text), then
|
||
:func:`restore_masks` turns them back into labels or clickable badges.
|
||
|
||
Args:
|
||
text: The raw text content to scan.
|
||
file_path: Optional file path for logging context.
|
||
|
||
Returns:
|
||
(text_with_placeholders, entries) where *entries* is the list of
|
||
``(label, secret)`` tuples referenced by the placeholders, in
|
||
order of appearance.
|
||
"""
|
||
entries: list[tuple[str, str]] = []
|
||
|
||
def mask(original: str, label: str) -> str:
|
||
entries.append((label, original))
|
||
return f"{_PLACEHOLDER_OPEN}{len(entries) - 1}{_PLACEHOLDER_CLOSE}"
|
||
|
||
result, count = _redact(text, mask)
|
||
if count > 0:
|
||
logger.warning(f"Redacted {count} potential secret(s) from {file_path or '<unknown>'}")
|
||
return result, entries
|
||
|
||
|
||
def restore_masks(
|
||
text: str,
|
||
entries: list[tuple[str, str]],
|
||
*,
|
||
click_to_copy: bool = False,
|
||
) -> str:
|
||
"""Turn placeholders produced by :func:`redact_with_placeholders` back
|
||
into visible masks.
|
||
|
||
Args:
|
||
text: Rendered HTML still containing placeholders.
|
||
entries: The ``(label, secret)`` list returned alongside.
|
||
click_to_copy: When True (authenticated app preview), each mask
|
||
becomes ``<span class="secret-mask" data-secret="…">label</span>``
|
||
so a click copies the real value. When False (public shares,
|
||
PDF exports), only the plain label is restored — the secret
|
||
never reaches the page.
|
||
|
||
Returns:
|
||
The text with every placeholder replaced.
|
||
"""
|
||
if not entries:
|
||
return text
|
||
|
||
def _sub(match: re.Match) -> str:
|
||
idx = int(match.group(1))
|
||
if idx >= len(entries):
|
||
return ""
|
||
label, original = entries[idx]
|
||
label_esc = _html.escape(str(label), quote=False)
|
||
if not click_to_copy:
|
||
return label_esc
|
||
return (
|
||
'<span class="secret-mask" data-secret="'
|
||
+ _html.escape(str(original), quote=True)
|
||
+ '">'
|
||
+ label_esc
|
||
+ "</span>"
|
||
)
|
||
|
||
return _PLACEHOLDER_RE.sub(_sub, text)
|
||
|
||
|
||
def redact_file_content(content: str, file_path: str = "") -> str:
|
||
"""Redact a file's content for preview rendering.
|
||
|
||
Args:
|
||
content: Raw file content.
|
||
file_path: Optional file path for logging context.
|
||
|
||
Returns:
|
||
Redacted content string.
|
||
"""
|
||
redacted, count = redact(content)
|
||
if count > 0:
|
||
logger.warning(f"Redacted {count} potential secret(s) from {file_path or '<unknown>'}")
|
||
return redacted
|