agnes-the-ai-analyst/app/api/scripts.py
ZdenekSrotyr e9d7af3cce feat(rbac+marketplace): RBAC v13 + Claude Code marketplace + #81/#83/#44 hardening
This squashes 13 commits from ma/staging plus a small docstring translation
into a single coherent unit. Three workstreams.

== RBAC v13 redesign ==
- Drops core.viewer/analyst/km_admin/admin hierarchy and the
  internal_roles / group_mappings / user_role_grants / plugin_access tables.
- Replaced by user_group_members + resource_grants. Atomic v12→v13 backfill
  wrapped in BEGIN/COMMIT; ROLLBACK leaves schema_version at 12 for retry.
- Two authorization primitives in app.auth.access:
    require_admin                        — Admin-group god-mode
    require_resource_access(rt, "{path}") — entity-scoped grants
  Single DB lookup per request; no session cache; no implies BFS.
- /admin/access UI (single page) replaces /admin/role-mapping +
  /admin/plugin-access. CLI `da admin group/grant *` replaces
  `da admin role/mapping/grant-role/revoke-role/effective-roles`.
- ResourceType.TABLE listing-only — admins can record table grants,
  runtime enforcement still flows through legacy dataset_permissions
  (migration plan in docs/TODO-rbac-data-enforcement.md).

== Claude Code marketplace ==
- Aggregated /marketplace.zip + /marketplace.git/* (PAT-gated,
  RBAC-filtered, content-addressed cache via dulwich).
- Admin god-mode dropped on the marketplace surface — admins curate
  their own view via grants like everyone else.
- Bare-repo cache materializes per RBAC-filtered ETag; stale entries
  not pruned in this iteration (disclaimed in git_backend.py docstring).

== #81 #83 #44 security/ops hardening ==
- #81 Group A — orchestrator ATTACH allow-listing (extension/url/alias).
- #81 Group B — Keboola extractor 3-state exit codes:
    0 success / 1 total fail / 2 PARTIAL fail
  Sync API logs PARTIAL FAILURE alert on exit 2. Operators with binary
  alerting must teach it the new partial signal.
- #81 Group C — schema v10 view_ownership; rejects silent overwrite
  of a prior connector's view name on collision.
- #81 Group D — extractor-side identifier validation.
- #83 — Jira webhook fail-closed when JIRA_WEBHOOK_SECRET unset
  + path-traversal fix.
- #44 — entire /api/scripts/* surface is admin-only (planted-script +
  sandbox-bypass risk closed).

== Web UI polish + deploy fix ==
- /admin/access: live grant-count badges (no stale snapshot revert),
  shared-header CSS link added to /catalog and /admin/{tables,permissions},
  per-resource-type colored stripes.
- docker-compose.host-mount.yml: bind,rbind so dual-disk hosts don't
  silently shadow sub-mounts and write state to the wrong disk.

== OSS vendor-neutralization (waves 1+2) ==
- scripts/grpn/ → scripts/ops/. Customer-specific identifiers
  (project IDs, internal hostnames, dev/prod VM IPs, brand names)
  replaced with placeholders across code, docs, Terraform, Caddyfile,
  OAuth probe, and planning docs. Downstream infra repos that copied
  scripts/grpn/agnes-tls-rotate.sh or agnes-auto-upgrade.sh must
  update the path.

== Translation ==
- src/repositories/user_groups.py::ensure_system docstring translated
  from Czech to English for codebase consistency.

Co-authored-by: Mina Rustamyan <mina@keboola.com>
2026-04-28 14:25:04 +02:00

237 lines
7.6 KiB
Python

"""Script management and execution endpoints."""
import os
import subprocess
import sys
import tempfile
import uuid
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from typing import Optional
import duckdb
from app.auth.access import require_admin
from app.auth.dependencies import _get_db
from src.repositories.notifications import ScriptRepository
router = APIRouter(prefix="/api/scripts", tags=["scripts"])
SCRIPT_TIMEOUT = int(os.environ.get("SCRIPT_TIMEOUT", "300")) # 5 min default
SCRIPT_MAX_OUTPUT = int(os.environ.get("SCRIPT_MAX_OUTPUT", "65536")) # 64KB
class DeployScriptRequest(BaseModel):
name: str
source: str
schedule: Optional[str] = None
class RunScriptRequest(BaseModel):
name: Optional[str] = None
source: Optional[str] = None
class ScriptResponse(BaseModel):
id: str
name: str
schedule: Optional[str]
owner: Optional[str]
@router.get("")
async def list_scripts(
user: dict = Depends(require_admin),
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
"""List deployed scripts. Admin-only."""
repo = ScriptRepository(conn)
scripts = repo.list_all()
return {"scripts": scripts, "count": len(scripts)}
@router.post("/deploy", status_code=201)
async def deploy_script(
request: DeployScriptRequest,
user: dict = Depends(require_admin),
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
"""Deploy a Python script to be run on the server (optionally on schedule). Admin-only."""
repo = ScriptRepository(conn)
script_id = str(uuid.uuid4())
repo.deploy(
id=script_id,
name=request.name,
owner=user["id"],
schedule=request.schedule,
source=request.source,
)
return ScriptResponse(
id=script_id, name=request.name,
schedule=request.schedule, owner=user["id"],
)
@router.post("/{script_id}/run")
async def run_deployed_script(
script_id: str,
user: dict = Depends(require_admin),
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
"""Run a deployed script by ID. Admin-only."""
repo = ScriptRepository(conn)
script = repo.get(script_id)
if not script:
raise HTTPException(status_code=404, detail="Script not found")
return _execute_script(script["source"], script["name"])
@router.post("/run")
async def run_adhoc_script(
request: RunScriptRequest,
user: dict = Depends(require_admin),
):
"""Run an ad-hoc Python script (not deployed). Admin-only."""
if not request.source:
raise HTTPException(status_code=400, detail="Script source required")
return _execute_script(request.source, request.name or "adhoc")
@router.delete("/{script_id}", status_code=204)
async def undeploy_script(
script_id: str,
user: dict = Depends(require_admin),
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
repo = ScriptRepository(conn)
if not repo.get(script_id):
raise HTTPException(status_code=404, detail="Script not found")
repo.undeploy(script_id)
def _execute_script(source: str, name: str) -> dict:
"""Execute a Python script in a sandboxed subprocess.
The blocklist below is defense-in-depth, not a primary trust boundary.
The role gate on the route (admin-only) is the actual boundary; the
blocklist catches obvious mistakes, not a hostile admin."""
# Comprehensive safety checks — block dangerous patterns
blocked_patterns = [
# Direct imports of dangerous modules
"import subprocess", "from subprocess",
"import shutil", "from shutil",
"import ctypes", "from ctypes",
"import importlib", "from importlib",
"import socket", "from socket",
"import requests", "from requests",
"import httpx", "from httpx",
"import urllib", "from urllib",
"import http", "from http",
# Dynamic import bypasses
"__import__",
"importlib",
# Code execution bypasses
"exec(",
"eval(",
"compile(",
# OS-level access
"import os", "from os",
"import sys", "from sys",
"import signal", "from signal",
# File access bypasses
"open(",
"pathlib",
# Dangerous builtins
"globals()",
"locals()",
"getattr(",
"setattr(",
"delattr(",
"breakpoint(",
# Introspection-chain dunders that can pivot to RCE.
# `__init__`/`__getattribute__` deliberately omitted: substring
# match would flag every `def __init__(self):`.
"__subclasses__",
"__globals__",
"__class__",
"__base__",
"__bases__",
"__mro__",
"__dict__",
"__code__",
"__builtins__",
]
import ast
BLOCKED_MODULES = {"os", "sys", "subprocess", "shutil", "ctypes", "importlib", "socket",
"requests", "httpx", "urllib", "http", "signal", "pathlib", "builtins"}
BLOCKED_FUNCTIONS = {"exec", "eval", "compile", "open", "globals", "locals",
"getattr", "setattr", "delattr", "breakpoint", "__import__",
"vars"}
try:
tree = ast.parse(source)
except SyntaxError as e:
raise HTTPException(status_code=400, detail=f"Script syntax error: {e}")
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
if alias.name.split(".")[0] in BLOCKED_MODULES:
raise HTTPException(status_code=400, detail=f"Blocked import: {alias.name}")
elif isinstance(node, ast.ImportFrom):
if node.module and node.module.split(".")[0] in BLOCKED_MODULES:
raise HTTPException(status_code=400, detail=f"Blocked import: {node.module}")
elif isinstance(node, ast.Call):
if isinstance(node.func, ast.Name) and node.func.id in BLOCKED_FUNCTIONS:
raise HTTPException(status_code=400, detail=f"Blocked function: {node.func.id}")
source_lower = source.lower()
for pattern in blocked_patterns:
if pattern.lower() in source_lower:
raise HTTPException(
status_code=400,
detail=f"Script contains disallowed pattern: {pattern.split('(')[0].strip()}",
)
data_dir = os.environ.get("DATA_DIR", "./data")
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
f.write(source)
f.flush()
script_path = f.name
try:
result = subprocess.run(
[sys.executable, script_path],
capture_output=True,
text=True,
timeout=SCRIPT_TIMEOUT,
env={
"PATH": "/usr/bin:/usr/local/bin",
"DATA_DIR": data_dir,
"HOME": "/tmp",
# Deliberately exclude VIRTUAL_ENV and PYTHONPATH
# to prevent access to installed packages
},
cwd="/tmp", # restrict working directory
)
stdout = result.stdout[:SCRIPT_MAX_OUTPUT]
stderr = result.stderr[:SCRIPT_MAX_OUTPUT]
return {
"name": name,
"exit_code": result.returncode,
"stdout": stdout,
"stderr": stderr,
"truncated": len(result.stdout) > SCRIPT_MAX_OUTPUT or len(result.stderr) > SCRIPT_MAX_OUTPUT,
}
except subprocess.TimeoutExpired:
return {
"name": name,
"exit_code": -1,
"stdout": "",
"stderr": f"Script timed out after {SCRIPT_TIMEOUT}s",
"truncated": False,
}
finally:
os.unlink(script_path)