This squashes 13 commits from ma/staging plus a small docstring translation
into a single coherent unit. Three workstreams.
== RBAC v13 redesign ==
- Drops core.viewer/analyst/km_admin/admin hierarchy and the
internal_roles / group_mappings / user_role_grants / plugin_access tables.
- Replaced by user_group_members + resource_grants. Atomic v12→v13 backfill
wrapped in BEGIN/COMMIT; ROLLBACK leaves schema_version at 12 for retry.
- Two authorization primitives in app.auth.access:
require_admin — Admin-group god-mode
require_resource_access(rt, "{path}") — entity-scoped grants
Single DB lookup per request; no session cache; no implies BFS.
- /admin/access UI (single page) replaces /admin/role-mapping +
/admin/plugin-access. CLI `da admin group/grant *` replaces
`da admin role/mapping/grant-role/revoke-role/effective-roles`.
- ResourceType.TABLE listing-only — admins can record table grants,
runtime enforcement still flows through legacy dataset_permissions
(migration plan in docs/TODO-rbac-data-enforcement.md).
== Claude Code marketplace ==
- Aggregated /marketplace.zip + /marketplace.git/* (PAT-gated,
RBAC-filtered, content-addressed cache via dulwich).
- Admin god-mode dropped on the marketplace surface — admins curate
their own view via grants like everyone else.
- Bare-repo cache materializes per RBAC-filtered ETag; stale entries
not pruned in this iteration (disclaimed in git_backend.py docstring).
== #81 #83 #44 security/ops hardening ==
- #81 Group A — orchestrator ATTACH allow-listing (extension/url/alias).
- #81 Group B — Keboola extractor 3-state exit codes:
0 success / 1 total fail / 2 PARTIAL fail
Sync API logs PARTIAL FAILURE alert on exit 2. Operators with binary
alerting must teach it the new partial signal.
- #81 Group C — schema v10 view_ownership; rejects silent overwrite
of a prior connector's view name on collision.
- #81 Group D — extractor-side identifier validation.
- #83 — Jira webhook fail-closed when JIRA_WEBHOOK_SECRET unset
+ path-traversal fix.
- #44 — entire /api/scripts/* surface is admin-only (planted-script +
sandbox-bypass risk closed).
== Web UI polish + deploy fix ==
- /admin/access: live grant-count badges (no stale snapshot revert),
shared-header CSS link added to /catalog and /admin/{tables,permissions},
per-resource-type colored stripes.
- docker-compose.host-mount.yml: bind,rbind so dual-disk hosts don't
silently shadow sub-mounts and write state to the wrong disk.
== OSS vendor-neutralization (waves 1+2) ==
- scripts/grpn/ → scripts/ops/. Customer-specific identifiers
(project IDs, internal hostnames, dev/prod VM IPs, brand names)
replaced with placeholders across code, docs, Terraform, Caddyfile,
OAuth probe, and planning docs. Downstream infra repos that copied
scripts/grpn/agnes-tls-rotate.sh or agnes-auto-upgrade.sh must
update the path.
== Translation ==
- src/repositories/user_groups.py::ensure_system docstring translated
from Czech to English for codebase consistency.
Co-authored-by: Mina Rustamyan <mina@keboola.com>
237 lines
7.6 KiB
Python
237 lines
7.6 KiB
Python
"""Script management and execution endpoints."""
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import uuid
|
|
from fastapi import APIRouter, Depends, HTTPException
|
|
from pydantic import BaseModel
|
|
from typing import Optional
|
|
|
|
import duckdb
|
|
|
|
from app.auth.access import require_admin
|
|
from app.auth.dependencies import _get_db
|
|
from src.repositories.notifications import ScriptRepository
|
|
|
|
router = APIRouter(prefix="/api/scripts", tags=["scripts"])
|
|
|
|
SCRIPT_TIMEOUT = int(os.environ.get("SCRIPT_TIMEOUT", "300")) # 5 min default
|
|
SCRIPT_MAX_OUTPUT = int(os.environ.get("SCRIPT_MAX_OUTPUT", "65536")) # 64KB
|
|
|
|
|
|
class DeployScriptRequest(BaseModel):
|
|
name: str
|
|
source: str
|
|
schedule: Optional[str] = None
|
|
|
|
|
|
class RunScriptRequest(BaseModel):
|
|
name: Optional[str] = None
|
|
source: Optional[str] = None
|
|
|
|
|
|
class ScriptResponse(BaseModel):
|
|
id: str
|
|
name: str
|
|
schedule: Optional[str]
|
|
owner: Optional[str]
|
|
|
|
|
|
@router.get("")
|
|
async def list_scripts(
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
"""List deployed scripts. Admin-only."""
|
|
repo = ScriptRepository(conn)
|
|
scripts = repo.list_all()
|
|
return {"scripts": scripts, "count": len(scripts)}
|
|
|
|
|
|
@router.post("/deploy", status_code=201)
|
|
async def deploy_script(
|
|
request: DeployScriptRequest,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
"""Deploy a Python script to be run on the server (optionally on schedule). Admin-only."""
|
|
repo = ScriptRepository(conn)
|
|
script_id = str(uuid.uuid4())
|
|
repo.deploy(
|
|
id=script_id,
|
|
name=request.name,
|
|
owner=user["id"],
|
|
schedule=request.schedule,
|
|
source=request.source,
|
|
)
|
|
return ScriptResponse(
|
|
id=script_id, name=request.name,
|
|
schedule=request.schedule, owner=user["id"],
|
|
)
|
|
|
|
|
|
@router.post("/{script_id}/run")
|
|
async def run_deployed_script(
|
|
script_id: str,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
"""Run a deployed script by ID. Admin-only."""
|
|
repo = ScriptRepository(conn)
|
|
script = repo.get(script_id)
|
|
if not script:
|
|
raise HTTPException(status_code=404, detail="Script not found")
|
|
return _execute_script(script["source"], script["name"])
|
|
|
|
|
|
@router.post("/run")
|
|
async def run_adhoc_script(
|
|
request: RunScriptRequest,
|
|
user: dict = Depends(require_admin),
|
|
):
|
|
"""Run an ad-hoc Python script (not deployed). Admin-only."""
|
|
if not request.source:
|
|
raise HTTPException(status_code=400, detail="Script source required")
|
|
return _execute_script(request.source, request.name or "adhoc")
|
|
|
|
|
|
@router.delete("/{script_id}", status_code=204)
|
|
async def undeploy_script(
|
|
script_id: str,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
repo = ScriptRepository(conn)
|
|
if not repo.get(script_id):
|
|
raise HTTPException(status_code=404, detail="Script not found")
|
|
repo.undeploy(script_id)
|
|
|
|
|
|
def _execute_script(source: str, name: str) -> dict:
|
|
"""Execute a Python script in a sandboxed subprocess.
|
|
|
|
The blocklist below is defense-in-depth, not a primary trust boundary.
|
|
The role gate on the route (admin-only) is the actual boundary; the
|
|
blocklist catches obvious mistakes, not a hostile admin."""
|
|
# Comprehensive safety checks — block dangerous patterns
|
|
blocked_patterns = [
|
|
# Direct imports of dangerous modules
|
|
"import subprocess", "from subprocess",
|
|
"import shutil", "from shutil",
|
|
"import ctypes", "from ctypes",
|
|
"import importlib", "from importlib",
|
|
"import socket", "from socket",
|
|
"import requests", "from requests",
|
|
"import httpx", "from httpx",
|
|
"import urllib", "from urllib",
|
|
"import http", "from http",
|
|
# Dynamic import bypasses
|
|
"__import__",
|
|
"importlib",
|
|
# Code execution bypasses
|
|
"exec(",
|
|
"eval(",
|
|
"compile(",
|
|
# OS-level access
|
|
"import os", "from os",
|
|
"import sys", "from sys",
|
|
"import signal", "from signal",
|
|
# File access bypasses
|
|
"open(",
|
|
"pathlib",
|
|
# Dangerous builtins
|
|
"globals()",
|
|
"locals()",
|
|
"getattr(",
|
|
"setattr(",
|
|
"delattr(",
|
|
"breakpoint(",
|
|
# Introspection-chain dunders that can pivot to RCE.
|
|
# `__init__`/`__getattribute__` deliberately omitted: substring
|
|
# match would flag every `def __init__(self):`.
|
|
"__subclasses__",
|
|
"__globals__",
|
|
"__class__",
|
|
"__base__",
|
|
"__bases__",
|
|
"__mro__",
|
|
"__dict__",
|
|
"__code__",
|
|
"__builtins__",
|
|
]
|
|
import ast
|
|
|
|
BLOCKED_MODULES = {"os", "sys", "subprocess", "shutil", "ctypes", "importlib", "socket",
|
|
"requests", "httpx", "urllib", "http", "signal", "pathlib", "builtins"}
|
|
BLOCKED_FUNCTIONS = {"exec", "eval", "compile", "open", "globals", "locals",
|
|
"getattr", "setattr", "delattr", "breakpoint", "__import__",
|
|
"vars"}
|
|
|
|
try:
|
|
tree = ast.parse(source)
|
|
except SyntaxError as e:
|
|
raise HTTPException(status_code=400, detail=f"Script syntax error: {e}")
|
|
|
|
for node in ast.walk(tree):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
if alias.name.split(".")[0] in BLOCKED_MODULES:
|
|
raise HTTPException(status_code=400, detail=f"Blocked import: {alias.name}")
|
|
elif isinstance(node, ast.ImportFrom):
|
|
if node.module and node.module.split(".")[0] in BLOCKED_MODULES:
|
|
raise HTTPException(status_code=400, detail=f"Blocked import: {node.module}")
|
|
elif isinstance(node, ast.Call):
|
|
if isinstance(node.func, ast.Name) and node.func.id in BLOCKED_FUNCTIONS:
|
|
raise HTTPException(status_code=400, detail=f"Blocked function: {node.func.id}")
|
|
|
|
source_lower = source.lower()
|
|
for pattern in blocked_patterns:
|
|
if pattern.lower() in source_lower:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Script contains disallowed pattern: {pattern.split('(')[0].strip()}",
|
|
)
|
|
|
|
data_dir = os.environ.get("DATA_DIR", "./data")
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
|
|
f.write(source)
|
|
f.flush()
|
|
script_path = f.name
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
[sys.executable, script_path],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=SCRIPT_TIMEOUT,
|
|
env={
|
|
"PATH": "/usr/bin:/usr/local/bin",
|
|
"DATA_DIR": data_dir,
|
|
"HOME": "/tmp",
|
|
# Deliberately exclude VIRTUAL_ENV and PYTHONPATH
|
|
# to prevent access to installed packages
|
|
},
|
|
cwd="/tmp", # restrict working directory
|
|
)
|
|
stdout = result.stdout[:SCRIPT_MAX_OUTPUT]
|
|
stderr = result.stderr[:SCRIPT_MAX_OUTPUT]
|
|
return {
|
|
"name": name,
|
|
"exit_code": result.returncode,
|
|
"stdout": stdout,
|
|
"stderr": stderr,
|
|
"truncated": len(result.stdout) > SCRIPT_MAX_OUTPUT or len(result.stderr) > SCRIPT_MAX_OUTPUT,
|
|
}
|
|
except subprocess.TimeoutExpired:
|
|
return {
|
|
"name": name,
|
|
"exit_code": -1,
|
|
"stdout": "",
|
|
"stderr": f"Script timed out after {SCRIPT_TIMEOUT}s",
|
|
"truncated": False,
|
|
}
|
|
finally:
|
|
os.unlink(script_path)
|