995e4cd366
* fix(scheduler): HTTP marketplaces job + SCHEDULER_API_TOKEN shared secret Two scheduler-reliability bugs surfaced after the v0.12.1 USER-agnes flip: 1. The marketplaces job called src.marketplace.sync_marketplaces() in-process from the scheduler container, racing the app's long-lived system.duckdb handle. DuckDB rejects cross-process writers — every cron tick 500-ed on "Could not set lock on file ... PID 0". 2. The data-refresh + new marketplaces jobs both 401-ed on the API because SCHEDULER_API_TOKEN was never propagated by the Terraform startup script. The scheduler had no credential to authenticate with. Fix: - New POST /api/marketplaces/sync-all (admin-only) drives the nightly refresh through the app process so it inherits the existing DB connection. - Scheduler swaps fn->http for marketplaces; all jobs are now plain HTTP and the scheduler is reduced to a cron clock. - New app/auth/scheduler_token.py adds a shared-secret auth path. The startup script generates a 256-bit secret on first boot, persists it across reboots, and writes it to /opt/agnes/.env. Both containers source the same .env. The app validates incoming Bearer tokens against the env var (constant-time, length-floored) and resolves matches to a synthetic scheduler@system.local user that's a member of the Admin system group. Audit-log entries from the scheduler are attributed to this user. - app/main.py seeds the synthetic user at startup so the first cron tick has a valid actor; lazy seed in get_scheduler_user covers token rotation before the next app restart. Tests: 5 new in tests/test_auth_scheduler_token.py covering empty/short secret rejection, exact-match comparison, idempotent user seeding, and lazy provisioning. 142 marketplace + scheduler tests + 96 auth tests remain green. Existing VMs with .env from before this change need a one-time re-provisioning (re-run startup-script or rotate via openssl rand); documented in CHANGELOG. * fix(audit): use '_all' sentinel for bulk marketplace sync — Devin review #127 Avoids the literal string 'marketplace:None' in the audit_log resource column when the bulk sync endpoint writes its summary row. * fix(scheduler): unblock event loop + per-job timeouts — Devin review #127 Two findings from Devin re-review on commit 5fbad15: 1. BUG: trigger_sync_all was async def, so FastAPI ran it on the asyncio event loop. sync_marketplaces() does blocking I/O (subprocess git clones up to GIT_TIMEOUT_SEC=300 each, threading.Lock, DuckDB writes) and would freeze every concurrent request for the duration of a bulk sync. Switched to plain def so FastAPI auto-routes to the thread pool. 2. ANALYSIS: scheduler used a fixed 120s httpx timeout for every POST. Bulk marketplace sync iterates the registry under a single lock with up to 300s per repo — easily exceeds 120s on 2-3 slow repos. The scheduler then sees a timeout, doesn't update last_run, and re-fires on the next 30s tick, queueing redundant work. Per-job timeout override added to the JOBS tuple; marketplaces gets 900s (15 min), data-refresh keeps 120s, health-check 30s. * fix(auth): require_session_token rejects scheduler shared secret — Devin review #127 require_session_token gates /auth/tokens (PAT minting). Pre-fix it only rejected JWTs with typ=pat — but the scheduler shared secret is an opaque string, so verify_token() returns None, payload becomes {}, and the PAT-claim check silently passed. A caller bearing SCHEDULER_API_TOKEN could mint persistent PATs that survive a secret rotation. Added explicit is_scheduler_token() check before the PAT-claim check; new regression test in tests/test_auth_scheduler_token.py. Devin's other note (pre-existing async def trigger_sync at marketplaces.py:392 also calls blocking sync_one) — Devin flagged it as out-of-scope for this PR and I agree; tracking separately. * release(0.17.0): cut + clean up CHANGELOG duplicates Cuts 0.17.0 (minor: scheduler shared-secret auth + sync-all endpoint plus the deploy-shape fixes that landed since the last release tag). Bumps pyproject from 0.15.0 — also corrects the missed bump from PR #120 (v0.16.0 was tagged on GitHub and shipped as :stable, but pyproject stayed at 0.15.0, so /api/version, /cli/latest, and `da --version` had been under-reporting the running release). Removes the long-form duplicate entries for 0.13.0 / 0.14.0 / 0.15.0 above [0.16.0] — the canonical short summaries (with GitHub-release links) already exist below 0.16.0, the long forms were leftover state from before those versions were cut and have been silently shadowed ever since.
95 lines
2.8 KiB
TOML
95 lines
2.8 KiB
TOML
[project]
|
|
name = "agnes-the-ai-analyst"
|
|
version = "0.17.0"
|
|
description = "Agnes — AI Data Analyst platform for AI analytical systems"
|
|
requires-python = ">=3.11,<3.14"
|
|
license = "MIT"
|
|
readme = "README.md"
|
|
|
|
dependencies = [
|
|
# Core database
|
|
"duckdb>=0.9.0",
|
|
# Web framework (FastAPI)
|
|
"fastapi>=0.115.0",
|
|
"uvicorn[standard]>=0.32.0",
|
|
"python-multipart>=0.0.26",
|
|
"jinja2>=3.1.0",
|
|
"starlette>=0.41.0",
|
|
# Authentication
|
|
"PyJWT>=2.8.0",
|
|
"itsdangerous>=2.1.0",
|
|
"authlib>=1.6.11",
|
|
"argon2-cffi>=23.1.0",
|
|
# HTTP client
|
|
"httpx>=0.27.0",
|
|
# CLI
|
|
"typer>=0.12.0",
|
|
"rich>=13.0.0",
|
|
# Configuration
|
|
"python-dotenv>=1.0.0",
|
|
"pyyaml>=6.0",
|
|
# Data processing
|
|
"pandas>=2.0.0",
|
|
"pyarrow>=12.0.0",
|
|
"pytz>=2024.1",
|
|
# SQL parsing — server-side WHERE validator for /api/v2/scan (app/api/where_validator.py)
|
|
# Minimum 30.x — older versions had walk() yielding (node, parent, key)
|
|
# tuples instead of expression nodes, which would silently bypass the
|
|
# WHERE-validator structural checks (isinstance(tuple, exp.Subquery)
|
|
# is always False). 30.x yields nodes directly.
|
|
"sqlglot>=30.0.0",
|
|
# Data source connectors
|
|
"google-cloud-bigquery>=3.0.0",
|
|
"google-cloud-bigquery-storage>=2.0.0",
|
|
# Google Workspace Cloud Identity / Admin SDK (Workspace group membership sync)
|
|
"google-api-python-client>=2.0.0",
|
|
# Profiler visualizations
|
|
"matplotlib>=3.8.0",
|
|
"numpy>=1.24.0",
|
|
# Claude Code marketplace endpoint — pure-Python git server mounted in FastAPI
|
|
"dulwich>=0.22.0",
|
|
"a2wsgi>=1.10.0",
|
|
# In-process TTL cache for marketplace etag (transitively present via
|
|
# google-auth, declared explicitly here because we depend on it directly).
|
|
"cachetools>=5.3.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
# keboola-legacy: install kbcstorage>=0.9.0 manually if you need the legacy
|
|
# Keboola client fallback (primary path uses DuckDB Keboola extension)
|
|
dev = [
|
|
"pytest>=9.0.0",
|
|
"pytest-timeout>=2.0.0",
|
|
"pytest-xdist>=3.0.0",
|
|
"faker>=24.0.0",
|
|
"anthropic>=0.30.0",
|
|
"openai>=1.30.0",
|
|
# jsonschema validates the corporate-memory extraction-tool golden fixtures
|
|
# under tests/test_corporate_memory_v1.py (extraction.json, correction.json,
|
|
# confidence_calibration.json). Production code does not depend on it.
|
|
"jsonschema>=4.0.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
da = "cli.main:app"
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["app", "src", "connectors", "cli", "services", "config"]
|
|
|
|
[tool.ruff]
|
|
line-length = 120
|
|
target-version = "py313"
|
|
|
|
[tool.uv]
|
|
dev-dependencies = [
|
|
"pytest>=9.0.0",
|
|
"pytest-timeout>=2.0.0",
|
|
"pytest-xdist>=3.0.0",
|
|
"faker>=24.0.0",
|
|
"anthropic>=0.30.0",
|
|
"openai>=1.30.0",
|
|
]
|