From a48524509ad277a52ff41288d5895d2f994f92c8 Mon Sep 17 00:00:00 2001 From: ZdenekSrotyr <139972147+ZdenekSrotyr@users.noreply.github.com> Date: Thu, 14 May 2026 20:54:22 +0200 Subject: [PATCH] docs: consolidate and de-clutter the documentation tree (#306) CLAUDE.md rewritten (708 -> ~320 lines): four overlapping release sections collapsed to one, stale v1->v35 schema history dropped (it lives in CHANGELOG), marketplace endpoint internals and verbose process sections moved out or tightened. New focused docs: - docs/RELEASING.md - release process, deploy workflows, CI quirks (RELEASE_TEMPLATE.md folded in as an appendix) - docs/marketplace.md - marketplace ingestion + re-serving internals - docs/README.md - documentation index by audience, linked from README.md and CLAUDE.md Archived under docs/archive/: docs/superpowers/ (52 historical planning artifacts), HACKATHON.md, pd-ps-comments.md, security-audit-2026-04.md, future/NOTIFICATIONS.md. Removed the docs/auto-install.md stub. Fixed dangling links in connectors/jira/README.md and dev_docs/README.md, repointed code/doc references to archived paths. --- .gitignore | 1 + AGENTS.md | 7 + CHANGELOG.md | 3 + CLAUDE.md | 549 +++--------------- README.md | 7 +- connectors/bigquery/access.py | 4 +- connectors/jira/README.md | 2 +- dev_docs/README.md | 36 +- docs/QUICKSTART.md | 2 +- docs/README.md | 73 +++ docs/RELEASE_CHECKLIST.md | 2 +- docs/RELEASE_TEMPLATE.md | 37 -- docs/RELEASING.md | 270 +++++++++ docs/{ => archive}/HACKATHON.md | 0 docs/{future => archive}/NOTIFICATIONS.md | 0 docs/archive/README.md | 31 + docs/{ => archive}/pd-ps-comments.md | 0 docs/{ => archive}/security-audit-2026-04.md | 5 + .../plans/2026-03-27-01-duckdb-state-layer.md | 0 .../plans/2026-03-27-02-complete-system.md | 0 .../2026-04-08-final-integration-fixes.md | 0 .../plans/2026-04-08-production-hardening.md | 0 .../plans/2026-04-08-security-hardening.md | 0 .../plans/2026-04-09-dead-code-cleanup.md | 0 .../plans/2026-04-09-deployment-readiness.md | 0 .../plans/2026-04-09-final-polish.md | 0 .../plans/2026-04-09-security-fixes.md | 0 .../plans/2026-04-10-analyst-bootstrap.md | 0 .../plans/2026-04-10-business-metrics.md | 0 .../plans/2026-04-10-metadata-writer.md | 0 .../plans/2026-04-11-remote-query.md | 0 .../2026-04-12-comprehensive-test-suite.md | 0 .../plans/2026-04-21-deployment-log.md | 0 .../plans/2026-04-21-hackathon-dry-run.md | 0 .../plans/2026-04-21-issues-14-and-10.md | 0 .../2026-04-21-multi-customer-deployment.md | 0 .../plans/2026-04-21-user-mgmt-pat-cli.md | 0 .../2026-04-22-cloudflare-access-auth.md | 0 ...-27-bq-pipeline-views-and-metadata-auth.md | 0 .../2026-04-27-claude-fetch-primitives.md | 0 .../plans/2026-04-29-dev-debug-toolbar.md | 0 ...26-04-29-issue-134-bq-access-unify-plan.md | 0 .../plans/2026-04-29-issues-77-78-79-89.md | 0 .../2026-04-30-customizable-welcome-prompt.md | 0 .../2026-05-01-admin-tables-form-cleanup.md | 0 .../2026-05-04-clean-analyst-bootstrap.md | 0 .../plans/2026-05-04-unified-setup-prompt.md | 0 .../plans/2026-05-06-cli-auto-upgrade.md | 0 ...26-05-07-source-agnostic-table-metadata.md | 0 .../plans/2026-05-11-activity-center-mvp.md | 0 .../2026-05-11-admin-observability-spec.md | 0 .../2026-05-12-platform-telemetry-epic.md | 0 .../2026-05-13-design-system-unification.md | 0 .../2026-05-13-standalone-pages-framework.md | 0 .../specs/2026-03-27-refactoring-design.md | 0 .../2026-03-30-core-refactoring-design.md | 0 .../specs/2026-03-31-data-access-control.md | 0 ...-04-09-multi-instance-deployment-design.md | 0 ...-04-10-porting-internal-features-design.md | 0 .../specs/2026-04-11-remote-query-design.md | 0 ...4-12-comprehensive-test-strategy-design.md | 0 .../specs/2026-04-14-connector-kit-design.md | 0 ...26-04-21-multi-customer-deployment-spec.md | 0 ...26-04-27-claude-fetch-primitives-design.md | 0 .../2026-04-29-dev-debug-toolbar-design.md | 0 ...-04-29-issue-134-bq-access-unify-design.md | 0 ...5-03-issue-160-da-query-remote-fix-spec.md | 0 ...26-05-04-clean-analyst-bootstrap-design.md | 0 .../specs/2026-05-06-cli-auto-upgrade-spec.md | 0 ...-07-source-agnostic-table-metadata-spec.md | 0 docs/auto-install.md | 5 - docs/marketplace.md | 101 ++++ services/session_processors/verification.py | 2 +- services/verification_detector/schemas.py | 2 +- src/orchestrator_security.py | 4 +- 75 files changed, 602 insertions(+), 541 deletions(-) create mode 100644 AGENTS.md create mode 100644 docs/README.md delete mode 100644 docs/RELEASE_TEMPLATE.md create mode 100644 docs/RELEASING.md rename docs/{ => archive}/HACKATHON.md (100%) rename docs/{future => archive}/NOTIFICATIONS.md (100%) create mode 100644 docs/archive/README.md rename docs/{ => archive}/pd-ps-comments.md (100%) rename docs/{ => archive}/security-audit-2026-04.md (98%) rename docs/{ => archive}/superpowers/plans/2026-03-27-01-duckdb-state-layer.md (100%) rename docs/{ => archive}/superpowers/plans/2026-03-27-02-complete-system.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-08-final-integration-fixes.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-08-production-hardening.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-08-security-hardening.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-09-dead-code-cleanup.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-09-deployment-readiness.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-09-final-polish.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-09-security-fixes.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-10-analyst-bootstrap.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-10-business-metrics.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-10-metadata-writer.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-11-remote-query.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-12-comprehensive-test-suite.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-21-deployment-log.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-21-hackathon-dry-run.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-21-issues-14-and-10.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-21-multi-customer-deployment.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-21-user-mgmt-pat-cli.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-22-cloudflare-access-auth.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-27-bq-pipeline-views-and-metadata-auth.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-27-claude-fetch-primitives.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-29-dev-debug-toolbar.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-29-issue-134-bq-access-unify-plan.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-29-issues-77-78-79-89.md (100%) rename docs/{ => archive}/superpowers/plans/2026-04-30-customizable-welcome-prompt.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-01-admin-tables-form-cleanup.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-04-clean-analyst-bootstrap.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-04-unified-setup-prompt.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-06-cli-auto-upgrade.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-07-source-agnostic-table-metadata.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-11-activity-center-mvp.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-11-admin-observability-spec.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-12-platform-telemetry-epic.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-13-design-system-unification.md (100%) rename docs/{ => archive}/superpowers/plans/2026-05-13-standalone-pages-framework.md (100%) rename docs/{ => archive}/superpowers/specs/2026-03-27-refactoring-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-03-30-core-refactoring-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-03-31-data-access-control.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-09-multi-instance-deployment-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-10-porting-internal-features-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-11-remote-query-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-12-comprehensive-test-strategy-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-14-connector-kit-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-21-multi-customer-deployment-spec.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-27-claude-fetch-primitives-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-29-dev-debug-toolbar-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-04-29-issue-134-bq-access-unify-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-05-03-issue-160-da-query-remote-fix-spec.md (100%) rename docs/{ => archive}/superpowers/specs/2026-05-04-clean-analyst-bootstrap-design.md (100%) rename docs/{ => archive}/superpowers/specs/2026-05-06-cli-auto-upgrade-spec.md (100%) rename docs/{ => archive}/superpowers/specs/2026-05-07-source-agnostic-table-metadata-spec.md (100%) delete mode 100644 docs/auto-install.md create mode 100644 docs/marketplace.md diff --git a/.gitignore b/.gitignore index d13723a..1827ab3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Claude Code .claude/ +CLAUDE.local.md # Local dev data (copied from server for testing) dev_data/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b9d832a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,7 @@ +# AGENTS.md + +Agent instructions for this repository live in **[`CLAUDE.md`](CLAUDE.md)** — read it first. + +`CLAUDE.md` is the single source of truth for any AI coding agent working here (not just Claude Code): project structure, the `extract.duckdb` architecture, the data-querying agent rails, and the project conventions — release process, changelog discipline, vendor-agnostic OSS rules, and commit/PR etiquette. + +Full documentation index: [`docs/README.md`](docs/README.md). diff --git a/CHANGELOG.md b/CHANGELOG.md index e9aefb5..69051e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ CalVer image tags (`stable-YYYY.MM.N`, `dev-YYYY.MM.N`) are produced for every C ## [Unreleased] +### Internal +- Documentation tree cleaned up and consolidated. `CLAUDE.md` rewritten (708 → ~320 lines): the four overlapping release sections, the stale `v1→v35` DuckDB schema history, and the marketplace endpoint internals moved out to focused docs; preachy process sections tightened. New `docs/RELEASING.md` (release process + deploy workflows + CI quirks, with `RELEASE_TEMPLATE.md` folded in as an appendix) and `docs/marketplace.md` (marketplace ingestion + re-serving internals). Historical planning artifacts (`docs/superpowers/`, 52 files) and dated one-off docs (`HACKATHON.md`, `pd-ps-comments.md`, `security-audit-2026-04.md`, `future/NOTIFICATIONS.md`) moved under `docs/archive/`. New `docs/README.md` documentation index organized by audience, linked from `README.md` and `CLAUDE.md`. Removed the `docs/auto-install.md` stub. Fixed dangling doc links in `connectors/jira/README.md` and `dev_docs/README.md`, and repointed code/doc references to the archived paths (or dropped the pointer where the target was already a dead reference on `main`). Added a root `AGENTS.md` pointing to `CLAUDE.md` as the single source of truth for any AI coding agent, and `CLAUDE.local.md` to `.gitignore`. + ## [0.54.14] — 2026-05-14 ### Changed diff --git a/CLAUDE.md b/CLAUDE.md index 919c595..c42990d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,38 +2,19 @@ Open-source data distribution platform for AI analytical systems. Extracts data from sources into DuckDB, serves via FastAPI, and distributes parquets to analysts who use Claude Code for local analysis. +Full documentation index: [`docs/README.md`](docs/README.md). + ## First-Time Setup -When a user opens this project for the first time, guide them through interactive setup: +When a user opens this project for the first time, guide them through interactive setup. Ask for: -### Step 1: Gather Information -Ask the user for: -1. Company domain (e.g., "acme.com") - used for Google OAuth -2. Data source type: keboola / bigquery / csv -3. Instance name (e.g., "Acme Data Analyst") +1. Company domain (e.g. `acme.com`) — used for Google OAuth +2. Data source type — `keboola` / `bigquery` / `csv` +3. Instance name (e.g. `Acme Data Analyst`) -### Step 2: Generate Configuration -1. Copy `config/instance.yaml.example` to `config/instance.yaml` -2. Fill in values from Step 1 -3. If Keboola: ask for Storage API token, stack URL, project ID -4. Create `.env` from `config/.env.template` +Then: copy `config/instance.yaml.example` → `config/instance.yaml` and fill it in, copy `config/.env.template` → `.env` and add data-source credentials, and register tables via the admin API (`POST /api/admin/register-table`) or the web UI at `/admin/tables`. -### Step 3: Register Tables -1. Use the FastAPI admin API (`POST /api/admin/register-table`, then `PUT /api/admin/registry/{id}` for updates) or webapp UI to register tables -2. Tables are stored in DuckDB `table_registry` with source_type, bucket, source_table, query_mode -3. For migration from old format: `python scripts/migrate_registry_to_duckdb.py` - -### Step 4: Docker Deployment -```bash -docker compose up # Start app + scheduler -docker compose --profile full up # Include telegram bot - -# HTTPS mode — Caddy + corporate-CA certs at /data/state/certs -docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.tls.yml \ - --profile tls up -d -``` - -See `docs/DEPLOYMENT.md` → **TLS** for cert provisioning + `scripts/ops/agnes-tls-rotate.sh` (daily refetch from `TLS_FULLCHAIN_URL`, `SIGUSR1` reload on diff, no-op when unchanged). The infra repo's `startup.sh` installs this as a systemd timer automatically. +Full step-by-step (local dev, Docker, TLS) lives in [`docs/QUICKSTART.md`](docs/QUICKSTART.md) and [`docs/DEPLOYMENT.md`](docs/DEPLOYMENT.md). New-instance GCP deployment is [`docs/ONBOARDING.md`](docs/ONBOARDING.md). ## Project Structure @@ -59,7 +40,7 @@ See `docs/DEPLOYMENT.md` → **TLS** for cert provisioning + `scripts/ops/agnes- ├── scripts/ # Utility + migration scripts ├── config/ # Configuration templates (instance.yaml.example) ├── docs/ # Documentation + metric YAML definitions -└── tests/ # Test suite (633 tests) +└── tests/ # Test suite ``` ## Architecture: extract.duckdb Contract @@ -71,28 +52,6 @@ Every data source produces the same output: └── data/ ← parquet files (local sources only) ``` -### Remote table support (`_remote_attach`) - -Extractors with remote/passthrough tables (query_mode='remote') include a `_remote_attach` table -in extract.duckdb so the orchestrator can re-ATTACH the external DuckDB extension at query time: - -```sql -CREATE TABLE _remote_attach ( - alias VARCHAR, -- DuckDB alias used in views, e.g. 'kbc' - extension VARCHAR, -- Extension name, e.g. 'keboola' - url VARCHAR, -- Connection URL - token_env VARCHAR -- Env-var name holding the auth token, OR empty for - -- extensions with built-in auth (e.g. BigQuery uses the - -- GCE metadata server — see `connectors/bigquery/auth.py`). -); -``` - -The orchestrator reads this table, installs/loads the extension, fetches the token -(via `token_env` lookup, or via the extension-specific auth path when `token_env=''`), -creates a session-scoped DuckDB SECRET when the extension requires one (BigQuery), and -ATTACHes the external source. Views referencing `kbc."bucket"."table"` then resolve correctly. -This mechanism is generic — any connector can plug in. - The SyncOrchestrator scans `/data/extracts/*/extract.duckdb`, ATTACHes each into master `analytics.duckdb`, and creates views. ``` @@ -111,17 +70,23 @@ The SyncOrchestrator scans `/data/extracts/*/extract.duckdb`, ATTACHes each into SyncOrchestrator.rebuild() ATTACH → master views in analytics.duckdb │ - ┌──────────┼──────────┐ - ▼ ▼ ▼ - FastAPI CLI - (serve) (agnes pull) + ┌──────────┴──────────┐ + ▼ ▼ + FastAPI CLI + (serve) (agnes pull) ``` -Source modes: -- **Batch pull** (Keboola, `query_mode='local'`): DuckDB extension downloads to parquet, scheduled -- **Remote attach** (BigQuery, `query_mode='remote'`): DuckDB BQ extension, no download, queries go to BQ -- **Materialized SQL** (BigQuery, `query_mode='materialized'`): scheduler runs admin-registered SQL through DuckDB BQ extension (via `BqAccess` from `connectors/bigquery/access.py`) and writes the result to `/data/extracts/bigquery/data/.parquet`. Distributed via the same manifest + `agnes pull` flow as Keboola tables. Cost guardrail via `data_source.bigquery.max_bytes_per_materialize` (default 10 GiB; set `0` to disable — YAML `null` falls through to the default). -- **Real-time push** (Jira): Webhooks update parquets incrementally +Source modes (per-table `query_mode`): +- **Batch pull** (Keboola, `local`): DuckDB extension downloads to parquet, scheduled. +- **Remote attach** (BigQuery, `remote`): DuckDB BQ extension, no download, queries go to BQ. +- **Materialized SQL** (`materialized`): scheduler runs admin-registered SQL through DuckDB and writes the result to a parquet under `/data/extracts//data/`. Distributed via the same manifest + `agnes pull` flow as local tables. BigQuery cost guardrail: `data_source.bigquery.max_bytes_per_materialize` (default 10 GiB; `0` disables). +- **Real-time push** (Jira): webhooks update parquets incrementally. + +### Remote table support (`_remote_attach`) + +Extractors with `query_mode='remote'` tables include a `_remote_attach` table in `extract.duckdb` (`alias`, `extension`, `url`, `token_env`) so the orchestrator can re-ATTACH the external DuckDB extension at query time — installing/loading the extension, fetching the token (via `token_env` lookup, or an extension-specific auth path when `token_env=''`, e.g. BigQuery's GCE metadata server), creating a session-scoped SECRET when required, and ATTACHing the source so views like `kbc."bucket"."table"` resolve. The mechanism is generic — any connector can plug in. + +Deeper architecture notes: [`docs/architecture.md`](docs/architecture.md). ## Configuration @@ -140,7 +105,7 @@ uv pip install ".[dev]" uvicorn app.main:app --reload # Run tests -pytest tests/ -v +.venv/bin/pytest tests/ --tb=short -n auto -q # Trigger sync manually curl -X POST http://localhost:8000/api/sync/trigger @@ -158,25 +123,15 @@ docker compose up - `SessionStart` → `agnes pull --quiet` — pulls fresh parquets at the start of every Claude Code session - `SessionEnd` → `agnes push --quiet` — uploads session jsonl + `CLAUDE.local.md` to the server -Both pass `--quiet` so they don't pollute Claude Code stdout, and trail with `|| true` so a server outage never blocks a session. Workspace-level (not user-home) so the hooks fire only when Claude Code opens this analyst workspace, not in unrelated sessions on the same machine. +Both pass `--quiet` so they don't pollute Claude Code stdout, and trail with `|| true` so a server outage never blocks a session. Workspace-level (not user-home) so the hooks fire only when Claude Code opens this analyst workspace. Admin RBAC for auto-sync: `query_mode IN ('local', 'materialized')` plus a `resource_grants` row for one of the analyst's groups → table appears in their manifest → `agnes pull` downloads it. No per-user sync config; the admin layer is the single source of truth. ## Business Metrics -Standardized metric definitions live in DuckDB (`metric_definitions` table). Import starter pack: +Standardized metric definitions live in DuckDB (`metric_definitions` table). Import the starter pack with `agnes admin metrics import docs/metrics/`. -```bash -agnes admin metrics import docs/metrics/ -``` - -### For AI agents analyzing data: -Before computing any business metric, look up the canonical definition: -1. `agnes catalog --metrics` — find the relevant metric -2. `agnes catalog --metrics --show revenue/mrr` — read the SQL and business rules -3. Use the SQL from the metric definition, adapt to the specific question - -Never invent metric calculations — always use the canonical definitions. +**For AI agents analyzing data:** before computing any business metric, look up the canonical definition — `agnes catalog --metrics` to find it, `agnes catalog --metrics --show revenue/mrr` to read the SQL and business rules. Use that SQL, adapted to the question. Never invent metric calculations. ## Querying Agnes data — agent rails @@ -277,432 +232,86 @@ in your `agnes query` calls — there's no `--where` on local since fetch is imp side + `agnes query --remote` for the other; full cross-remote JOIN requires more thought (see #101 for design space). -## Marketplace Repositories - -Admin-managed git repos cloned nightly to `${DATA_DIR}/marketplaces//` -so FastAPI can read their contents from disk. - -- Register via `/admin/marketplaces` (admin UI) or `POST /api/marketplaces`. -- Scheduler calls `POST /api/marketplaces/sync-all` (admin-only, authed via `SCHEDULER_API_TOKEN`) at `daily 03:00` UTC. Routing through HTTP keeps the app the sole writer to `system.duckdb` — the previous in-process call from the scheduler container raced the app's long-lived DB handle and 500-ed on `Could not set lock on file`. -- Manual re-sync from the UI ("Sync now") hits `POST /api/marketplaces/{id}/sync`. -- PATs for private repos persist to `${DATA_DIR}/state/.env_overlay` (chmod 600) as `AGNES_MARKETPLACE__TOKEN`. DuckDB stores only the env-var name (`token_env`), never the secret. -- Registry lives in DuckDB table `marketplace_registry` (schema v9). -- After each successful sync, `src/marketplace.py` parses `.claude-plugin/marketplace.json` - from the cloned repo and caches the plugin list in `marketplace_plugins` - (keyed by `(marketplace_id, plugin_name)`). -- `src/marketplace.py` handles clone/fetch/reset with token redaction in any surfaced error message. - -## Access control (v13) - -Two layers, no role hierarchy. Full reference: [`docs/RBAC.md`](docs/RBAC.md). - -- `user_groups` — named groups. Two seeded as `is_system=TRUE` at startup: - `Admin` (god-mode short-circuit on every authorization check) and - `Everyone` (auto-membership for every user). -- `user_group_members` — `(user_id, group_id, source)`. `source ∈ - {admin, google_sync, system_seed}` so each writer only manipulates its own - rows; Google's nightly DELETE+INSERT does not clobber admin-added members. -- `resource_grants` — generic `(group, resource_type, resource_id)` triple. - Replaces `plugin_access` from v12; the same shape now covers any future - entity-scoped grant (datasets, knowledge categories, …). - -Resource types are an `app.resource_types.ResourceType` `StrEnum` paired with -a `ResourceTypeSpec` registered in `RESOURCE_TYPES` — adding a new one is one -enum member, one `list_blocks(conn)` delegate (projects domain tables into the -`(block → items)` shape the /admin/access tree renders), and one spec entry. -No DB migration, no second wiring step. Endpoints gate with either -`require_admin` (app-level) or `require_resource_access(ResourceType.X, -"{path}")` (entity-level), both from `app.auth.access`. - -Admin UI: `/admin/access`. CLI: `agnes admin group {list,create,delete,members, -add-member,remove-member}` and `agnes admin grant {list,create,delete}`. - -## Claude Code marketplace endpoint - -Agnes serves a single aggregated Claude Code marketplace over two channels, -both gated by PAT auth and filtered per caller: - -- `GET /marketplace.zip` — deterministic ZIP download with `ETag` / - `If-None-Match` (304 when content unchanged). Consumed by a client-side - SessionStart hook. -- `GET /marketplace.git/*` — git smart-HTTP (dulwich via a2wsgi). Registered - in Claude Code once, then Claude Code owns the clone/fetch cycle. - -Auth: ZIP uses `Authorization: Bearer `. Git uses HTTP Basic where the -password field carries the PAT (`https://x:@host/marketplace.git/`) — -git CLI does not speak Bearer. - -Content: filtered via `src.marketplace_filter.resolve_allowed_plugins` which -joins `resource_grants ↔ marketplace_plugins` (matching -`mp.marketplace_id || '/' || mp.name = rg.resource_id`) scoped to the -caller's `user_group_members`. Admin is treated as a regular group here — -no god-mode shortcut for the marketplace feed, so admins curate their own -view by granting plugins to the Admin group (or any group they belong to). -On-disk layout in the served ZIP / git tree uses a slug-prefixed directory -(`plugins/-/`) so two marketplaces shipping a same-named -plugin don't overwrite each other's files. The synth marketplace.json's -`name` field, however, is the plugin's authoritative name from its own -`.claude-plugin/plugin.json` (with a fallback to the upstream -marketplace.json `name`) — Claude Code's `/plugin` UI resolves a loaded -plugin back to its catalog entry by `plugin.json` name, so the catalog -entry's `name` must match. Same-named plugins from two upstream -marketplaces therefore collide in the catalog by design; admin RBAC -(which grants survive the filter) decides which one wins, identical to -how Claude Code behaves when a user adds two upstream marketplaces with -overlapping plugin names directly. `/marketplace/info` exposes both -`name` and `prefixed_name` so operators can disambiguate. - -Cache: content-addressed bare repos at `${DATA_DIR}/marketplaces/git-cache/` -keyed by sha256(filtered content). Two users with the same RBAC view share -one repo; content change → new repo next to the old one. No TTL / prune yet. - -User registration inside Claude Code: - -``` -# ZIP channel (typically via a SessionStart hook that unpacks into ./marketplace/) -curl -H "Authorization: Bearer $AGNES_PAT" https://agnes.example.com/marketplace.zip - -# Git channel — one-time registration. Two paths; pick the first that works. - -# (a) Direct registration — preferred when it works. -/plugin marketplace add https://x:$AGNES_PAT@agnes.example.com/marketplace.git/ - -# (b) Two-step fallback — required when (a) fails. Bun-compiled `claude` on -# macOS / Windows ignores the OS trust store and CA env vars on the -# marketplace HTTPS path, so direct add can fail with TLS errors against -# a private-CA Agnes instance even when system tools work fine. System -# `git` honors GIT_SSL_CAINFO + the OS trust store, so cloning manually -# and pointing Claude Code at the local clone sidesteps the Bun TLS path -# entirely. -git clone https://x:$AGNES_PAT@agnes.example.com/marketplace.git/ ~/agnes-marketplace -claude plugin marketplace add ~/agnes-marketplace -# Optional hardening: strip the PAT from the cloned repo's origin so it -# doesn't sit in plaintext at ~/agnes-marketplace/.git/config — re-clone via -# the dashboard's setup flow when the PAT rotates. -git -C ~/agnes-marketplace remote set-url origin https://agnes.example.com/marketplace.git/ -``` - -The dashboard-served setup payload (see `app/web/setup_instructions.py`) already -branches between (a) and (b) automatically based on platform when a private CA -is in play. The block above is the manual equivalent for users registering -outside that flow (e.g. operators bringing up a new instance, or -analysts whose first attempt failed and need to retry by hand). - ## Hybrid Queries (BigQuery + Local) -Server-side only. Admins can POST `{sql, register_bq: {alias: bq_sql}}` to -`/api/query/hybrid` (see `app/api/query_hybrid.py`), which runs the BQ -sub-queries server-side (where BQ credentials live) and joins the result -against the server's local parquet views in a single DuckDB session. +Server-side only. Admins can POST `{sql, register_bq: {alias: bq_sql}}` to `/api/query/hybrid` (see `app/api/query_hybrid.py`), which runs the BQ sub-queries server-side (where BQ credentials live) and joins the result against the server's local parquet views in a single DuckDB session. -There is no analyst-facing CLI flag for this — analysts who need to combine -a local table with a remote one should `agnes snapshot create` a filtered -subset of the remote table and `agnes query` the join locally, or run the -join server-side via `agnes query --remote`. The earlier `agnes query ---register-bq` flag ran in-process on the caller's machine and required -local BigQuery credentials that analysts don't have; it was removed. +There is no analyst-facing CLI flag for this — analysts who need to combine a local table with a remote one should `agnes snapshot create` a filtered subset of the remote table and `agnes query` the join locally, or run the join server-side via `agnes query --remote`. + +## Marketplace + +Agnes ingests admin-registered Claude Code marketplaces (git repos cloned nightly to `${DATA_DIR}/marketplaces//`) and re-serves a single aggregated, RBAC-filtered marketplace back to user instances over two PAT-gated channels: `GET /marketplace.zip` and `GET /marketplace.git/*`. Content is filtered per caller by joining `resource_grants ↔ marketplace_plugins` against the caller's groups. + +Full reference — ingestion, the served endpoint, RBAC filtering, user registration inside Claude Code: [`docs/marketplace.md`](docs/marketplace.md). Content-authoring side (`marketplace-metadata.json`): [`docs/curated-marketplace-format.md`](docs/curated-marketplace-format.md). + +## Access control + +Two layers, no role hierarchy: + +- `user_groups` — named groups. `Admin` (god-mode short-circuit on every authorization check) and `Everyone` (auto-membership) are seeded as `is_system=TRUE`. +- `user_group_members` — `(user_id, group_id, source)`; `source` segregates writers so Google's nightly sync doesn't clobber admin-added members. +- `resource_grants` — generic `(group, resource_type, resource_id)` triples for any entity-scoped grant. + +Gate endpoints with `Depends(require_admin)` (app-level mutations) or `Depends(require_resource_access(ResourceType.X, "{path}"))` (entity-scoped), both from `app.auth.access`. Add a resource type by extending the `ResourceType` `StrEnum` and registering a `ResourceTypeSpec` (with a `list_blocks` projection delegate) in `app/resource_types.py` — no DB migration. + +Admin UI: `/admin/access`. CLI: `agnes admin group …` and `agnes admin grant …`. Full reference: [`docs/RBAC.md`](docs/RBAC.md). ## Extensibility ### Data Sources (extract.duckdb contract) -New connector = `connectors//extractor.py` producing `extract.duckdb + data/`. -Must create `_meta` table with columns: table_name, description, rows, size_bytes, extracted_at, query_mode. -Orchestrator ATTACHes it automatically. +New connector = `connectors//extractor.py` producing `extract.duckdb + data/`. Must create a `_meta` table with columns: `table_name`, `description`, `rows`, `size_bytes`, `extracted_at`, `query_mode`. The orchestrator ATTACHes it automatically. ### Authentication Auth providers in `app/auth/` (FastAPI-based): -- **Google**: OAuth via Google (Workspace group memberships pulled at sign-in — see `docs/auth-groups.md` for the GCP setup checklist + the `security` label gotcha) -- **Email**: Email magic link (itsdangerous token) +- **Google**: OAuth via Google (Workspace group memberships pulled at sign-in — see [`docs/auth-groups.md`](docs/auth-groups.md) for the GCP setup checklist + the `security` label gotcha) +- **Email**: magic link (itsdangerous token) - **Desktop**: JWT for API -### RBAC - -See **[Access control (v13)](#access-control-v13)** above and [`docs/RBAC.md`](docs/RBAC.md) for the full reference. TL;DR for module authors: gate endpoints with `Depends(require_admin)` for app-level mutations or `Depends(require_resource_access(ResourceType.X, "{path}"))` for entity-scoped grants. Add a new resource type by extending the `ResourceType` `StrEnum` and registering a `ResourceTypeSpec` (with a `list_blocks` projection delegate) in `app/resource_types.py`. - -## Release & deploy workflows - -Two separate release.yml-style workflows produce GHCR images. Pick the one that matches what you're shipping. - -### `release.yml` — auto-build on every push -Runs on **every** push to **every** branch. -- Push to `main` → `:stable`, `:stable-YYYY.MM.N` (CalVer). -- Push to non-main `/` → `:dev`, `:dev-YYYY.MM.N`, `:dev-`, and (when prefix isn't a Git Flow convention) `:dev--latest` alias. - -VMs that pin to a floating tag (`:dev`, `:dev--latest`) auto-upgrade within ~5 min via the cron in `agnes-auto-upgrade.sh`. Convenient for per-developer dev VMs; **footgun for shared dev VMs** (last pusher wins, regardless of who). - -### `keboola-deploy.yml` — tag-triggered, explicit deploy only -Runs **only** on git tags matching `keboola-deploy-*`. Publishes: -- `:keboola-deploy-` — immutable, tied to the exact commit -- `:keboola-deploy-latest` — floating alias the consumer pins to - -**Operator workflow:** -```bash -git checkout -git tag keboola-deploy- -git push origin keboola-deploy- -# → workflow builds + publishes both tags -# → VM cron picks up :keboola-deploy-latest within ~5 min -# → manual cron trigger (skip the wait): sudo /usr/local/bin/agnes-auto-upgrade.sh on the VM -``` - -Use this when the consumer (e.g. a customer dev VM) needs **deploy-when-I-decide** semantics — no surprise rollouts from upstream branch pushes by other contributors. The infra repo pins `image_tag = "keboola-deploy-latest"` on the relevant VM. - -### Module versioning -The customer-instance Terraform module under `infra/modules/customer-instance/` is published as `infra-vMAJOR.MINOR.PATCH` git tags (separate from app CalVer tags). Bump on any module-API change; downstream infra repos pin to the tag in their `source = "github.com/keboola/agnes-the-ai-analyst//infra/modules/customer-instance?ref=infra-v1.X.Y"`. - -After merging a module change to `main`: -```bash -git tag infra-vX.Y.Z origin/main -git push origin infra-vX.Y.Z -``` - -### Replacing a VM after a startup-script change -Module sets `lifecycle { ignore_changes = [metadata_startup_script] }` on `google_compute_instance.vm` so normal `terraform apply` doesn't churn running VMs. To propagate a startup-script update, trigger the consumer's apply workflow manually with the VM resource address — typical workflow_dispatch input is `recreate_targets='module.agnes.google_compute_instance.vm[""]'`. - ## Key Implementation Details ### DuckDB Schema (src/db.py) -- Schema v35 with auto-migration v1→…→v35 (v5 adds `users.active`, v6 adds `personal_access_tokens`, v7 adds `personal_access_tokens.last_used_ip`, v8/v9 added the legacy internal_roles/role-grants tables, v10 added `view_ownership` for cross-connector view-name collision detection (issue #81 Group C), v11 added marketplace_registry + marketplace_plugins + user_groups + plugin_access, v12 added users.groups JSON + user_groups.is_system, **v13 replaces internal_roles/group_mappings/user_role_grants/plugin_access with user_group_members + resource_grants and drops users.groups JSON**, v14 adds FK constraints on user_group_members + resource_grants after orphan cleanup, v15 adds knowledge_items context-engineering columns + contradictions + session_extraction_state, v16 adds verification_evidence, v17 adds knowledge_item_relations, v18 drops stranded non-google memberships from google-managed groups, **v19 drops legacy `dataset_permissions`, `access_requests` tables and `users.role`, `table_registry.is_public` columns — table access is now exclusively per-group via `resource_grants(resource_type='table')`**, **v20 adds `source_query` TEXT to `table_registry` to back `query_mode='materialized'` (BigQuery scheduled-query parquet path)**, **v21 adds `welcome_template` singleton table backing the Agent Setup Prompt admin override (`/admin/agent-prompt`)**, **v22 reserves the `setup_banner` table — feature dropped mid-development; table retained for forward compatibility with already-migrated instances**, **v23 adds `claude_md_template` singleton table backing the Agent Workspace Prompt admin override (`/admin/workspace-prompt`)**, **v24 rewrites materialized BQ `source_query` from DuckDB-flavor `bq."ds"."t"` to BQ-native `` `.ds.t` `` so the new wrapping path accepts them; idempotent + warns when project unconfigured**, **v25 adds `store_entities` + `user_store_installs` + `user_plugin_optouts` backing the flea-market and my-stack views (now served at `/marketplace?tab=flea` + `/marketplace?tab=my`; the original standalone `/store` and `/my-ai-stack` page routes were dropped post-v25) — the served marketplace is now `(admin_granted ∖ opt_outs) ∪ store_installs`**, **v26 unifies Keboola `query_mode='local'` rows into `'materialized'` — the old local mode (DuckDB Keboola extension's COPY through QueryService) is replaced by the new Storage API export-async path which works regardless of project flags; existing `local` Keboola rows are flipped, NULL `source_query` means full-table export**, **v27 adds 7 columns to `table_registry` for Keboola per-table sync-strategy support: `incremental_window_days`, `max_history_days`, `incremental_column`, `where_filters`, `partition_by`, `partition_granularity`, `initial_load_chunk_days`. Layered on top of v26: admins can opt specific tables back to `query_mode='local'` (via the Direct extract Edit-modal radio) to enable the new dispatcher. The pre-existing `sync_strategy` column (default `'full_refresh'`) is reused — pre-v27 it was inert catalog metadata; post-v27 the Keboola extractor dispatches off it (`full_refresh` | `incremental` | `partitioned`). All new columns NULL on existing rows; meaningful only when paired with the matching strategy.**, **v28 introduces explicit-install (Model B) for curated marketplace plugins — served set flips from `(rbac ∖ user_plugin_optouts)` to `(rbac ∩ subscriptions)`. The `user_plugin_optouts` table+columns are reused (no DDL rename) so existing operator instances skip migration churn; row PRESENCE flips meaning from "excluded" to "subscribed", and the migration wipes existing rows so the inverted reading starts from a clean baseline. Also adds `marketplace_plugins.created_at` (per-plugin newest-first sort on /marketplace), backfilled from parent `marketplace_registry.registered_at` so existing plugins get a sensible date until the next sync overwrites with `CURRENT_TIMESTAMP`.**, **v29 adds `store_submissions` table backing flea-market upload guardrails (manifest + static-security + LLM-review verdicts) plus `store_entities.visibility_status` (`pending | approved | hidden`) — entity visible in flea browse only when `visibility_status='approved'`. Existing rows backfilled to `'approved'` so live flea content stays visible.**, **v30 adds `store_submissions.{file_size, bundle_sha256, bundle_purged_at}` so blocked-inline bundles persist for forensics + admin rescan/override (instead of the prior rmtree-on-reject); SHA256 survives the 30-day TTL purge, `bundle_purged_at` flips on at purge time so detail page can render "purged on YYYY-MM-DD"**, **v31 reshapes `store_submissions` (drops legacy unique on `entity_id` so multiple submissions per entity work — re-uploads/rescans land as new rows; idempotent table rebuild)**, **v32 adds `store_entities.{archived_at, archived_by}` columns plus broadens `visibility_status` enum to include `'archived'` for soft-delete; `DELETE /api/store/entities/{id}` is now soft (archive) by default, hard delete moves to `?hard=true` (admin-only)**, **v33 drops `store_submissions.retry_count` — counter mixed automatic LLM retries (capped) with admin-initiated rescans (unbounded), no useful semantics; admin Rescan button + audit_log carry the operational signal**, **v34 ensures `idx_store_submissions_entity` exists after the v33 column drop (DuckDB rebuilds the table sans index when dropping a column referenced by an index)**, **v35 broadens `store_entities.visibility_status` enum to include lifecycle value beyond `'archived'` already added in v32 — column-rebuild migration to register the new value with DuckDB's CHECK constraint, so `set_visibility('archived')` works against the constrained column. Also marks the architectural cutover from denormalizing `'archived'`/`'deleted'` onto `store_submissions.status` to LEFT-JOINing `store_entities` at query time: verdict (sub.status) becomes immutable forensic record, lifecycle (entity.visibility_status) becomes the live source of truth that the admin queue's Archived chip filters by.** — see CHANGELOG and docs/RBAC.md) +- Auto-migrating schema (`v1 → vN`). The current version and migration ladder live in `src/db.py`; per-version schema change notes are in `CHANGELOG.md` — do not maintain a duplicate history here. - `table_registry`: id, name, source_type, bucket, source_table, query_mode, sync_schedule, etc. -- `sync_state`, `sync_history`: track extraction progress +- `sync_state`, `sync_history`: track extraction progress. - `users`, `audit_log`: account state + audit trail. RBAC lives in `user_groups` + `user_group_members` + `resource_grants`. -- System DB at `{DATA_DIR}/state/system.duckdb` -- Analytics DB at `{DATA_DIR}/analytics/server.duckdb` +- System DB at `{DATA_DIR}/state/system.duckdb`, analytics DB at `{DATA_DIR}/analytics/server.duckdb`. ### SyncOrchestrator (src/orchestrator.py) -- `rebuild()`: scans extracts dir, ATTACHes all, creates master views, updates sync_state -- `rebuild_source(name)`: single source (used after Jira webhooks) -- Thread-safe via `_rebuild_lock` +- `rebuild()`: scans extracts dir, ATTACHes all, creates master views, updates sync_state. +- `rebuild_source(name)`: single source (used after Jira webhooks). +- Thread-safe via `_rebuild_lock`. ### Connector Pattern -- **Keboola**: `connectors/keboola/extractor.py` uses DuckDB Keboola extension, fallback to `client.py` -- **BigQuery**: `connectors/bigquery/extractor.py` uses DuckDB BQ extension (remote-only, no download) -- **Jira**: `connectors/jira/webhook.py` → `incremental_transform.py` → `extract_init.py` updates `_meta` -- `connectors/keboola/client.py`: legacy Keboola Storage API wrapper (kept as fallback) +- **Keboola**: `connectors/keboola/extractor.py` uses the DuckDB Keboola extension, falls back to `client.py` (legacy Storage API wrapper). +- **BigQuery**: `connectors/bigquery/extractor.py` uses the DuckDB BQ extension (remote-only, no download). +- **Jira**: `connectors/jira/webhook.py` → `incremental_transform.py` → `extract_init.py` updates `_meta`. ### Config Loading -1. `config/loader.py` loads `instance.yaml` -2. `app/instance_config.py` exposes `get_data_source_type()`, `get_value()` -3. Table config lives in DuckDB `table_registry` (not markdown files) +1. `config/loader.py` loads `instance.yaml`. +2. `app/instance_config.py` exposes `get_data_source_type()`, `get_value()`. +3. Table config lives in DuckDB `table_registry` (not markdown files). ### Files NOT to modify (stable infrastructure) -- `connectors/jira/file_lock.py` - Advisory file locking -- `connectors/jira/transform.py` - Core Jira transform logic -- `services/ws_gateway/` - WebSocket notification gateway +- `connectors/jira/file_lock.py` — advisory file locking +- `connectors/jira/transform.py` — core Jira transform logic +- `services/ws_gateway/` — WebSocket notification gateway -## Vendor-agnostic OSS — no customer-specific content +## Release process -This repo is the public OSS distribution. **Nothing customer-specific belongs in code, configuration defaults, comments, docs, commit messages, PR titles, or PR bodies.** That includes: +Full recipe, deploy workflows, and CI quirks: [`docs/RELEASING.md`](docs/RELEASING.md). The non-negotiable rules: -- Specific deployments or brands (private VM names, internal product brands, organization names that aren't already public sponsors). -- Cloud project IDs, internal hostnames, runbook paths from a particular install (`/opt/`, `.`, `prj--…`, internal SA emails). -- Cross-references to private repos (`/#NN`). Describe the integration in generic terms or link to public examples instead. +- **Changelog discipline.** Every PR that changes user-visible behavior MUST add a bullet under `## [Unreleased]` in `CHANGELOG.md`, in the same PR — grouped Added/Changed/Fixed/Removed/Internal, `**BREAKING**` prefix for breaking changes. No follow-ups. +- **Release-cut belongs to the PR.** The version bump (`pyproject.toml`) + CHANGELOG rename + new empty `[Unreleased]` are the LAST commit on the PR that earned the version — never a standalone follow-up PR. If a PR lands the only `[Unreleased]` content, the release-cut ships in the same merge. After merge: tag `vX.Y.Z` on the merge commit + create the GitHub Release. +- **Run the full test suite before every push** — `.venv/bin/pytest tests/ --tb=short -n auto -q` (this is what CI runs). Failures in code you touched: fix before pushing. Failures unrelated to your diff: confirm with `git stash` they reproduce on a clean branch, note them in the PR body, don't block on them. -When you motivate a change, frame it abstractly ("behind a TLS-terminating reverse proxy", "in containerized deploys") rather than naming a specific operator. When you show examples, use placeholders (`example.com`, ``, ``). When config has reasonable defaults pulled from one deployment's habits, generalize them or surface them as documented examples — not hard-coded assumptions. +## Project conventions -Customer-specific automation, hostnames, and identities live in private infra repos that *consume* this OSS. The OSS describes capabilities, defaults, and configuration knobs — not how a specific operator wired them up. +### Vendor-agnostic OSS — no customer-specific content +This repo is the public OSS distribution. **Nothing customer-specific belongs in code, config defaults, comments, docs, commit messages, or PR titles/bodies** — no specific deployments or brands, cloud project IDs, internal hostnames, runbook paths, internal SA emails, or cross-references to private repos. Frame motivations abstractly ("behind a TLS-terminating reverse proxy"); use placeholders in examples (`example.com`, ``, ``). Customer-specific automation lives in the private infra repos that *consume* this OSS. Before opening a PR, scan the diff and PR body for customer-specific tokens. -## Changelog discipline — non-negotiable +### Issue economy — fix or close, don't spawn +The default reaction to "I noticed something while doing X" is **fix it now**, **close it as moot after audit**, or **leave a `TODO` in the touching diff** — not "file an issue". Before filing any follow-up issue: verify the claim is still true on current `main` (issues routinely cite moved line numbers and deleted call sites — if the premise is gone, close the parent), and check whether it's a ≤30-min, ≤1-file fix you could just do in the current PR. Filing is acceptable only for multi-file refactors with open design questions, production changes needing operator coordination, unclear cross-team ownership, or bugs whose fix would balloon the current PR ≥3×. When investigating an existing issue, reproduce the symptom on current `main` first; if it doesn't fire, close with a comment documenting the audit. When in doubt: fix it, or close it. -**Every PR that adds, removes, or changes user-visible behavior MUST update `CHANGELOG.md` in the same PR.** No exceptions, no follow-ups, no "I'll do it after merge". User-visible = anything an operator, end-user, or downstream integrator can observe: CLI flags / output / exit codes, REST endpoints / payloads / status codes, web UI, `instance.yaml` schema, env vars, `extract.duckdb` contract, Docker / compose / Caddyfile knobs, default behaviors, breaking changes, security fixes. - -**How:** -- Add a bullet under the topmost `## [Unreleased]` heading (create one if missing — it sits above the latest released version). -- Group by `### Added` / `### Changed` / `### Fixed` / `### Removed` / `### Internal` (Keep-a-Changelog sections). -- Mark breaking changes with `**BREAKING**` at the start of the bullet — operators grep for that string before bumping the pin. -- Reference the relevant doc/runbook if one exists (e.g. `see docs/auth-groups.md`), don't restate it. -- Internal-only changes (refactors, test additions, dependency bumps without behavior change) go under `### Internal` — still log them, just keep them terse. - -**When you cut a release:** -- Rename `## [Unreleased]` → `## [X.Y.Z] — YYYY-MM-DD`. -- Append a new empty `## [Unreleased]` section at the top so the next PR has somewhere to land. -- Bump `version` in `pyproject.toml` to match `X.Y.Z`. -- Tag the merge commit as `vX.Y.Z` and push the tag. - -**If you find yourself opening a PR without a CHANGELOG entry, stop and add one before requesting review.** Reviewers should bounce PRs that touch user-visible behavior without a changelog update — same way they'd bounce a PR with no test changes for new logic. - -## Release-cut belongs to the PR — non-negotiable - -**The version bump + CHANGELOG rename + new empty `[Unreleased]` are the LAST commit on the PR that earned the version. Never a standalone follow-up PR.** - -When a PR lands the only `[Unreleased]` content (or is the last in a queue of in-flight feature PRs), the release-cut MUST ship as part of the same merge. Standalone release-cut PRs add review-overhead PRs to history with no behavior change of their own and pollute `git log` with the worst kind of churn — bookkeeping commits separated from the work that earned them. - -**Mandatory checklist before approving / enabling auto-merge on ANY PR:** - -1. **Stop.** Will this PR land alone in `[Unreleased]` (no other in-flight PRs queued behind it)? -2. **If yes**, the release-cut is REQUIRED in the same PR before merge. BEFORE pushing the final commit: - - Bump `pyproject.toml` to `X.Y.Z` - - Rename `## [Unreleased]` → `## [X.Y.Z] — YYYY-MM-DD`, add a new empty `## [Unreleased]` on top - - Either squash these into the consolidation commit OR add as a separate `release: X.Y.Z` commit on the same branch -3. **THEN** push, approve, enable auto-merge. -4. After auto-merge fires: tag `vX.Y.Z` against the merge commit + create a GitHub Release. Done — one PR, one merge, one release. - -**Failure mode to avoid:** enabling auto-merge on the feature PR thinking "I'll add the release-cut after." Auto-merge fires faster than the second commit lands. The window closes; the only fix is a standalone release-cut PR — exactly what this rule prohibits. - -**Acceptable standalone release-cut** (rare): only when `[Unreleased]` accumulated bullets from MULTIPLE already-merged PRs AND no further behavior-change PR is queued — i.e. the cut is the only outstanding work and there's no PR to attach it to. - -## Release workflow — concrete recipe - -The rule above tells you WHAT to ship in a release-cut. This recipe tells you HOW to land one end-to-end without tripping on the operational quirks. Follow it linearly the first few times; once you've internalized the steps the order matters less, but the **non-obvious gotchas at the end** never go away. - -### Happy path (8 steps) - -```bash -# 1. Branch from a fresh checkout. iCloud Drive worktrees randomly hang -# on git operations — use a fresh shallow clone in /tmp instead. -cd /tmp && git clone --depth 50 --branch main \ - https://github.com/keboola/agnes-the-ai-analyst.git agnes- -cd agnes- && git checkout -b zs/ - -# 2. Make the change + tests. Run the AREA pytest while iterating -# (e.g. `pytest tests/test_X.py -p no:xdist -q`). - -# 3. Add a CHANGELOG bullet under [Unreleased]. -# Group: Added | Changed | Fixed | Removed | Internal -# Mark BREAKING with **BREAKING** prefix. - -# 4. Commit the change(s). Multiple logical commits OK; release-cut -# will be a SEPARATE last commit (next step). DO NOT bundle the -# release-cut into the same commit as the change — it pollutes -# the SHA that auto-close keywords reference and makes revert -# targeted at the change-only difficult. - -# 5. Run the full pytest suite locally: -# `pytest tests/ -p no:xdist -q` (or `-n auto` if xdist works). -# Pre-existing fails (e.g. test_readers_in_pre_init_dir under -# subprocess timeout) are OK to ignore; verify by reverting your -# diff and reproducing on bare main. - -# 6. Release-cut commit (LAST commit on the PR per the rule above): -# - Bump pyproject.toml: version = "X.Y.Z" -# - Rename `## [Unreleased]` → `## [X.Y.Z] — YYYY-MM-DD` -# - Add a fresh empty `## [Unreleased]` line above -# Commit message: `release: X.Y.Z — ` - -# 7. Push branch + open PR + enable auto-merge SQUASH: -# git push -u origin HEAD -# gh pr create --repo keboola/agnes-the-ai-analyst \ -# --head --title "<...>" --body "<...>" -# gh pr merge --repo keboola/agnes-the-ai-analyst \ -# --squash --auto --delete-branch - -# 8. After auto-merge fires (poll or `Monitor`): -# git fetch origin --tags -# git tag vX.Y.Z -# git push origin vX.Y.Z -# gh release create vX.Y.Z --repo keboola/agnes-the-ai-analyst \ -# --title "vX.Y.Z — <...>" --notes "" -``` - -### Picking the next version - -`pyproject.toml`'s current `version` is the **next-release target** (post-cut from the previous release). Pre-1.0 we patch-bump for everything that doesn't break operator-facing APIs: - -- `instance.yaml` schema additions, new env vars, new endpoints → patch (e.g. 0.54.3 → 0.54.4) -- New CLI subcommands, BREAKING removals, schema migrations → still patch within the current 0.5x cycle (no minor bumps cut today) -- The CHANGELOG `**BREAKING**` marker is what operators grep for; the version number is secondary - -Always check `git tag -l "v0.X*"` before naming — if `v0.54.0` is already tagged, the next one is `v0.54.1`, even if `pyproject.toml` still says `0.54.0` from a stale post-cut commit (we've shipped that race before). - -### Authoring expectations on the PR - -- **Self-PRs** (you're both author and reviewer): GitHub forbids self-approve. If branch protection requires N approving reviews (we don't today — `required_approving_review_count = 0`), you need someone else to approve. With our current 0-review setup, self-PRs can still merge automatically once required CI passes. -- **Other people's PRs you're taking over**: dismiss any prior CHANGES_REQUESTED reviews (yours or someone else's) before auto-merge can fire. `gh pr review --approve --body "..."` after pushing your fixes. -- **Devin Review**: not a required check today; runs in parallel and posts a comment. Don't wait on it for merge unless the human reviewer explicitly asks. - -### CI quirks you WILL hit - -- **`gh pr checks` glosses CANCELLED as `fail`.** When you force-push (rebase, amend), GitHub auto-cancels the in-flight `Release` workflow run on the older SHA. Those cancelled jobs show up as "fail" in the PR's check summary and tab forever, even after newer runs succeed. **Look at the conclusion column, not just the count.** Rule of thumb: if the same check name appears with both `pass` and `fail` rows, the `fail` row is from an older auto-cancelled SHA. Verify with `gh api repos/keboola/agnes-the-ai-analyst/commits//check-runs` — the raw API distinguishes `cancelled` from `failure` truthfully. -- **Branch protection's "strict" mode caches cancelled `test` as blocking** even after newer `test` runs succeed. Symptom: `mergeable_state: blocked` despite all required checks green on the latest SHA. Fix: re-run the cancelled `Release` workflow run (`gh run rerun `); once its `test` job lands as success, the block clears. We've hit this on PRs #273, #281, #285, #286. -- **Required checks** (per branch protection): `test` + `docker-build` only. Other workflows (`cli-wheel-clean-install`, `build-and-push`, `Release`-pipeline, Devin Review) are advisory — green/red doesn't gate merge. -- **`enforce_admins: true`** in branch protection means `--admin` flag on `gh pr merge` does NOT bypass. Don't try; just fix the underlying block. - -### Recovery when something derails - -- **Force-pushed and lost auto-merge?** GitHub *usually* preserves auto-merge across force-pushes for the same PR; if it cleared, just re-run `gh pr merge --squash --auto --delete-branch`. -- **Release-cut commit forgot to land?** That's the failure mode the "Release-cut belongs to the PR" rule prevents. If it happens anyway: open a follow-on PR with ONLY the release-cut commit, ship it, and write up why in your post-mortem comment. -- **Wrong version number tagged?** `git tag -d vX.Y.Z && git push --delete origin vX.Y.Z` then re-tag against the right SHA. Update the GitHub Release if you already created it. - -## Issue economy — fix or close, don't spawn - -**The default reaction to "I noticed something while doing X" is NOT "let me file an issue." The default is one of: fix it now, close as moot after audit, or leave a `TODO` in the touching diff.** - -This codebase has accumulated issues that turn out to be: -- Already fixed in a different PR but the issue stayed open -- Stale (the code structure that motivated them is gone) -- Phantom (the symptom described doesn't actually fire on current main) -- Trivially fixable in 5 minutes inside the PR you're already in - -Filing follow-up issues for these wastes everyone's attention. Issue count grows, signal-to-noise drops, real bugs sit alongside obsolete tickets, and the next person triaging asks "what's actually live here?" Quoting one observed pattern from this repo: a takeover-review PR found 3 "LOW hygiene items," filed them as a follow-up issue, then a week later the same author (me) closed the issue moot after a deeper audit showed the production callers already handled the problem correctly. Net contribution: 1 distracting issue + 2 round-trips of context-switching, zero behavior change. - -### Mandatory checks BEFORE filing any follow-up issue - -1. **Is the underlying claim still true on main?** Audit the code paths the issue describes. Issues from > 2 weeks ago routinely cite line numbers that have moved, function names that were renamed, and call sites that were deleted in unrelated work. If the underlying premise is gone → **close the parent, don't file a child**. -2. **Could you fix it in the PR you're already in (≤ 30 min, ≤ 1 file)?** If yes, just fix it. Bundle into a separate commit so the diff stays reviewable; the release-cut already gives you the version-bump vehicle. **Filing a follow-up "to keep this PR focused" is almost always wrong** — the focus argument trades 5 minutes of additional review now for an indefinite-future round-trip later. -3. **Is the fix a single-file change with obvious tests?** Same as #2 — fix it, don't file. -4. **If you're filing because the work needs design discussion** (interface choice, multi-file refactor, performance tradeoff) — fine, file with sufficient context that the next person can act without re-deriving. Include: code anchors with line numbers, exact reproduction steps, what you considered and rejected, and the design questions the next author needs to answer. - -### Audit-first reflex when investigating an existing issue - -Before writing ANY code on an open issue, **verify the symptom on current main**: - -- Reproduce the bug locally (or in a fresh clone of main). If you can't reproduce, the issue is probably stale — close with comment explaining what you found. -- Grep for the cited line numbers / function names. If they've moved, the issue's code anchors are stale; either update them or close. -- Check git log + recent merges — the issue may already be fixed by a PR that landed after the issue was filed but didn't reference it. - -When the audit shows the issue is moot, **close it with a closing comment that documents the audit** (what you grepped, what you checked, why the symptom doesn't fire today). Future readers seeing the closed issue need to know it was deliberately closed after audit, not abandoned. - -### Patterns to avoid - -- **"Found a small thing while reviewing — let me file an issue"** without checking whether it's a 5-minute fix you could do in this PR. -- **"Sub-agent flagged 3 LOW findings — let me file an issue"** without checking which of them are still valid post-audit. -- **"The issue says X is broken — let me build a fix"** without first verifying X is actually broken on current main. -- **"This deserves a follow-up issue"** when the residual is a single-line cleanup. -- **Filing two issues to close one** (e.g. closing #163 by filing #287 and #288 — net +1 open). - -### Acceptable filing scenarios - -- Multi-file refactor with design questions the current PR can't resolve. -- Production behavior change that needs operator coordination (e.g. requires a config rollout before code can be enabled). -- Cross-team work where ownership is unclear and the issue is the way to flag it. -- Bugs you can repro but the fix would balloon the current PR's scope ≥ 3×. - -### Acceptable closing scenarios (close, don't fix) - -- Audit shows the symptom doesn't fire on current main (phantom issue). -- Underlying code structure was deleted in unrelated work (stale issue). -- Resolved by a PR that didn't reference the issue number (close with link to the PR + commit). -- Original author indicates the requirement changed (idea-level issues). - -When in doubt: **fix it, or close it**. Filing is the third choice, not the first. - -## Run tests before every push — non-negotiable - -**Before `git push`, run the full pytest suite locally.** CI runs the same command (`.github/workflows/ci.yml:29` → `pytest tests/ -v --tb=short -n auto`); a failure that surfaces in CI was discoverable in 90 seconds locally. Pushing first and watching CI fail wastes operator time, slows the PR, and trains everyone to ignore CI badges. - -**Command (matches CI):** - -```bash -.venv/bin/pytest tests/ --tb=short -n auto -q -``` - -`-n auto` parallelizes across CPU cores; the suite runs in ~90s on a modern laptop. Local-only env (no `instance.yaml`, dev defaults) is fine — fixtures use `fresh_db` per-test isolation. - -**When tests fail:** -- **Failures in code you touched** → fix before pushing. Update test expectations when the behavior change is intentional and documented (e.g. template restructure that changes assertion strings). -- **Failures unrelated to your diff** → confirm with `git stash && pytest && git stash pop`. If they reproduce on a clean branch, they are pre-existing — note them in the PR body but don't block your push on them. Don't silently skip; flag them so someone owns the fix. -- **Flaky tests** → re-run once. Two consecutive failures = real failure, fix or quarantine with a tracked issue. - -**Smoke shortcuts (when full suite is too slow during iteration):** -- `pytest tests/ -k -q` for area-scoped checks while iterating -- `pytest tests/test_X.py tests/test_Y.py -q` for the modules you touched - -But the **full** `pytest tests/ -n auto` runs once before push. No exceptions. - -If a CHANGELOG entry, doc edit, or pure-formatting commit genuinely doesn't touch any code path the tests exercise, you can skip the full run — but be honest with yourself about whether that's actually the case. - -## Git Commits & Pull Requests - -- Keep commit messages clean and concise -- Do not include AI attribution in commits or PRs -- Before opening a PR, scan the diff and the PR body for the customer-specific tokens listed above (`grep -niE '||...'`). If anything matches, generalize or remove it. +### Git commits & pull requests +- Keep commit messages clean and concise. +- Do not include AI attribution in commits or PRs. diff --git a/README.md b/README.md index 500aecd..096e3d7 100644 --- a/README.md +++ b/README.md @@ -182,12 +182,15 @@ See `config/instance.yaml.example` for all available options. ## Documentation -- [Hackathon TL;DR](docs/HACKATHON.md) — condensed deploy + dev playbooks (for both humans and AI agents) +**Full index: [docs/README.md](docs/README.md)** — every doc, organized by audience (analyst / operator / developer). + +Key entry points: + +- [Quickstart](docs/QUICKSTART.md) — local development setup - [Onboarding Guide](docs/ONBOARDING.md) — end-to-end Terraform deployment into a GCP project (recommended for production) - [Deployment Guide](docs/DEPLOYMENT.md) — chooses between Terraform and Docker Compose; covers OSS self-host - [Configuration Reference](docs/CONFIGURATION.md) — `instance.yaml`, env vars, per-instance options - [Architecture](ARCHITECTURE.md) — orchestrator, extractors, DB layout -- [Quickstart](docs/QUICKSTART.md) — local development ## Contributing diff --git a/connectors/bigquery/access.py b/connectors/bigquery/access.py index 03fdd24..a2e7338 100644 --- a/connectors/bigquery/access.py +++ b/connectors/bigquery/access.py @@ -1,8 +1,8 @@ """Single entry point for BigQuery access — config resolution, client construction, DuckDB-extension session, and Google-API error translation. -See docs/superpowers/specs/2026-04-29-issue-134-bq-access-unify-design.md for the -full design rationale. +See docs/archive/superpowers/specs/2026-04-29-issue-134-bq-access-unify-design.md +for the full design rationale. """ from __future__ import annotations diff --git a/connectors/jira/README.md b/connectors/jira/README.md index 2417df9..e2f3164 100644 --- a/connectors/jira/README.md +++ b/connectors/jira/README.md @@ -433,7 +433,7 @@ if not hmac.compare_digest(signature, expected): ## Schema Reference -See [docs/jira_schema.md](jira_schema.md) for detailed table schemas and example queries. +The Jira tables and their columns are described in [`docs/DATA_SOURCES.md`](../../docs/DATA_SOURCES.md). At runtime, inspect the live schema with `agnes schema ` and `agnes describe
`. ## Historical Backfill diff --git a/dev_docs/README.md b/dev_docs/README.md index 6ccd32d..eba3b0f 100644 --- a/dev_docs/README.md +++ b/dev_docs/README.md @@ -2,30 +2,32 @@ This folder contains documentation for **developers and server administrators** only. -**⚠️ This folder is NOT synced to analyst machines** - it stays on the server and in the git repository only. +**⚠️ This folder is NOT synced to analyst machines** — it stays on the server and in the git repository only. ## Contents ### Server Administration -- `server.md` - Data broker server configuration and management -- `disaster-recovery.md` - Recovery procedures for server failures -- `security.md` - Security audit report and hardening guidelines -- `jira.md` - Jira webhook integration and server-side processing +- `server.md` — data broker server configuration and management +- `disaster-recovery.md` — recovery procedures for server failures +- `security.md` — security audit report and hardening guidelines ### Application Development -- `desktop-app.md` - macOS desktop app architecture and development -- `telegram_bot.md` - Telegram notification bot technical docs -- `design-system.md` - UI/UX design system for web applications -- `insights.md` - Activity Center dashboard feature documentation +- `desktop-app.md` — macOS desktop app architecture and development +- `telegram_bot.md` — Telegram notification bot technical docs +- `design-system.md` — UI/UX design system for web applications +- `insights.md` — Activity Center dashboard feature documentation +- `session_explore.md` — session exploration tooling -### Planning & Meetings -- `meetings/` - Meeting transcripts, summaries, and notes +Jira webhook integration and server-side processing is documented in +[`../connectors/jira/README.md`](../connectors/jira/README.md). ## For Analysts -If you're an analyst looking for documentation on how to **use** the platform, see the `docs/` folder instead: -- `docs/GETTING_STARTED.md` - Quick start guide -- `docs/data_description.md` - Data schemas and table definitions -- `docs/metrics/` - Business metric definitions -- `docs/jira_schema.md` - Jira data schema reference -- `docs/notifications.md` - How to use Telegram notifications in your scripts +If you're an analyst looking for documentation on how to **use** the platform, +see the `docs/` folder instead — start at [`../docs/README.md`](../docs/README.md) +for the full index. Key entry points: + +- `docs/QUICKSTART.md` — quick start guide +- `docs/DATA_SOURCES.md` — data sources and table definitions +- `docs/metrics/` — business metric definitions +- `docs/HOWTO/` — task-oriented analyst cookbook diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 224484b..fcfcdb4 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -86,4 +86,4 @@ Open the project in Claude Code. The CLAUDE.md file will guide the AI assistant ## Hackathon -See [`HACKATHON.md`](HACKATHON.md) for the deploy-and-develop playbook. Per-developer dev VMs are the supported pattern — point your VM at your branch image with `gcloud compute ssh --command "sudo sed -i 's/^AGNES_TAG=.*/AGNES_TAG=dev-/' /opt/agnes/.env && sudo /usr/local/bin/agnes-auto-upgrade.sh"`. +See [`archive/HACKATHON.md`](archive/HACKATHON.md) for the deploy-and-develop playbook (archived event runbook). Per-developer dev VMs are the supported pattern — point your VM at your branch image with `gcloud compute ssh --command "sudo sed -i 's/^AGNES_TAG=.*/AGNES_TAG=dev-/' /opt/agnes/.env && sudo /usr/local/bin/agnes-auto-upgrade.sh"`. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..7ffe226 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,73 @@ +# Agnes documentation + +Index of all documentation, organized by who needs it. New here? Start with the +row that matches your role. + +| You are… | Start with | +|----------|-----------| +| **Analyst** — using Agnes to query data | [`QUICKSTART.md`](QUICKSTART.md), then [`HOWTO/`](HOWTO/) | +| **Operator** — deploying & running an instance | [`PLATFORM_SETUP.md`](PLATFORM_SETUP.md) | +| **Developer** — working on Agnes itself | [`../ARCHITECTURE.md`](../ARCHITECTURE.md) + [`architecture.md`](architecture.md) | + +--- + +## For analysts + +Using the platform to analyze data. + +- [`QUICKSTART.md`](QUICKSTART.md) — local setup + first sync +- [`HOWTO/`](HOWTO/) — task-oriented cookbook (querying, snapshots, common workflows) +- [`DATA_SOURCES.md`](DATA_SOURCES.md) — data source connectors (Keboola, BigQuery, CSV) and how tables surface +- [`metrics/`](metrics/) — canonical business-metric definitions (YAML) +- [`HEADLESS_USAGE.md`](HEADLESS_USAGE.md) — PAT auth for CI / headless clients + +## For operators + +Deploying, configuring, and running an Agnes instance. + +- [`PLATFORM_SETUP.md`](PLATFORM_SETUP.md) — **the consolidated operator playbook** (bootstrap, TLS, marketplaces, scheduler, telemetry) +- [`ONBOARDING.md`](ONBOARDING.md) — end-to-end Terraform deployment into a new GCP project +- [`DEPLOYMENT.md`](DEPLOYMENT.md) — picks between the Terraform and Docker Compose paths +- [`CONFIGURATION.md`](CONFIGURATION.md) — `instance.yaml`, env vars, per-instance options +- [`state-dir.md`](state-dir.md) — persistent data layout (`data` + `state` tiers, mount layouts, migration) +- [`RBAC.md`](RBAC.md) — access control: groups, members, resource grants +- [`auth-google-oauth.md`](auth-google-oauth.md) — Google OAuth setup + operator gotchas +- [`auth-groups.md`](auth-groups.md) — Google Workspace group sync +- [`admin/query-modes.md`](admin/query-modes.md) — table registration query modes +- [`agent-setup-prompt.md`](agent-setup-prompt.md) — customize the `/setup` page banner +- [`agent-workspace-prompt.md`](agent-workspace-prompt.md) — customize the generated analyst `CLAUDE.md` +- [`initial-workspace-override.md`](initial-workspace-override.md) — per-instance analyst-workspace skeleton override +- [`curated-marketplace-format.md`](curated-marketplace-format.md) — authoring `marketplace-metadata.json` for curated marketplaces +- [`observability.md`](observability.md) — PostHog integration (exceptions, tracing, session replay) +- [`operator/news-content-guide.md`](operator/news-content-guide.md) — editorial guidelines for in-app news content + +## For developers + +Working on the Agnes codebase. + +- [`../ARCHITECTURE.md`](../ARCHITECTURE.md) — high-level system overview (the summary) +- [`architecture.md`](architecture.md) — detailed architecture reference (module map, extract.duckdb contract, components) +- [`../CLAUDE.md`](../CLAUDE.md) — project instructions for AI agents working in this repo +- [`development.md`](development.md) — logging, request correlation, debug toolbar +- [`local-development.md`](local-development.md) — `LOCAL_DEV_MODE` setup (what's mocked vs. real) +- [`RELEASING.md`](RELEASING.md) — release process, deploy workflows, CI quirks +- [`RELEASE_CHECKLIST.md`](RELEASE_CHECKLIST.md) — pre-merge checks for bootstrap-path changes +- [`testing/`](testing/) — test plans (clean-analyst bootstrap, VM test) +- [`marketplace.md`](marketplace.md) — Claude Code marketplace ingestion + re-serving internals +- [`STORE_GUARDRAILS.md`](STORE_GUARDRAILS.md) — flea-market upload guardrails (static checks + LLM review) +- [`corporate-memory-governance.md`](corporate-memory-governance.md) — knowledge-distribution governance design +- [`ADR-corporate-memory-v1.md`](ADR-corporate-memory-v1.md) — ADR: corporate-memory v1 decisions +- [`llm-routing.md`](llm-routing.md) — design: provider-agnostic LLM routing +- [`sample-data.md`](sample-data.md) — sample data generator (e-commerce schema, size presets) +- [`theme-reference.html`](theme-reference.html) — web UI theme/color reference +- [`../dev_docs/`](../dev_docs/) — **server/developer-internal docs** (not synced to analyst machines): server ops, disaster recovery, security audit, desktop app, design system, Telegram bot + +Code-adjacent READMEs: [`../connectors/jira/README.md`](../connectors/jira/README.md), +[`../services/corporate_memory/README.md`](../services/corporate_memory/README.md), +[`../scripts/README.md`](../scripts/README.md). +Agent skill files: [`../cli/skills/`](../cli/skills/). + +## Other + +- [`../CHANGELOG.md`](../CHANGELOG.md) — full change history (Keep-a-Changelog format) +- [`archive/`](archive/) — historical planning artifacts and superseded docs; not maintained, see [`archive/README.md`](archive/README.md) diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md index b7e8cf6..85c8559 100644 --- a/docs/RELEASE_CHECKLIST.md +++ b/docs/RELEASE_CHECKLIST.md @@ -16,7 +16,7 @@ requesting review: click the analyst tile and copy the paste prompt. 4. Paste into Claude Code and let it run. `tree -a /tmp/test-analyst-1` and compare with the expected tree from the design spec - (`docs/superpowers/specs/2026-05-04-clean-analyst-bootstrap-design.md` §5.2). + (`docs/archive/superpowers/specs/2026-05-04-clean-analyst-bootstrap-design.md` §5.2). 5. `claude` in that folder. Three queries: - "What tables can I see?" - "SELECT count(\*) FROM " (a table from the catalog) diff --git a/docs/RELEASE_TEMPLATE.md b/docs/RELEASE_TEMPLATE.md deleted file mode 100644 index 3c32b6d..0000000 --- a/docs/RELEASE_TEMPLATE.md +++ /dev/null @@ -1,37 +0,0 @@ -# Release Notes Template - -Use this template when adding a new entry to `CHANGELOG.md`. - ---- - -## stable-YYYY.MM.N - -**Image:** `ghcr.io/keboola/agnes-the-ai-analyst:stable-YYYY.MM.N` -**Digest:** `sha256:...` (from `docker inspect --format='{{index .RepoDigests 0}}'`) -**Date:** YYYY-MM-DD - -### Added -- Feature description - -### Changed -- Change description - -### Fixed -- Bug fix description - -### Breaking Changes -- Description of breaking change -- **Migration guide:** Steps to upgrade from previous version - -### Deprecated -- Description of deprecated feature (will be removed in YYYY.MM.N) - ---- - -## Guidelines - -- Every merge to `main` creates a new `stable-YYYY.MM.N` release -- Include the image digest for verification with `cosign verify` -- Breaking changes require `BREAKING:` prefix in commit message -- Migration guides must include exact commands or config changes -- If a release deprecates the previous stable, note it explicitly diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 0000000..2c5d72a --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,270 @@ +# Releasing & deploying + +The full release process for Agnes. CLAUDE.md carries the short version; this +doc is the operational reference. Read it linearly the first few times — once +internalized, the order matters less, but the **non-obvious gotchas never go +away**. + +## Changelog discipline — non-negotiable + +**Every PR that adds, removes, or changes user-visible behavior MUST update +`CHANGELOG.md` in the same PR.** No exceptions, no follow-ups, no "I'll do it +after merge". User-visible = anything an operator, end-user, or downstream +integrator can observe: CLI flags / output / exit codes, REST endpoints / +payloads / status codes, web UI, `instance.yaml` schema, env vars, +`extract.duckdb` contract, Docker / compose / Caddyfile knobs, default +behaviors, breaking changes, security fixes. + +**How:** +- Add a bullet under the topmost `## [Unreleased]` heading (create one if + missing — it sits above the latest released version). +- Group by `### Added` / `### Changed` / `### Fixed` / `### Removed` / + `### Internal` (Keep-a-Changelog sections). +- Mark breaking changes with `**BREAKING**` at the start of the bullet — + operators grep for that string before bumping the pin. +- Reference the relevant doc/runbook if one exists (e.g. + `see docs/auth-groups.md`), don't restate it. +- Internal-only changes (refactors, test additions, dependency bumps without + behavior change) go under `### Internal` — still log them, just keep them + terse. + +Reviewers should bounce PRs that touch user-visible behavior without a +changelog update — same way they'd bounce a PR with no test changes for new +logic. + +## Release-cut belongs to the PR — non-negotiable + +**The version bump + CHANGELOG rename + new empty `[Unreleased]` are the LAST +commit on the PR that earned the version. Never a standalone follow-up PR.** + +When a PR lands the only `[Unreleased]` content (or is the last in a queue of +in-flight feature PRs), the release-cut MUST ship as part of the same merge. +Standalone release-cut PRs add review-overhead PRs to history with no behavior +change of their own and pollute `git log` with bookkeeping commits separated +from the work that earned them. + +**Mandatory checklist before approving / enabling auto-merge on ANY PR:** + +1. **Stop.** Will this PR land alone in `[Unreleased]` (no other in-flight PRs + queued behind it)? +2. **If yes**, the release-cut is REQUIRED in the same PR before merge. BEFORE + pushing the final commit: + - Bump `pyproject.toml` to `X.Y.Z` + - Rename `## [Unreleased]` → `## [X.Y.Z] — YYYY-MM-DD`, add a new empty + `## [Unreleased]` on top + - Either squash these into the consolidation commit OR add as a separate + `release: X.Y.Z` commit on the same branch +3. **THEN** push, approve, enable auto-merge. +4. After auto-merge fires: tag `vX.Y.Z` against the merge commit + create a + GitHub Release. Done — one PR, one merge, one release. + +**Failure mode to avoid:** enabling auto-merge on the feature PR thinking "I'll +add the release-cut after." Auto-merge fires faster than the second commit +lands. The window closes; the only fix is a standalone release-cut PR — exactly +what this rule prohibits. + +**Acceptable standalone release-cut** (rare): only when `[Unreleased]` +accumulated bullets from MULTIPLE already-merged PRs AND no further +behavior-change PR is queued — i.e. the cut is the only outstanding work and +there's no PR to attach it to. + +## Release workflow — concrete recipe + +### Happy path (8 steps) + +```bash +# 1. Branch from a fresh checkout. iCloud Drive worktrees randomly hang +# on git operations — use a fresh shallow clone in /tmp instead. +cd /tmp && git clone --depth 50 --branch main \ + https://github.com/keboola/agnes-the-ai-analyst.git agnes- +cd agnes- && git checkout -b zs/ + +# 2. Make the change + tests. Run the AREA pytest while iterating +# (e.g. `pytest tests/test_X.py -p no:xdist -q`). + +# 3. Add a CHANGELOG bullet under [Unreleased]. +# Group: Added | Changed | Fixed | Removed | Internal +# Mark BREAKING with **BREAKING** prefix. + +# 4. Commit the change(s). Multiple logical commits OK; release-cut +# will be a SEPARATE last commit (next step). DO NOT bundle the +# release-cut into the same commit as the change — it pollutes +# the SHA that auto-close keywords reference and makes revert +# targeted at the change-only difficult. + +# 5. Run the full pytest suite locally: +# `pytest tests/ -p no:xdist -q` (or `-n auto` if xdist works). +# Pre-existing fails (e.g. test_readers_in_pre_init_dir under +# subprocess timeout) are OK to ignore; verify by reverting your +# diff and reproducing on bare main. + +# 6. Release-cut commit (LAST commit on the PR per the rule above): +# - Bump pyproject.toml: version = "X.Y.Z" +# - Rename `## [Unreleased]` → `## [X.Y.Z] — YYYY-MM-DD` +# - Add a fresh empty `## [Unreleased]` line above +# Commit message: `release: X.Y.Z — ` + +# 7. Push branch + open PR + enable auto-merge SQUASH: +# git push -u origin HEAD +# gh pr create --repo keboola/agnes-the-ai-analyst \ +# --head --title "<...>" --body "<...>" +# gh pr merge --repo keboola/agnes-the-ai-analyst \ +# --squash --auto --delete-branch + +# 8. After auto-merge fires (poll or `Monitor`): +# git fetch origin --tags +# git tag vX.Y.Z +# git push origin vX.Y.Z +# gh release create vX.Y.Z --repo keboola/agnes-the-ai-analyst \ +# --title "vX.Y.Z — <...>" --notes "" +``` + +### Picking the next version + +`pyproject.toml`'s current `version` is the **next-release target** (post-cut +from the previous release). Pre-1.0 we patch-bump for everything that doesn't +break operator-facing APIs: + +- `instance.yaml` schema additions, new env vars, new endpoints → patch (e.g. + 0.54.3 → 0.54.4) +- New CLI subcommands, BREAKING removals, schema migrations → still patch within + the current 0.5x cycle (no minor bumps cut today) +- The CHANGELOG `**BREAKING**` marker is what operators grep for; the version + number is secondary + +Always check `git tag -l "v0.X*"` before naming — if `v0.54.0` is already +tagged, the next one is `v0.54.1`, even if `pyproject.toml` still says `0.54.0` +from a stale post-cut commit (we've shipped that race before). + +### Authoring expectations on the PR + +- **Self-PRs** (you're both author and reviewer): GitHub forbids self-approve. + If branch protection requires N approving reviews (we don't today — + `required_approving_review_count = 0`), you need someone else to approve. With + our current 0-review setup, self-PRs can still merge automatically once + required CI passes. +- **Other people's PRs you're taking over**: dismiss any prior + CHANGES_REQUESTED reviews (yours or someone else's) before auto-merge can + fire. `gh pr review --approve --body "..."` after pushing your fixes. +- **Devin Review**: not a required check today; runs in parallel and posts a + comment. Don't wait on it for merge unless the human reviewer explicitly asks. + +### CI quirks you WILL hit + +- **`gh pr checks` glosses CANCELLED as `fail`.** When you force-push (rebase, + amend), GitHub auto-cancels the in-flight `Release` workflow run on the older + SHA. Those cancelled jobs show up as "fail" in the PR's check summary and tab + forever, even after newer runs succeed. **Look at the conclusion column, not + just the count.** Rule of thumb: if the same check name appears with both + `pass` and `fail` rows, the `fail` row is from an older auto-cancelled SHA. + Verify with `gh api repos/keboola/agnes-the-ai-analyst/commits//check-runs` + — the raw API distinguishes `cancelled` from `failure` truthfully. +- **Branch protection's "strict" mode caches cancelled `test` as blocking** even + after newer `test` runs succeed. Symptom: `mergeable_state: blocked` despite + all required checks green on the latest SHA. Fix: re-run the cancelled + `Release` workflow run (`gh run rerun `); once its `test` job lands as + success, the block clears. We've hit this on PRs #273, #281, #285, #286. +- **Required checks** (per branch protection): `test` + `docker-build` only. + Other workflows (`cli-wheel-clean-install`, `build-and-push`, + `Release`-pipeline, Devin Review) are advisory — green/red doesn't gate merge. +- **`enforce_admins: true`** in branch protection means `--admin` flag on + `gh pr merge` does NOT bypass. Don't try; just fix the underlying block. + +### Recovery when something derails + +- **Force-pushed and lost auto-merge?** GitHub *usually* preserves auto-merge + across force-pushes for the same PR; if it cleared, just re-run + `gh pr merge --squash --auto --delete-branch`. +- **Release-cut commit forgot to land?** That's the failure mode the + "Release-cut belongs to the PR" rule prevents. If it happens anyway: open a + follow-on PR with ONLY the release-cut commit, ship it, and write up why in + your post-mortem comment. +- **Wrong version number tagged?** `git tag -d vX.Y.Z && git push --delete + origin vX.Y.Z` then re-tag against the right SHA. Update the GitHub Release if + you already created it. + +## Deploy workflows + +Two separate release.yml-style workflows produce GHCR images. Pick the one that +matches what you're shipping. + +### `release.yml` — auto-build on every push + +Runs on **every** push to **every** branch. +- Push to `main` → `:stable`, `:stable-YYYY.MM.N` (CalVer). +- Push to non-main `/` → `:dev`, `:dev-YYYY.MM.N`, + `:dev-`, and (when prefix isn't a Git Flow convention) + `:dev--latest` alias. + +VMs that pin to a floating tag (`:dev`, `:dev--latest`) auto-upgrade +within ~5 min via the cron in `agnes-auto-upgrade.sh`. Convenient for +per-developer dev VMs; **footgun for shared dev VMs** (last pusher wins, +regardless of who). + +### `keboola-deploy.yml` — tag-triggered, explicit deploy only + +Runs **only** on git tags matching `keboola-deploy-*`. Publishes: +- `:keboola-deploy-` — immutable, tied to the exact commit +- `:keboola-deploy-latest` — floating alias the consumer pins to + +**Operator workflow:** +```bash +git checkout +git tag keboola-deploy- +git push origin keboola-deploy- +# → workflow builds + publishes both tags +# → VM cron picks up :keboola-deploy-latest within ~5 min +# → manual cron trigger (skip the wait): sudo /usr/local/bin/agnes-auto-upgrade.sh on the VM +``` + +Use this when the consumer (e.g. a customer dev VM) needs +**deploy-when-I-decide** semantics — no surprise rollouts from upstream branch +pushes by other contributors. The infra repo pins +`image_tag = "keboola-deploy-latest"` on the relevant VM. + +### Module versioning + +The customer-instance Terraform module under `infra/modules/customer-instance/` +is published as `infra-vMAJOR.MINOR.PATCH` git tags (separate from app CalVer +tags). Bump on any module-API change; downstream infra repos pin to the tag in +their `source = "github.com/keboola/agnes-the-ai-analyst//infra/modules/customer-instance?ref=infra-v1.X.Y"`. + +After merging a module change to `main`: +```bash +git tag infra-vX.Y.Z origin/main +git push origin infra-vX.Y.Z +``` + +### Replacing a VM after a startup-script change + +Module sets `lifecycle { ignore_changes = [metadata_startup_script] }` on +`google_compute_instance.vm` so normal `terraform apply` doesn't churn running +VMs. To propagate a startup-script update, trigger the consumer's apply workflow +manually with the VM resource address — typical workflow_dispatch input is +`recreate_targets='module.agnes.google_compute_instance.vm[""]'`. + +## Appendix: CHANGELOG entry skeleton + +Copy this when adding to `## [Unreleased]` in `CHANGELOG.md`. Drop the sections +you don't need; keep the Keep-a-Changelog order. + +```markdown +### Added +- New feature description. + +### Changed +- Change description. **BREAKING** prefix + migration steps if operator-facing. + +### Fixed +- Bug fix description. + +### Removed +- **BREAKING** removed feature — what replaces it. + +### Internal +- Refactors, test additions, dependency bumps with no behavior change. +``` + +At release-cut time `## [Unreleased]` is renamed to `## [X.Y.Z] — YYYY-MM-DD` +and a fresh empty `## [Unreleased]` is added on top. CI publishes the matching +`stable-YYYY.MM.N` image tag for the merge commit (see Deploy workflows above). diff --git a/docs/HACKATHON.md b/docs/archive/HACKATHON.md similarity index 100% rename from docs/HACKATHON.md rename to docs/archive/HACKATHON.md diff --git a/docs/future/NOTIFICATIONS.md b/docs/archive/NOTIFICATIONS.md similarity index 100% rename from docs/future/NOTIFICATIONS.md rename to docs/archive/NOTIFICATIONS.md diff --git a/docs/archive/README.md b/docs/archive/README.md new file mode 100644 index 0000000..8f0cca8 --- /dev/null +++ b/docs/archive/README.md @@ -0,0 +1,31 @@ +# Archived documentation + +Historical artifacts kept for reference but **not maintained**. Nothing here is +current guidance — paths, line numbers, and APIs cited inside these files +reflect the state of the repo when they were written. For live docs, start at +[`../README.md`](../README.md). + +## Contents + +- **`superpowers/`** — implementation plans (`plans/`) and design specs + (`specs/`) from past development sprints. Each file is a point-in-time + planning artifact for a feature that has since shipped (or been dropped). + Useful as a record of *why* something was built a certain way; useless as a + guide to *what the code does now* — read the code and `CHANGELOG.md` for that. +- **`HACKATHON.md`** — condensed deploy + dev playbook written for a hackathon + sprint. Superseded by [`../QUICKSTART.md`](../QUICKSTART.md), + [`../DEPLOYMENT.md`](../DEPLOYMENT.md), and [`../ONBOARDING.md`](../ONBOARDING.md). +- **`NOTIFICATIONS.md`** — early spec for Telegram notifications via scripts + + crontab. The feature shipped as a service; see + [`../../dev_docs/telegram_bot.md`](../../dev_docs/telegram_bot.md). +- **`pd-ps-comments.md`** — review notes on the corporate-memory v1 branch. + Accepted decisions live in [`../ADR-corporate-memory-v1.md`](../ADR-corporate-memory-v1.md). +- **`security-audit-2026-04.md`** — point-in-time security audit snapshot. + Findings addressed in later releases; see `CHANGELOG.md`. + +## Policy + +Don't edit archived files to "fix" stale references — rewriting historical +planning artifacts is revisionist and loses the record of what was actually +decided when. If an archived doc is genuinely worthless, delete it; otherwise +leave it as-is. diff --git a/docs/pd-ps-comments.md b/docs/archive/pd-ps-comments.md similarity index 100% rename from docs/pd-ps-comments.md rename to docs/archive/pd-ps-comments.md diff --git a/docs/security-audit-2026-04.md b/docs/archive/security-audit-2026-04.md similarity index 98% rename from docs/security-audit-2026-04.md rename to docs/archive/security-audit-2026-04.md index 93f0ca7..0f58d2e 100644 --- a/docs/security-audit-2026-04.md +++ b/docs/archive/security-audit-2026-04.md @@ -1,5 +1,10 @@ # Security audit — Agnes AI Data Analyst +> **Archived.** This is a point-in-time audit snapshot. Findings have been +> triaged and addressed in subsequent releases — check `CHANGELOG.md` (search +> for the relevant file/endpoint) for the fixes that landed since 2026-04-22. +> Kept for historical reference; not a live tracker. + **Date:** 2026-04-22 **Branch audited:** `main` at commit `cbb7733` **Method:** parallel review passes over four scope areas — (1) secrets/SQLi/authz/SSRF, (2) auth flows & route wiring, (3) templates & UI wiring/XSS, (4) data layer & config & dead code. Findings deduped across the passes, severities adjusted to real-world exploitability. diff --git a/docs/superpowers/plans/2026-03-27-01-duckdb-state-layer.md b/docs/archive/superpowers/plans/2026-03-27-01-duckdb-state-layer.md similarity index 100% rename from docs/superpowers/plans/2026-03-27-01-duckdb-state-layer.md rename to docs/archive/superpowers/plans/2026-03-27-01-duckdb-state-layer.md diff --git a/docs/superpowers/plans/2026-03-27-02-complete-system.md b/docs/archive/superpowers/plans/2026-03-27-02-complete-system.md similarity index 100% rename from docs/superpowers/plans/2026-03-27-02-complete-system.md rename to docs/archive/superpowers/plans/2026-03-27-02-complete-system.md diff --git a/docs/superpowers/plans/2026-04-08-final-integration-fixes.md b/docs/archive/superpowers/plans/2026-04-08-final-integration-fixes.md similarity index 100% rename from docs/superpowers/plans/2026-04-08-final-integration-fixes.md rename to docs/archive/superpowers/plans/2026-04-08-final-integration-fixes.md diff --git a/docs/superpowers/plans/2026-04-08-production-hardening.md b/docs/archive/superpowers/plans/2026-04-08-production-hardening.md similarity index 100% rename from docs/superpowers/plans/2026-04-08-production-hardening.md rename to docs/archive/superpowers/plans/2026-04-08-production-hardening.md diff --git a/docs/superpowers/plans/2026-04-08-security-hardening.md b/docs/archive/superpowers/plans/2026-04-08-security-hardening.md similarity index 100% rename from docs/superpowers/plans/2026-04-08-security-hardening.md rename to docs/archive/superpowers/plans/2026-04-08-security-hardening.md diff --git a/docs/superpowers/plans/2026-04-09-dead-code-cleanup.md b/docs/archive/superpowers/plans/2026-04-09-dead-code-cleanup.md similarity index 100% rename from docs/superpowers/plans/2026-04-09-dead-code-cleanup.md rename to docs/archive/superpowers/plans/2026-04-09-dead-code-cleanup.md diff --git a/docs/superpowers/plans/2026-04-09-deployment-readiness.md b/docs/archive/superpowers/plans/2026-04-09-deployment-readiness.md similarity index 100% rename from docs/superpowers/plans/2026-04-09-deployment-readiness.md rename to docs/archive/superpowers/plans/2026-04-09-deployment-readiness.md diff --git a/docs/superpowers/plans/2026-04-09-final-polish.md b/docs/archive/superpowers/plans/2026-04-09-final-polish.md similarity index 100% rename from docs/superpowers/plans/2026-04-09-final-polish.md rename to docs/archive/superpowers/plans/2026-04-09-final-polish.md diff --git a/docs/superpowers/plans/2026-04-09-security-fixes.md b/docs/archive/superpowers/plans/2026-04-09-security-fixes.md similarity index 100% rename from docs/superpowers/plans/2026-04-09-security-fixes.md rename to docs/archive/superpowers/plans/2026-04-09-security-fixes.md diff --git a/docs/superpowers/plans/2026-04-10-analyst-bootstrap.md b/docs/archive/superpowers/plans/2026-04-10-analyst-bootstrap.md similarity index 100% rename from docs/superpowers/plans/2026-04-10-analyst-bootstrap.md rename to docs/archive/superpowers/plans/2026-04-10-analyst-bootstrap.md diff --git a/docs/superpowers/plans/2026-04-10-business-metrics.md b/docs/archive/superpowers/plans/2026-04-10-business-metrics.md similarity index 100% rename from docs/superpowers/plans/2026-04-10-business-metrics.md rename to docs/archive/superpowers/plans/2026-04-10-business-metrics.md diff --git a/docs/superpowers/plans/2026-04-10-metadata-writer.md b/docs/archive/superpowers/plans/2026-04-10-metadata-writer.md similarity index 100% rename from docs/superpowers/plans/2026-04-10-metadata-writer.md rename to docs/archive/superpowers/plans/2026-04-10-metadata-writer.md diff --git a/docs/superpowers/plans/2026-04-11-remote-query.md b/docs/archive/superpowers/plans/2026-04-11-remote-query.md similarity index 100% rename from docs/superpowers/plans/2026-04-11-remote-query.md rename to docs/archive/superpowers/plans/2026-04-11-remote-query.md diff --git a/docs/superpowers/plans/2026-04-12-comprehensive-test-suite.md b/docs/archive/superpowers/plans/2026-04-12-comprehensive-test-suite.md similarity index 100% rename from docs/superpowers/plans/2026-04-12-comprehensive-test-suite.md rename to docs/archive/superpowers/plans/2026-04-12-comprehensive-test-suite.md diff --git a/docs/superpowers/plans/2026-04-21-deployment-log.md b/docs/archive/superpowers/plans/2026-04-21-deployment-log.md similarity index 100% rename from docs/superpowers/plans/2026-04-21-deployment-log.md rename to docs/archive/superpowers/plans/2026-04-21-deployment-log.md diff --git a/docs/superpowers/plans/2026-04-21-hackathon-dry-run.md b/docs/archive/superpowers/plans/2026-04-21-hackathon-dry-run.md similarity index 100% rename from docs/superpowers/plans/2026-04-21-hackathon-dry-run.md rename to docs/archive/superpowers/plans/2026-04-21-hackathon-dry-run.md diff --git a/docs/superpowers/plans/2026-04-21-issues-14-and-10.md b/docs/archive/superpowers/plans/2026-04-21-issues-14-and-10.md similarity index 100% rename from docs/superpowers/plans/2026-04-21-issues-14-and-10.md rename to docs/archive/superpowers/plans/2026-04-21-issues-14-and-10.md diff --git a/docs/superpowers/plans/2026-04-21-multi-customer-deployment.md b/docs/archive/superpowers/plans/2026-04-21-multi-customer-deployment.md similarity index 100% rename from docs/superpowers/plans/2026-04-21-multi-customer-deployment.md rename to docs/archive/superpowers/plans/2026-04-21-multi-customer-deployment.md diff --git a/docs/superpowers/plans/2026-04-21-user-mgmt-pat-cli.md b/docs/archive/superpowers/plans/2026-04-21-user-mgmt-pat-cli.md similarity index 100% rename from docs/superpowers/plans/2026-04-21-user-mgmt-pat-cli.md rename to docs/archive/superpowers/plans/2026-04-21-user-mgmt-pat-cli.md diff --git a/docs/superpowers/plans/2026-04-22-cloudflare-access-auth.md b/docs/archive/superpowers/plans/2026-04-22-cloudflare-access-auth.md similarity index 100% rename from docs/superpowers/plans/2026-04-22-cloudflare-access-auth.md rename to docs/archive/superpowers/plans/2026-04-22-cloudflare-access-auth.md diff --git a/docs/superpowers/plans/2026-04-27-bq-pipeline-views-and-metadata-auth.md b/docs/archive/superpowers/plans/2026-04-27-bq-pipeline-views-and-metadata-auth.md similarity index 100% rename from docs/superpowers/plans/2026-04-27-bq-pipeline-views-and-metadata-auth.md rename to docs/archive/superpowers/plans/2026-04-27-bq-pipeline-views-and-metadata-auth.md diff --git a/docs/superpowers/plans/2026-04-27-claude-fetch-primitives.md b/docs/archive/superpowers/plans/2026-04-27-claude-fetch-primitives.md similarity index 100% rename from docs/superpowers/plans/2026-04-27-claude-fetch-primitives.md rename to docs/archive/superpowers/plans/2026-04-27-claude-fetch-primitives.md diff --git a/docs/superpowers/plans/2026-04-29-dev-debug-toolbar.md b/docs/archive/superpowers/plans/2026-04-29-dev-debug-toolbar.md similarity index 100% rename from docs/superpowers/plans/2026-04-29-dev-debug-toolbar.md rename to docs/archive/superpowers/plans/2026-04-29-dev-debug-toolbar.md diff --git a/docs/superpowers/plans/2026-04-29-issue-134-bq-access-unify-plan.md b/docs/archive/superpowers/plans/2026-04-29-issue-134-bq-access-unify-plan.md similarity index 100% rename from docs/superpowers/plans/2026-04-29-issue-134-bq-access-unify-plan.md rename to docs/archive/superpowers/plans/2026-04-29-issue-134-bq-access-unify-plan.md diff --git a/docs/superpowers/plans/2026-04-29-issues-77-78-79-89.md b/docs/archive/superpowers/plans/2026-04-29-issues-77-78-79-89.md similarity index 100% rename from docs/superpowers/plans/2026-04-29-issues-77-78-79-89.md rename to docs/archive/superpowers/plans/2026-04-29-issues-77-78-79-89.md diff --git a/docs/superpowers/plans/2026-04-30-customizable-welcome-prompt.md b/docs/archive/superpowers/plans/2026-04-30-customizable-welcome-prompt.md similarity index 100% rename from docs/superpowers/plans/2026-04-30-customizable-welcome-prompt.md rename to docs/archive/superpowers/plans/2026-04-30-customizable-welcome-prompt.md diff --git a/docs/superpowers/plans/2026-05-01-admin-tables-form-cleanup.md b/docs/archive/superpowers/plans/2026-05-01-admin-tables-form-cleanup.md similarity index 100% rename from docs/superpowers/plans/2026-05-01-admin-tables-form-cleanup.md rename to docs/archive/superpowers/plans/2026-05-01-admin-tables-form-cleanup.md diff --git a/docs/superpowers/plans/2026-05-04-clean-analyst-bootstrap.md b/docs/archive/superpowers/plans/2026-05-04-clean-analyst-bootstrap.md similarity index 100% rename from docs/superpowers/plans/2026-05-04-clean-analyst-bootstrap.md rename to docs/archive/superpowers/plans/2026-05-04-clean-analyst-bootstrap.md diff --git a/docs/superpowers/plans/2026-05-04-unified-setup-prompt.md b/docs/archive/superpowers/plans/2026-05-04-unified-setup-prompt.md similarity index 100% rename from docs/superpowers/plans/2026-05-04-unified-setup-prompt.md rename to docs/archive/superpowers/plans/2026-05-04-unified-setup-prompt.md diff --git a/docs/superpowers/plans/2026-05-06-cli-auto-upgrade.md b/docs/archive/superpowers/plans/2026-05-06-cli-auto-upgrade.md similarity index 100% rename from docs/superpowers/plans/2026-05-06-cli-auto-upgrade.md rename to docs/archive/superpowers/plans/2026-05-06-cli-auto-upgrade.md diff --git a/docs/superpowers/plans/2026-05-07-source-agnostic-table-metadata.md b/docs/archive/superpowers/plans/2026-05-07-source-agnostic-table-metadata.md similarity index 100% rename from docs/superpowers/plans/2026-05-07-source-agnostic-table-metadata.md rename to docs/archive/superpowers/plans/2026-05-07-source-agnostic-table-metadata.md diff --git a/docs/superpowers/plans/2026-05-11-activity-center-mvp.md b/docs/archive/superpowers/plans/2026-05-11-activity-center-mvp.md similarity index 100% rename from docs/superpowers/plans/2026-05-11-activity-center-mvp.md rename to docs/archive/superpowers/plans/2026-05-11-activity-center-mvp.md diff --git a/docs/superpowers/plans/2026-05-11-admin-observability-spec.md b/docs/archive/superpowers/plans/2026-05-11-admin-observability-spec.md similarity index 100% rename from docs/superpowers/plans/2026-05-11-admin-observability-spec.md rename to docs/archive/superpowers/plans/2026-05-11-admin-observability-spec.md diff --git a/docs/superpowers/plans/2026-05-12-platform-telemetry-epic.md b/docs/archive/superpowers/plans/2026-05-12-platform-telemetry-epic.md similarity index 100% rename from docs/superpowers/plans/2026-05-12-platform-telemetry-epic.md rename to docs/archive/superpowers/plans/2026-05-12-platform-telemetry-epic.md diff --git a/docs/superpowers/plans/2026-05-13-design-system-unification.md b/docs/archive/superpowers/plans/2026-05-13-design-system-unification.md similarity index 100% rename from docs/superpowers/plans/2026-05-13-design-system-unification.md rename to docs/archive/superpowers/plans/2026-05-13-design-system-unification.md diff --git a/docs/superpowers/plans/2026-05-13-standalone-pages-framework.md b/docs/archive/superpowers/plans/2026-05-13-standalone-pages-framework.md similarity index 100% rename from docs/superpowers/plans/2026-05-13-standalone-pages-framework.md rename to docs/archive/superpowers/plans/2026-05-13-standalone-pages-framework.md diff --git a/docs/superpowers/specs/2026-03-27-refactoring-design.md b/docs/archive/superpowers/specs/2026-03-27-refactoring-design.md similarity index 100% rename from docs/superpowers/specs/2026-03-27-refactoring-design.md rename to docs/archive/superpowers/specs/2026-03-27-refactoring-design.md diff --git a/docs/superpowers/specs/2026-03-30-core-refactoring-design.md b/docs/archive/superpowers/specs/2026-03-30-core-refactoring-design.md similarity index 100% rename from docs/superpowers/specs/2026-03-30-core-refactoring-design.md rename to docs/archive/superpowers/specs/2026-03-30-core-refactoring-design.md diff --git a/docs/superpowers/specs/2026-03-31-data-access-control.md b/docs/archive/superpowers/specs/2026-03-31-data-access-control.md similarity index 100% rename from docs/superpowers/specs/2026-03-31-data-access-control.md rename to docs/archive/superpowers/specs/2026-03-31-data-access-control.md diff --git a/docs/superpowers/specs/2026-04-09-multi-instance-deployment-design.md b/docs/archive/superpowers/specs/2026-04-09-multi-instance-deployment-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-09-multi-instance-deployment-design.md rename to docs/archive/superpowers/specs/2026-04-09-multi-instance-deployment-design.md diff --git a/docs/superpowers/specs/2026-04-10-porting-internal-features-design.md b/docs/archive/superpowers/specs/2026-04-10-porting-internal-features-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-10-porting-internal-features-design.md rename to docs/archive/superpowers/specs/2026-04-10-porting-internal-features-design.md diff --git a/docs/superpowers/specs/2026-04-11-remote-query-design.md b/docs/archive/superpowers/specs/2026-04-11-remote-query-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-11-remote-query-design.md rename to docs/archive/superpowers/specs/2026-04-11-remote-query-design.md diff --git a/docs/superpowers/specs/2026-04-12-comprehensive-test-strategy-design.md b/docs/archive/superpowers/specs/2026-04-12-comprehensive-test-strategy-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-12-comprehensive-test-strategy-design.md rename to docs/archive/superpowers/specs/2026-04-12-comprehensive-test-strategy-design.md diff --git a/docs/superpowers/specs/2026-04-14-connector-kit-design.md b/docs/archive/superpowers/specs/2026-04-14-connector-kit-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-14-connector-kit-design.md rename to docs/archive/superpowers/specs/2026-04-14-connector-kit-design.md diff --git a/docs/superpowers/specs/2026-04-21-multi-customer-deployment-spec.md b/docs/archive/superpowers/specs/2026-04-21-multi-customer-deployment-spec.md similarity index 100% rename from docs/superpowers/specs/2026-04-21-multi-customer-deployment-spec.md rename to docs/archive/superpowers/specs/2026-04-21-multi-customer-deployment-spec.md diff --git a/docs/superpowers/specs/2026-04-27-claude-fetch-primitives-design.md b/docs/archive/superpowers/specs/2026-04-27-claude-fetch-primitives-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-27-claude-fetch-primitives-design.md rename to docs/archive/superpowers/specs/2026-04-27-claude-fetch-primitives-design.md diff --git a/docs/superpowers/specs/2026-04-29-dev-debug-toolbar-design.md b/docs/archive/superpowers/specs/2026-04-29-dev-debug-toolbar-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-29-dev-debug-toolbar-design.md rename to docs/archive/superpowers/specs/2026-04-29-dev-debug-toolbar-design.md diff --git a/docs/superpowers/specs/2026-04-29-issue-134-bq-access-unify-design.md b/docs/archive/superpowers/specs/2026-04-29-issue-134-bq-access-unify-design.md similarity index 100% rename from docs/superpowers/specs/2026-04-29-issue-134-bq-access-unify-design.md rename to docs/archive/superpowers/specs/2026-04-29-issue-134-bq-access-unify-design.md diff --git a/docs/superpowers/specs/2026-05-03-issue-160-da-query-remote-fix-spec.md b/docs/archive/superpowers/specs/2026-05-03-issue-160-da-query-remote-fix-spec.md similarity index 100% rename from docs/superpowers/specs/2026-05-03-issue-160-da-query-remote-fix-spec.md rename to docs/archive/superpowers/specs/2026-05-03-issue-160-da-query-remote-fix-spec.md diff --git a/docs/superpowers/specs/2026-05-04-clean-analyst-bootstrap-design.md b/docs/archive/superpowers/specs/2026-05-04-clean-analyst-bootstrap-design.md similarity index 100% rename from docs/superpowers/specs/2026-05-04-clean-analyst-bootstrap-design.md rename to docs/archive/superpowers/specs/2026-05-04-clean-analyst-bootstrap-design.md diff --git a/docs/superpowers/specs/2026-05-06-cli-auto-upgrade-spec.md b/docs/archive/superpowers/specs/2026-05-06-cli-auto-upgrade-spec.md similarity index 100% rename from docs/superpowers/specs/2026-05-06-cli-auto-upgrade-spec.md rename to docs/archive/superpowers/specs/2026-05-06-cli-auto-upgrade-spec.md diff --git a/docs/superpowers/specs/2026-05-07-source-agnostic-table-metadata-spec.md b/docs/archive/superpowers/specs/2026-05-07-source-agnostic-table-metadata-spec.md similarity index 100% rename from docs/superpowers/specs/2026-05-07-source-agnostic-table-metadata-spec.md rename to docs/archive/superpowers/specs/2026-05-07-source-agnostic-table-metadata-spec.md diff --git a/docs/auto-install.md b/docs/auto-install.md deleted file mode 100644 index 43ca72c..0000000 --- a/docs/auto-install.md +++ /dev/null @@ -1,5 +0,0 @@ -# Auto-Install Guide - -For deployment instructions, see [DEPLOYMENT.md](DEPLOYMENT.md). - -For local development setup, see the [README](../README.md#development). diff --git a/docs/marketplace.md b/docs/marketplace.md new file mode 100644 index 0000000..df4658b --- /dev/null +++ b/docs/marketplace.md @@ -0,0 +1,101 @@ +# Marketplace internals + +How Agnes ingests admin-registered Claude Code marketplaces and re-serves a +single aggregated, RBAC-filtered marketplace back to user instances. CLAUDE.md +carries a one-paragraph summary; this doc is the reference. + +For the *content-authoring* side (cover photos, demo videos, doc links via +`marketplace-metadata.json`), see [`curated-marketplace-format.md`](curated-marketplace-format.md). + +## Marketplace repositories (ingestion) + +Admin-managed git repos cloned nightly to `${DATA_DIR}/marketplaces//` so +FastAPI can read their contents from disk. + +- Register via `/admin/marketplaces` (admin UI) or `POST /api/marketplaces`. +- Scheduler calls `POST /api/marketplaces/sync-all` (admin-only, authed via + `SCHEDULER_API_TOKEN`) at `daily 03:00` UTC. Routing through HTTP keeps the + app the sole writer to `system.duckdb` — the previous in-process call from the + scheduler container raced the app's long-lived DB handle and 500-ed on + `Could not set lock on file`. +- Manual re-sync from the UI ("Sync now") hits `POST /api/marketplaces/{id}/sync`. +- PATs for private repos persist to `${DATA_DIR}/state/.env_overlay` (chmod 600) + as `AGNES_MARKETPLACE__TOKEN`. DuckDB stores only the env-var name + (`token_env`), never the secret. +- Registry lives in DuckDB table `marketplace_registry`. +- After each successful sync, `src/marketplace.py` parses + `.claude-plugin/marketplace.json` from the cloned repo and caches the plugin + list in `marketplace_plugins` (keyed by `(marketplace_id, plugin_name)`). +- `src/marketplace.py` handles clone/fetch/reset with token redaction in any + surfaced error message. + +## Claude Code marketplace endpoint (re-serving) + +Agnes serves a single aggregated Claude Code marketplace over two channels, both +gated by PAT auth and filtered per caller: + +- `GET /marketplace.zip` — deterministic ZIP download with `ETag` / + `If-None-Match` (304 when content unchanged). Consumed by a client-side + SessionStart hook. +- `GET /marketplace.git/*` — git smart-HTTP (dulwich via a2wsgi). Registered in + Claude Code once, then Claude Code owns the clone/fetch cycle. + +**Auth:** ZIP uses `Authorization: Bearer `. Git uses HTTP Basic where the +password field carries the PAT (`https://x:@host/marketplace.git/`) — git +CLI does not speak Bearer. + +**Content:** filtered via `src.marketplace_filter.resolve_allowed_plugins` which +joins `resource_grants ↔ marketplace_plugins` (matching +`mp.marketplace_id || '/' || mp.name = rg.resource_id`) scoped to the caller's +`user_group_members`. Admin is treated as a regular group here — no god-mode +shortcut for the marketplace feed, so admins curate their own view by granting +plugins to the Admin group (or any group they belong to). + +On-disk layout in the served ZIP / git tree uses a slug-prefixed directory +(`plugins/-/`) so two marketplaces shipping a same-named plugin +don't overwrite each other's files. The synth marketplace.json's `name` field, +however, is the plugin's authoritative name from its own +`.claude-plugin/plugin.json` (with a fallback to the upstream marketplace.json +`name`) — Claude Code's `/plugin` UI resolves a loaded plugin back to its +catalog entry by `plugin.json` name, so the catalog entry's `name` must match. +Same-named plugins from two upstream marketplaces therefore collide in the +catalog by design; admin RBAC (which grants survive the filter) decides which +one wins, identical to how Claude Code behaves when a user adds two upstream +marketplaces with overlapping plugin names directly. `/marketplace/info` exposes +both `name` and `prefixed_name` so operators can disambiguate. + +**Cache:** content-addressed bare repos at `${DATA_DIR}/marketplaces/git-cache/` +keyed by sha256(filtered content). Two users with the same RBAC view share one +repo; content change → new repo next to the old one. No TTL / prune yet. + +## User registration inside Claude Code + +``` +# ZIP channel (typically via a SessionStart hook that unpacks into ./marketplace/) +curl -H "Authorization: Bearer $AGNES_PAT" https://agnes.example.com/marketplace.zip + +# Git channel — one-time registration. Two paths; pick the first that works. + +# (a) Direct registration — preferred when it works. +/plugin marketplace add https://x:$AGNES_PAT@agnes.example.com/marketplace.git/ + +# (b) Two-step fallback — required when (a) fails. Bun-compiled `claude` on +# macOS / Windows ignores the OS trust store and CA env vars on the +# marketplace HTTPS path, so direct add can fail with TLS errors against +# a private-CA Agnes instance even when system tools work fine. System +# `git` honors GIT_SSL_CAINFO + the OS trust store, so cloning manually +# and pointing Claude Code at the local clone sidesteps the Bun TLS path +# entirely. +git clone https://x:$AGNES_PAT@agnes.example.com/marketplace.git/ ~/agnes-marketplace +claude plugin marketplace add ~/agnes-marketplace +# Optional hardening: strip the PAT from the cloned repo's origin so it +# doesn't sit in plaintext at ~/agnes-marketplace/.git/config — re-clone via +# the dashboard's setup flow when the PAT rotates. +git -C ~/agnes-marketplace remote set-url origin https://agnes.example.com/marketplace.git/ +``` + +The dashboard-served setup payload (see `app/web/setup_instructions.py`) already +branches between (a) and (b) automatically based on platform when a private CA +is in play. The block above is the manual equivalent for users registering +outside that flow (e.g. operators bringing up a new instance, or analysts whose +first attempt failed and need to retry by hand). diff --git a/services/session_processors/verification.py b/services/session_processors/verification.py index 354e483..7d3a5b6 100644 --- a/services/session_processors/verification.py +++ b/services/session_processors/verification.py @@ -82,7 +82,7 @@ class VerificationProcessor: # Confidence is computed in code from (source_type, detection_type). # The LLM is not trusted to set its own credibility — see Q3 in - # docs/pd-ps-comments.md and the ADR. + # docs/archive/pd-ps-comments.md and the ADR. detection_type = v.get("detection_type") try: confidence_value = compute_confidence("user_verification", detection_type) diff --git a/services/verification_detector/schemas.py b/services/verification_detector/schemas.py index 2e51cf6..bcfb9ff 100644 --- a/services/verification_detector/schemas.py +++ b/services/verification_detector/schemas.py @@ -2,7 +2,7 @@ Confidence is intentionally NOT part of this schema. It is derived in code from (source_type, detection_type) via services.corporate_memory.confidence — the LLM -is not trusted to set its own credibility (see docs/pd-ps-comments.md Q3). +is not trusted to set its own credibility (see docs/archive/pd-ps-comments.md Q3). """ VERIFICATION_SCHEMA: dict = { diff --git a/src/orchestrator_security.py b/src/orchestrator_security.py index 66369ba..a715633 100644 --- a/src/orchestrator_security.py +++ b/src/orchestrator_security.py @@ -4,9 +4,7 @@ The orchestrator reads `_remote_attach` rows that connectors write into their `extract.duckdb`, then calls `INSTALL`, `LOAD`, and `ATTACH` based on those values. Treating the connector as adversarial (compromised image, supply-chain, malicious fork) means the orchestrator picks **what** can be installed and -**which** env vars can be referenced — not the connector. See -`docs/superpowers/plans/2026-04-27-issue-81-trust-boundary.md` for the full -threat model. +**which** env vars can be referenced — not the connector. """ from __future__ import annotations