62336bfd32
* fix(rbac): stack-gate analyst table access via data_packages exclusively
Previously analysts could see a table in ``agnes catalog`` /
``/api/sync/manifest`` either by:
1. being in a group with ``resource_grants(group, 'table', id)``, or
2. being in a group with ``resource_grants(group, 'data_package', …)``
for a package containing the table.
Path 1 leaked: admins who minted a per-table grant without ever
wrapping the table in a data_package still shipped the table to
analysts — directly contradicting the unified-stack mental model
("the stack is the unit of access"). User report:
"i když to admin nedal do data package tak to by default uživatelé
dostali to by se nemělo stát".
New policy: analyst visibility is strictly stack-gated. A table is
visible iff at least one data_package containing it is in the
analyst's stack (required ∪ subscribed). Admin god-mode and the three
internal data-source tables (agnes_sessions / _telemetry / _audit
with row-level RBAC) keep their existing carve-outs.
Touched surfaces:
* ``src/rbac.can_access_table`` + ``get_accessible_tables`` —
routed through ``StackResolver.stack(user, DATA_PACKAGE)`` +
``data_package_tables`` join instead of ``resource_grants(table)``.
* ``app/api/sync._build_direct_tables_section`` — always returns
``[]`` (key kept for older CLI destructuring); per-table grants
no longer manifest.
* Standardised 403 detail across ``/api/data/*``, ``/api/query``,
``/api/v2/sample``, ``/api/v2/scan``, ``/api/v2/schema``:
``Table 'X' is not in your stack. Ask an admin to add it to a
Data Package you have access to (Required or in your stack),
then run `agnes pull` to refresh.`` Single source of truth lives
in ``src.rbac.table_not_in_stack_message`` so the wording stays
consistent across CLI surfaces.
UX side: ``/catalog/t/<id>`` (table detail page) dropped the four
editorial sections (Sample questions, What's inside, Things to know,
Pairs well with) per user feedback — the page's job is now
"what is this table, where do I find it" (hero + parent packages).
Tests:
* ``tests/conftest.grant_table_via_package`` / ``revoke_table_via_package``
— shared helpers that wrap a table in an auto-named data_package +
grant the package required to a custom group. Replaces the legacy
per-test ``_grant_table_to_analyst`` table-grant pattern.
* All 17 previously-failing legacy tests (test_access_control,
test_journey_rbac, test_audit_gap_*, test_rbac, …) migrated to use
the new helper; logic stays the same.
* ``tests/fixtures/analyst_bootstrap._grant_table_access`` updated
to wrap via data_package so the ``test_pat`` fixture's "two table
grants" semantics still ship parquets through ``agnes init``.
* New ``tests/test_table_not_in_stack_message.py`` locks in the
standardised 403 detail across the data + check-access endpoints.
5204 tests passing (added 1).
* fix(catalog): first-demo UX feedback — required-first grouping + longer card description
Two minor polish items from the 2026-05-19 stakeholder demo:
1. Required packages cluster at the top of the Browse grid instead of
being interleaved by ``created_at``. Sort key
``(requirement != 'required', name)`` runs before the adapter
call in both /catalog (data_packages) and /corporate-memory
(memory_domains) so the required block is visible without
scrolling. Regression test pins the order via
``data-id="…"`` position in rendered HTML.
2. ``.stack-card__desc`` line clamp bumped 2 → 4 lines. Two-line clamp
trailed almost every admin-authored description off in "…" before
the second clause, forcing a click-through to read it. The detail
page (/catalog/p/<slug>) keeps the unclamped body for longer
content.
* release: 0.55.3 — stack-gated analyst RBAC (BREAKING) + first-demo UX polish + #345 A/B/C/D + #347 UI consistency
228 lines
8.1 KiB
Python
228 lines
8.1 KiB
Python
"""E2E API tests — full server-side flow via FastAPI TestClient."""
|
|
|
|
import os
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import duckdb
|
|
import pytest
|
|
|
|
from tests.conftest import create_mock_extract
|
|
|
|
|
|
def _auth(token):
|
|
return {"Authorization": f"Bearer {token}"}
|
|
|
|
|
|
class TestFullSyncFlow:
|
|
"""Complete flow: register -> extract -> manifest -> download."""
|
|
|
|
def test_register_tables_and_get_catalog(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["admin_token"]
|
|
env = seeded_app["env"]
|
|
|
|
# Register tables
|
|
resp = c.post("/api/admin/register-table", json={
|
|
"name": "orders", "source_type": "keboola", "bucket": "in.c-crm",
|
|
"source_table": "orders", "query_mode": "local",
|
|
}, headers=_auth(t))
|
|
assert resp.status_code == 201
|
|
|
|
resp = c.post("/api/admin/register-table", json={
|
|
"name": "customers", "source_type": "keboola", "bucket": "in.c-crm",
|
|
"source_table": "customers", "query_mode": "local",
|
|
}, headers=_auth(t))
|
|
assert resp.status_code == 201
|
|
|
|
# Verify catalog
|
|
resp = c.get("/api/catalog/tables", headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
tables = resp.json()["tables"]
|
|
names = {tbl["name"] for tbl in tables}
|
|
assert "orders" in names
|
|
assert "customers" in names
|
|
|
|
def test_manifest_after_extract(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["admin_token"]
|
|
env = seeded_app["env"]
|
|
|
|
# Create mock extract with real data
|
|
create_mock_extract(env["extracts_dir"], "keboola", [
|
|
{"name": "orders", "data": [
|
|
{"id": "1", "product": "Widget", "price": "99.99"},
|
|
{"id": "2", "product": "Gadget", "price": "49.99"},
|
|
]},
|
|
{"name": "customers", "data": [
|
|
{"id": "1", "name": "Alice", "email": "alice@test.com"},
|
|
]},
|
|
])
|
|
|
|
# Run orchestrator to populate sync_state
|
|
from src.orchestrator import SyncOrchestrator
|
|
SyncOrchestrator().rebuild()
|
|
|
|
# Check manifest
|
|
resp = c.get("/api/sync/manifest", headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
manifest = resp.json()
|
|
assert "orders" in manifest["tables"]
|
|
assert "customers" in manifest["tables"]
|
|
assert manifest["tables"]["orders"]["rows"] == 2
|
|
assert manifest["tables"]["customers"]["rows"] == 1
|
|
assert "server_time" in manifest
|
|
|
|
def test_download_parquet_and_verify_content(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["admin_token"]
|
|
env = seeded_app["env"]
|
|
|
|
# Create extract
|
|
create_mock_extract(env["extracts_dir"], "keboola", [
|
|
{"name": "orders", "data": [
|
|
{"id": "1", "product": "Widget", "price": "99.99"},
|
|
{"id": "2", "product": "Gadget", "price": "49.99"},
|
|
]},
|
|
])
|
|
|
|
# Download parquet
|
|
resp = c.get("/api/data/orders/download", headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
assert "application/octet-stream" in resp.headers.get("content-type", "")
|
|
|
|
# Verify content by writing to temp file and reading with DuckDB
|
|
with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as f:
|
|
f.write(resp.content)
|
|
tmp_path = f.name
|
|
|
|
try:
|
|
conn = duckdb.connect()
|
|
rows = conn.execute(f"SELECT * FROM read_parquet('{tmp_path}') ORDER BY id").fetchall()
|
|
conn.close()
|
|
assert len(rows) == 2
|
|
assert rows[0][1] == "Widget" # product column
|
|
assert rows[1][1] == "Gadget"
|
|
finally:
|
|
os.unlink(tmp_path)
|
|
|
|
def test_download_nonexistent_table_404(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["admin_token"]
|
|
resp = c.get("/api/data/nonexistent/download", headers=_auth(t))
|
|
assert resp.status_code == 404
|
|
|
|
|
|
class TestRBACEnforcement:
|
|
"""Verify role-based access control across API endpoints."""
|
|
|
|
def test_analyst_cannot_register_table(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["analyst_token"]
|
|
resp = c.post("/api/admin/register-table", json={
|
|
"name": "test", "source_type": "keboola",
|
|
}, headers=_auth(t))
|
|
assert resp.status_code == 403
|
|
|
|
def test_analyst_can_read_manifest(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["analyst_token"]
|
|
resp = c.get("/api/sync/manifest", headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
|
|
def test_analyst_can_download_data_after_grant(self, seeded_app):
|
|
"""v19+: no implicit `is_public`. Analyst gets access via an explicit
|
|
resource_grants(group, "table", id) row."""
|
|
c = seeded_app["client"]
|
|
env = seeded_app["env"]
|
|
create_mock_extract(env["extracts_dir"], "keboola", [
|
|
{"name": "orders", "data": [{"id": "1"}]},
|
|
])
|
|
admin_t = seeded_app["admin_token"]
|
|
c.post("/api/admin/register-table", json={
|
|
"name": "orders", "source_type": "keboola", "bucket": "in.c-crm",
|
|
"source_table": "orders", "query_mode": "local",
|
|
}, headers=_auth(admin_t))
|
|
|
|
# Stack-gated RBAC: wrap the table in an auto data_package + grant
|
|
# the package to a custom group analyst1 belongs to.
|
|
from src.db import get_system_db
|
|
from tests.conftest import grant_table_via_package
|
|
conn = get_system_db()
|
|
try:
|
|
grant_table_via_package(
|
|
conn, "orders", "analyst1", group_name="e2e-analyst",
|
|
)
|
|
finally:
|
|
conn.close()
|
|
|
|
t = seeded_app["analyst_token"]
|
|
resp = c.get("/api/data/orders/download", headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
|
|
def test_admin_can_trigger_sync(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["admin_token"]
|
|
resp = c.post("/api/sync/trigger", headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
|
|
def test_analyst_cannot_trigger_sync(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["analyst_token"]
|
|
resp = c.post("/api/sync/trigger", headers=_auth(t))
|
|
assert resp.status_code == 403
|
|
|
|
def test_unauthenticated_blocked(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
resp = c.get("/api/sync/manifest")
|
|
assert resp.status_code == 401
|
|
|
|
|
|
class TestTableLifecycle:
|
|
"""Register -> update -> delete table via admin API."""
|
|
|
|
def test_full_lifecycle(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
t = seeded_app["admin_token"]
|
|
|
|
# Create
|
|
resp = c.post("/api/admin/register-table", json={
|
|
"name": "lifecycle_test", "source_type": "keboola",
|
|
"query_mode": "local", "description": "Test table",
|
|
}, headers=_auth(t))
|
|
assert resp.status_code == 201
|
|
table_id = resp.json()["id"]
|
|
|
|
# Read
|
|
resp = c.get("/api/admin/registry", headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
names = {tbl["name"] for tbl in resp.json()["tables"]}
|
|
assert "lifecycle_test" in names
|
|
|
|
# Update
|
|
resp = c.put(f"/api/admin/registry/{table_id}", json={
|
|
"query_mode": "remote",
|
|
}, headers=_auth(t))
|
|
assert resp.status_code == 200
|
|
|
|
# Verify update
|
|
resp = c.get("/api/admin/registry", headers=_auth(t))
|
|
table = next(tbl for tbl in resp.json()["tables"] if tbl["id"] == table_id)
|
|
assert table["query_mode"] == "remote"
|
|
|
|
# Delete
|
|
resp = c.delete(f"/api/admin/registry/{table_id}", headers=_auth(t))
|
|
assert resp.status_code == 204
|
|
|
|
# Verify gone
|
|
resp = c.get("/api/admin/registry", headers=_auth(t))
|
|
ids = {tbl["id"] for tbl in resp.json()["tables"]}
|
|
assert table_id not in ids
|
|
|
|
|
|
class TestSyncSubprocess:
|
|
def test_sync_trigger_returns_200(self, seeded_app):
|
|
c = seeded_app["client"]
|
|
resp = c.post("/api/sync/trigger", headers=_auth(seeded_app["admin_token"]))
|
|
assert resp.status_code == 200
|
|
assert resp.json()["status"] == "triggered"
|