From 37ad39c8a31d7d0b26c9d632c1357345bb423678 Mon Sep 17 00:00:00 2001 From: Vojtech <119944107+cvrysanek@users.noreply.github.com> Date: Thu, 14 May 2026 13:28:47 +0400 Subject: [PATCH] feat(home): status frame on /home (operator-gated, onboarded-only) (#297) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(home): status frame on /home — last sync, sessions, prompts, tokens, projects Adds the homepage status frame: a 5-card row above the install-hero / offboard-strip on /home showing the calling user's Last sync (their last `agnes pull`), Sessions, Prompts, Tokens used, and Projects worked on, with a 24h/7d pill toggle. Backed by `GET /api/me/home-stats?window=` (one DuckDB CTE joining `users` + `usage_session_summary` + `usage_events`) and SSR'd from the same `compute_home_stats` helper on initial paint so there's no spinner. The window toggle is the only JS-driven path. Side surfaces: - `GET /api/sync/manifest` now stamps `users.last_pull_at` so `agnes pull` (and the Claude Code SessionStart hook that wraps it) imprints the analyst's last sync time for the new card. - `usage_session_summary` gains four BIGINT token counters (input_tokens, output_tokens, cache_read_tokens, cache_creation_tokens) summed from JSONL `message.usage.*` per assistant turn. - `USAGE_PROCESSOR_VERSION` bumps 1 → 2 so the session-pipeline reprocess loop invalidates stale summaries and backfills tokens on the next tick. Schema migration v43 → v44 is idempotent ALTERs (last_pull_at + 4 token columns) — fresh installs receive them from `_SYSTEM_SCHEMA`, upgrade path runs `_v43_to_v44`. Defaults (NULL / 0) backfill existing rows cleanly. 9 new tests in tests/test_home_stats.py cover the migration, endpoint shapes (24h/7d/unknown/empty/missing-user), and the manifest-side last_pull_at bump. * docs(CHANGELOG): homepage status frame entries under [Unreleased] The post-rebase release-cut now belongs to whichever PR lands next after main rolled to 0.54.9. This PR logs its bullets under [Unreleased] (Added: homepage status frame, per-user pull tracking, token counters; Changed: schema v43 → v44 migration) so they ride out with the next release-cut. * fix(tests): bump test_schema_v42_migration asserts to v44 CI failed because tests/test_schema_v42_migration.py hardcoded `assert SCHEMA_VERSION == 43` and `assert v == 43` after init. v44 (homepage stats frame backing columns) was introduced in the preceding feat commit; this aligns the existing v42-era migration tests with the new schema version. * feat(home): gate status frame on operator flag + user.onboarded Two gates on the homepage status frame: 1. **Operator master switch** — `get_home_status_frame_visibility()` in app/instance_config.py mirrors the existing `get_home_automode_visibility()` shape: env var `AGNES_HOME_SHOW_STATUS_FRAME` > yaml `instance.home.show_status_frame` > default `True`. Cautious-rollout instances can disable the frame without forking; the yaml example documents both knobs. 2. **Onboarded gate** — the template only renders the frame when the caller's `users.onboarded` is true. First-day users see a clean install-hero before all-zero stat cards; the frame appears automatically on the next render after `agnes init` POSTs `/api/me/onboarded`. Router skips the `compute_home_stats` DB read entirely when either gate is closed; `home_stats` arrives at the template as None in that branch and the `{% if %}` shortcuts the include. Why both gates: PostHog feature flags evaluated and rejected — this codebase uses PostHog for analytics capture only, not feature gating; adding a per-user feature_enabled() call on the /home critical path would couple the homepage render to a remote eval and still require an admin master switch. The onboarded gate is a UX coherence rule layered on top of the operator switch, not an A/B test signal. 3 new tests in test_home_stats.py cover the env-var resolution (falsey values + default-true). The yaml example gets a `home:` block documenting both `show_automode` (pre-existing flag, was undocumented in the example) and `show_status_frame`. --- CHANGELOG.md | 34 +++ app/api/me.py | 131 +++++++++ app/api/sync.py | 20 +- app/instance_config.py | 26 ++ app/web/router.py | 16 ++ app/web/templates/_home_stats.html | 205 ++++++++++++++ app/web/templates/home_not_onboarded.html | 11 + config/instance.yaml.example | 7 + services/session_processors/usage_lib.py | 31 ++- src/db.py | 56 +++- src/repositories/usage.py | 9 +- tests/test_db_schema_version.py | 13 +- tests/test_home_stats.py | 319 ++++++++++++++++++++++ tests/test_schema_v42_migration.py | 13 +- 14 files changed, 877 insertions(+), 14 deletions(-) create mode 100644 app/web/templates/_home_stats.html create mode 100644 tests/test_home_stats.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f00d80..c067806 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,41 @@ CalVer image tags (`stable-YYYY.MM.N`, `dev-YYYY.MM.N`) are produced for every C ## [Unreleased] +### Added + +- **Homepage status frame.** The `/home` page now opens with a 5-card + status row above the install-hero / offboard-strip: **Last sync** + (your last `agnes pull`), **Sessions**, **Prompts**, **Tokens used**, + **Projects worked on**. A pill toggle switches the window between + 24h (default) and 7d. Backed by `GET /api/me/home-stats?window=` which + joins `users`, `usage_session_summary`, and `usage_events` in a + single DuckDB round-trip; the initial paint is SSR'd from the same + helper (`app.api.me.compute_home_stats`) so there's no spinner. + Visibility is gated on (a) the operator flag + `instance.home.show_status_frame` (yaml) / + `AGNES_HOME_SHOW_STATUS_FRAME` (env), default `true`, AND (b) the + caller being `onboarded`. Cautious-rollout instances can hide the + frame entirely; on every install, first-day users still see a clean + install-hero before zero-value stats show up. +- **Per-user pull tracking.** `GET /api/sync/manifest` now stamps + `users.last_pull_at` as a side effect. `agnes pull` (and the + Claude Code `SessionStart` hook that wraps it) imprints the + analyst's "last sync" timestamp for the new homepage card. +- **Token counters on `usage_session_summary`.** Four new BIGINT + columns (`input_tokens`, `output_tokens`, `cache_read_tokens`, + `cache_creation_tokens`) summed from JSONL `message.usage.*` per + assistant turn. `USAGE_PROCESSOR_VERSION` bumps 1 → 2, which the + session-pipeline reprocess loop uses to invalidate stale summaries + and backfill tokens on the next tick. + ### Changed + +- Schema migration **v43 → v44** (`_v43_to_v44`): idempotent `ALTER + TABLE … ADD COLUMN IF NOT EXISTS` for `users.last_pull_at` plus the + four token columns above. Fresh installs receive them inline from + `_SYSTEM_SCHEMA`; upgrade path runs the function. All new columns + default to NULL / 0 so existing rows backfill cleanly without a + separate migration step. - **Marketplace cover photos served with aggressive browser caching.** `/api/marketplace/curated/.../asset/...`, `/api/marketplace/curated/.../mirrored/...`, and `/api/store/entities/{id}/photo` now respond with diff --git a/app/api/me.py b/app/api/me.py index 913890f..7f5c41d 100644 --- a/app/api/me.py +++ b/app/api/me.py @@ -57,3 +57,134 @@ async def post_onboarded( result="ok", ) return {"status": "ok", "onboarded": target} + + +# --------------------------------------------------------------------------- +# GET /api/me/home-stats — backing data for the /home status frame +# --------------------------------------------------------------------------- + + +_WINDOW_INTERVALS = { + "24h": "INTERVAL 24 HOUR", + "7d": "INTERVAL 7 DAY", +} + + +def _username_for_stats(user: dict) -> str: + """Map a users row to the filesystem username used by the session + collector and stored in ``usage_session_summary.username``. + + Mirrors ``app.api.admin_user_sessions._username_from_user``: the + session collector writes JSONL under the OS username of the agent + process which, for current deployments, equals the email local-part. + Kept inline here so this endpoint has no cross-module dependency on + an admin-only helper; if the mapping evolves both copies must update. + """ + email: str = user.get("email", "") or "" + return email.split("@")[0] if "@" in email else email + + +def compute_home_stats( + conn: duckdb.DuckDBPyConnection, user: dict, window: str = "24h" +) -> dict: + """Pure helper that returns the home-stats payload for the given user. + + Shared by the HTTP endpoint and the /home Jinja handler (server-side + initial render). Unknown windows clamp to ``24h`` so callers never + need to pre-validate. Returns a dict with ISO-stringified + ``last_pull_at`` (or None) so the same shape works for both JSON + serialization and Jinja rendering. + """ + interval = _WINDOW_INTERVALS.get(window) + if interval is None: + window = "24h" + interval = _WINDOW_INTERVALS["24h"] + + username = _username_for_stats(user) + + # f-string interpolates only the validated interval literal above; + # all user-controlled input flows through bound parameters. + sql = f""" + WITH win AS ( + SELECT current_timestamp - {interval} AS since + ), + sess AS ( + SELECT + COUNT(*) AS sessions, + COALESCE(SUM(user_messages), 0) AS prompts, + COALESCE(SUM(input_tokens), 0) AS input_tokens, + COALESCE(SUM(output_tokens), 0) AS output_tokens, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read, + COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation + FROM usage_session_summary, win + WHERE username = ? AND started_at >= win.since + ), + proj AS ( + SELECT COUNT(DISTINCT cwd) AS projects + FROM usage_events, win + WHERE username = ? + AND cwd IS NOT NULL + AND occurred_at >= win.since + ), + u AS ( + SELECT last_pull_at FROM users WHERE id = ? + ) + SELECT + u.last_pull_at, + sess.sessions, sess.prompts, + sess.input_tokens, sess.output_tokens, + sess.cache_read, sess.cache_creation, + proj.projects + FROM u, sess, proj + """ + row = conn.execute(sql, [username, username, user["id"]]).fetchone() + + if row is None: + return { + "window": window, + "last_pull_at": None, + "sessions": 0, + "prompts": 0, + "tokens": { + "input": 0, "output": 0, + "cache_read": 0, "cache_creation": 0, + "total": 0, + }, + "projects": 0, + } + + (last_pull_at, sessions, prompts, + input_t, output_t, cache_read, cache_creation, projects) = row + return { + "window": window, + "last_pull_at": last_pull_at.isoformat() if last_pull_at else None, + "sessions": int(sessions or 0), + "prompts": int(prompts or 0), + "tokens": { + "input": int(input_t or 0), + "output": int(output_t or 0), + "cache_read": int(cache_read or 0), + "cache_creation": int(cache_creation or 0), + "total": int((input_t or 0) + (output_t or 0) + + (cache_read or 0) + (cache_creation or 0)), + }, + "projects": int(projects or 0), + } + + +@router.get("/home-stats") +async def get_home_stats( + window: str = "24h", + user: dict = Depends(get_current_user), + conn: duckdb.DuckDBPyConnection = Depends(_get_db), +): + """Return the five counters rendered in the /home status frame for + the calling user, over a 24-hour or 7-day window. + + Single round-trip: one DuckDB query joins ``users``, + ``usage_session_summary``, and ``usage_events`` so the homepage + renders without N+1. Missing rows (new user, no telemetry yet) + surface as zeros / null rather than 404 — the frame still renders + cleanly for first-day analysts. + """ + return compute_home_stats(conn, user, window) diff --git a/app/api/sync.py b/app/api/sync.py index cb9c486..1b1cf94 100644 --- a/app/api/sync.py +++ b/app/api/sync.py @@ -798,7 +798,25 @@ async def sync_manifest( user: dict = Depends(get_current_user), conn: duckdb.DuckDBPyConnection = Depends(_get_db), ): - """Return hash-based manifest of all synced data, filtered per user.""" + """Return hash-based manifest of all synced data, filtered per user. + + Side-effect: stamps ``users.last_pull_at`` so the /home status frame + can show when the analyst last pulled. This GET is the canonical + "I am about to sync" signal — agnes pull hits it first, then + downloads parquets whose hash changed. UI bumps (manifest browsed in + a browser session) also count; cheap and accurate enough for a + homepage card. + """ + try: + conn.execute( + "UPDATE users SET last_pull_at = current_timestamp WHERE id = ?", + [user["id"]], + ) + except Exception: + # Never block a pull because the stamp UPDATE hit a transient + # issue (locked WAL, partial migration window). The manifest + # itself is the load-bearing payload. + pass return _build_manifest_for_user(conn, user) diff --git a/app/instance_config.py b/app/instance_config.py index 207523e..14d27da 100644 --- a/app/instance_config.py +++ b/app/instance_config.py @@ -250,6 +250,32 @@ def get_home_automode_visibility() -> bool: return str(raw).strip().lower() not in ("0", "false", "no", "off", "") +def get_home_status_frame_visibility() -> bool: + """Whether /home renders the homepage status frame (Last sync, + Sessions, Prompts, Tokens, Projects). + + The template ALSO gates rendering on ``users.onboarded`` so a + fresh user sees a clean install-hero before the all-zero stat + cards. This helper is the operator-level master switch; the + onboarding gate is a UX coherence rule layered on top. + + Cautious-rollout instances that would rather not expose token + counters to analysts yet can disable with + ``AGNES_HOME_SHOW_STATUS_FRAME=0`` (or + ``instance.home.show_status_frame: false`` in YAML). + + Resolution: env var > ``instance.home.show_status_frame`` YAML > True. + Shape mirrors :func:`get_home_automode_visibility` so Terraform + overrides land the same way. + """ + raw = os.environ.get("AGNES_HOME_SHOW_STATUS_FRAME") + if raw is None: + raw = get_value("instance", "home", "show_status_frame", default=True) + if isinstance(raw, bool): + return raw + return str(raw).strip().lower() not in ("0", "false", "no", "off", "") + + def get_instance_name() -> str: return get_value("instance", "name", default="AI Data Analyst") diff --git a/app/web/router.py b/app/web/router.py index b31a11d..fd089f2 100644 --- a/app/web/router.py +++ b/app/web/router.py @@ -693,6 +693,20 @@ async def home_page( news = NewsTemplateRepository(conn).get_current_published() news_intro = news["intro"] if (news and news.get("intro")) else "" + # Homepage status frame (Last sync, Sessions, Prompts, Tokens, Projects). + # Gated on (a) operator flag instance.home.show_status_frame / + # AGNES_HOME_SHOW_STATUS_FRAME (default on), AND (b) the user being + # onboarded — first-day users see a clean install-hero before zero-value + # stat cards. When either gate is closed we skip the DB read entirely. + from app.api.me import compute_home_stats + from app.instance_config import get_home_status_frame_visibility + status_frame_enabled = get_home_status_frame_visibility() + home_stats = ( + compute_home_stats(conn, user, "24h") + if (status_frame_enabled and onboarded) + else None + ) + # Single template renders both states. The post-onboarding view keeps # the install-steps + connector prompts + auto-mode card visible — # they stay relevant for adding a second machine, a missing connector, @@ -706,6 +720,8 @@ async def home_page( onboarded=onboarded, is_admin=is_user_admin(user["id"], conn), news_intro=news_intro, + home_stats=home_stats, + status_frame_enabled=status_frame_enabled, ) return templates.TemplateResponse(request, "home_not_onboarded.html", ctx) diff --git a/app/web/templates/_home_stats.html b/app/web/templates/_home_stats.html new file mode 100644 index 0000000..c42a962 --- /dev/null +++ b/app/web/templates/_home_stats.html @@ -0,0 +1,205 @@ +{# Homepage status frame — five counters with a 24h/7d window toggle. + + Initial paint is server-rendered from `home_stats` in ctx (no + spinner, no FOUC). The toggle JS calls GET /api/me/home-stats?window= + and rewrites the card values in place. #} + + + +
+
+
Status
+
+ + +
+
+ +
+
+
Last sync
+
+ {% if home_stats.last_pull_at %}—{% else %}never{% endif %} +
+
your last agnes pull
+
+
+
Sessions
+
{{ home_stats.sessions }}
+
+ last {{ home_stats.window }} +
+
+
+
Prompts
+
{{ home_stats.prompts }}
+
user messages
+
+
+
Tokens
+
{{ home_stats.tokens.total }}
+
in + out + cache
+
+
+
Projects
+
{{ home_stats.projects }}
+
distinct cwd
+
+
+
+ + diff --git a/app/web/templates/home_not_onboarded.html b/app/web/templates/home_not_onboarded.html index 826a263..8009f54 100644 --- a/app/web/templates/home_not_onboarded.html +++ b/app/web/templates/home_not_onboarded.html @@ -1392,6 +1392,17 @@
+ {# Homepage status frame — five counters with 24h/7d toggle. + Two gates: (a) operator flag instance.home.show_status_frame / + AGNES_HOME_SHOW_STATUS_FRAME (default on, evaluated in router and + passed as `status_frame_enabled`); (b) the user being onboarded — + fresh users see a clean install-hero before zero-value stats. + Router skips `compute_home_stats` (saves the DB hit) when either + gate is closed, so `home_stats` is None in that branch. #} + {% if status_frame_enabled and onboarded and home_stats %} + {% include "_home_stats.html" %} + {% endif %} + {% set display_name = (user.name or (user.email or "").split("@")[0] or "there") %} {# Install-hero renders only for not-onboarded users. Once `agnes init` diff --git a/config/instance.yaml.example b/config/instance.yaml.example index 6d59f43..bf5a35b 100644 --- a/config/instance.yaml.example +++ b/config/instance.yaml.example @@ -50,6 +50,13 @@ instance: # hasn't pre-provisioned a shared OAuth app can request # one without leaving the workspace). Empty/unset hides # the button. Env override: AGNES_INSTANCE_ADMIN_EMAIL. + # home: # Per-instance toggles for /home content blocks. + # show_automode: true # Render Step-3 auto-accept-mode block. Default true. + # # Env: AGNES_HOME_SHOW_AUTOMODE. + # show_status_frame: true # Render the 5-card status frame (Last sync, Sessions, + # # Prompts, Tokens, Projects). Visible only to onboarded + # # users regardless of this flag. Default true. Env: + # # AGNES_HOME_SHOW_STATUS_FRAME. # --- Server --- server: diff --git a/services/session_processors/usage_lib.py b/services/session_processors/usage_lib.py index cfaf245..7a3cbe0 100644 --- a/services/session_processors/usage_lib.py +++ b/services/session_processors/usage_lib.py @@ -40,7 +40,7 @@ from dataclasses import dataclass from datetime import datetime, timezone, timedelta from typing import Iterator -USAGE_PROCESSOR_VERSION = 1 +USAGE_PROCESSOR_VERSION = 2 BUILTIN_TOOLS = frozenset({ "Bash", "Read", "Edit", "Write", "Grep", "Glob", "TodoWrite", @@ -323,6 +323,10 @@ def compute_summary(turns: list[dict], events: list[dict]) -> dict: user_messages = 0 assistant_messages = 0 model_counter: Counter = Counter() + input_tokens = 0 + output_tokens = 0 + cache_read_tokens = 0 + cache_creation_tokens = 0 for t in turns: ts = _parse_ts(t.get("timestamp")) @@ -337,6 +341,27 @@ def compute_summary(turns: list[dict], events: list[dict]) -> dict: m = msg.get("model") if m: model_counter[m] += 1 + # Anthropic API usage block on assistant turns. Older sessions + # may lack `cache_*` keys (pre-prompt-caching) — `.get(k, 0)` + # tolerates that. Non-int values (corrupted JSONL) are skipped + # to keep one bad turn from poisoning the whole summary. + usage = msg.get("usage") or {} + for key, accum in ( + ("input_tokens", "input_tokens"), + ("output_tokens", "output_tokens"), + ("cache_read_input_tokens", "cache_read_tokens"), + ("cache_creation_input_tokens", "cache_creation_tokens"), + ): + v = usage.get(key, 0) + if isinstance(v, int): + if accum == "input_tokens": + input_tokens += v + elif accum == "output_tokens": + output_tokens += v + elif accum == "cache_read_tokens": + cache_read_tokens += v + elif accum == "cache_creation_tokens": + cache_creation_tokens += v started_at = min(timestamps) if timestamps else None ended_at = max(timestamps) if timestamps else None @@ -373,6 +398,10 @@ def compute_summary(turns: list[dict], events: list[dict]) -> dict: "distinct_tools": distinct_tools, "distinct_skills": distinct_skills, "primary_model": primary_model, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cache_read_tokens": cache_read_tokens, + "cache_creation_tokens": cache_creation_tokens, "processor_version": USAGE_PROCESSOR_VERSION, } diff --git a/src/db.py b/src/db.py index c1d0545..d11aff8 100644 --- a/src/db.py +++ b/src/db.py @@ -40,7 +40,7 @@ def _maybe_instrument(con, db_tag: str): _SAFE_IDENTIFIER = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]{0,63}$") -SCHEMA_VERSION = 43 +SCHEMA_VERSION = 44 _SYSTEM_SCHEMA = """ CREATE TABLE IF NOT EXISTS schema_version ( @@ -70,7 +70,12 @@ CREATE TABLE IF NOT EXISTS users ( -- self-mark "I've already set up Agnes locally" button on /home. -- Default FALSE; explicit signal required to flip (no PAT-heuristic -- auto-flip per the brainstorm decision §D). - onboarded BOOLEAN NOT NULL DEFAULT FALSE + onboarded BOOLEAN NOT NULL DEFAULT FALSE, + -- v44: per-user pull timestamp. Bumped on every GET /api/sync/manifest + -- so `agnes pull` (and the SessionStart hook that wraps it) imprints + -- the user's last sync time. Powers the /home status frame's "Last + -- sync" card. + last_pull_at TIMESTAMP ); CREATE TABLE IF NOT EXISTS sync_state ( @@ -746,7 +751,16 @@ CREATE TABLE IF NOT EXISTS usage_session_summary ( distinct_skills INTEGER DEFAULT 0, primary_model VARCHAR, processor_version INTEGER NOT NULL, - extracted_at TIMESTAMP DEFAULT current_timestamp + extracted_at TIMESTAMP DEFAULT current_timestamp, + -- v44: per-session token counters summed from JSONL message.usage.*. + -- BIGINT because cache tokens routinely exceed INT range over long + -- sessions. Default 0 so existing rows backfill cleanly; the + -- processor's reprocess loop (driven by USAGE_PROCESSOR_VERSION + -- bump) overwrites with real values on next tick. + input_tokens BIGINT DEFAULT 0, + output_tokens BIGINT DEFAULT 0, + cache_read_tokens BIGINT DEFAULT 0, + cache_creation_tokens BIGINT DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_usage_session_user ON usage_session_summary(username); CREATE INDEX IF NOT EXISTS idx_usage_session_started ON usage_session_summary(started_at); @@ -2886,6 +2900,36 @@ def _v42_to_v43(conn: duckdb.DuckDBPyConnection) -> None: ) +def _v43_to_v44(conn: duckdb.DuckDBPyConnection) -> None: + """v44: homepage status frame backing columns. + + Adds ``users.last_pull_at`` (per-user manifest fetch timestamp) and + four BIGINT token counters on ``usage_session_summary`` + (``input_tokens``, ``output_tokens``, ``cache_read_tokens``, + ``cache_creation_tokens``). All idempotent ALTERs — fresh installs + receive the columns from ``_SYSTEM_SCHEMA`` and this is a no-op for + them; upgrade path picks them up. + + Token columns default to 0; existing summary rows backfill on the + next UsageProcessor tick because ``USAGE_PROCESSOR_VERSION`` bumps + from 1 → 2 in the same release, which the session-pipeline + reprocess loop uses to invalidate stale summaries. + """ + conn.execute( + "ALTER TABLE users ADD COLUMN IF NOT EXISTS last_pull_at TIMESTAMP" + ) + for col in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_creation_tokens", + ): + conn.execute( + f"ALTER TABLE usage_session_summary " + f"ADD COLUMN IF NOT EXISTS {col} BIGINT DEFAULT 0" + ) + + _V33_TO_V34_MIGRATIONS = [ # DuckDB blocks DROP COLUMN while indexes reference the table # ("Dependency Error: Cannot alter entry … because there are entries @@ -3159,6 +3203,10 @@ def _ensure_schema(conn: duckdb.DuckDBPyConnection) -> None: _v41_to_v42(conn) # v43 user_observability_views — saved-views for /admin/activity. _v42_to_v43(conn) + # v44 homepage-stats columns. _SYSTEM_SCHEMA already declares + # them on fresh installs (no-op ALTERs); kept here for the + # ladder's chronological readability. + _v43_to_v44(conn) # Fresh-install seed is handled by the unconditional # _seed_core_roles call at the bottom of _ensure_schema — # left as a no-op branch here so the migration ladder still @@ -3300,6 +3348,8 @@ def _ensure_schema(conn: duckdb.DuckDBPyConnection) -> None: _v41_to_v42(conn) if current < 43: _v42_to_v43(conn) + if current < 44: + _v43_to_v44(conn) conn.execute( "UPDATE schema_version SET version = ?, applied_at = current_timestamp", [SCHEMA_VERSION], diff --git a/src/repositories/usage.py b/src/repositories/usage.py index 165e8ac..d5fc4bf 100644 --- a/src/repositories/usage.py +++ b/src/repositories/usage.py @@ -36,8 +36,9 @@ class UsageRepository: active_seconds, wall_seconds, user_messages, assistant_messages, tool_calls, tool_errors, skill_invocations, subagent_dispatches, mcp_calls, slash_commands, distinct_tools, distinct_skills, - primary_model, processor_version) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + primary_model, input_tokens, output_tokens, cache_read_tokens, + cache_creation_tokens, processor_version) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ summary["session_file"], @@ -58,6 +59,10 @@ class UsageRepository: summary.get("distinct_tools", 0), summary.get("distinct_skills", 0), summary.get("primary_model"), + summary.get("input_tokens", 0), + summary.get("output_tokens", 0), + summary.get("cache_read_tokens", 0), + summary.get("cache_creation_tokens", 0), processor_version, ], ) diff --git a/tests/test_db_schema_version.py b/tests/test_db_schema_version.py index ac24fa4..4592f89 100644 --- a/tests/test_db_schema_version.py +++ b/tests/test_db_schema_version.py @@ -13,7 +13,7 @@ import duckdb from src.db import SCHEMA_VERSION, _ensure_schema, get_schema_version -def test_schema_version_is_43(): +def test_schema_version_is_44(): # v27 → v28: explicit-install (Model B) for curated marketplace plugins. # user_plugin_optouts row presence flips meaning from "excluded" to # "subscribed"; migration wipes existing rows so the inverted reading @@ -87,11 +87,18 @@ def test_schema_version_is_43(): # (per-session aggregate), usage_tool_daily + usage_plugin_daily # (daily rollups), usage_attribution_skills/agents/commands # (plugin manifest attribution). 10 indices for fast queries. - # v42 → v43 (this PR): user_observability_views — per-user saved + # v42 → v43: user_observability_views — per-user saved # filter combinations backing the unified /admin/activity # page (UNIQUE(user_id, name)). Schema is intentionally # opaque JSON because the UI evolves faster than DB. - assert SCHEMA_VERSION == 43 + # v43 → v44 (this PR): homepage status frame backing columns — + # users.last_pull_at (per-user manifest fetch timestamp, + # bumped by GET /api/sync/manifest) plus four BIGINT token + # counters on usage_session_summary (input_tokens, + # output_tokens, cache_read_tokens, cache_creation_tokens). + # USAGE_PROCESSOR_VERSION simultaneously bumps 1→2 so the + # reprocess loop backfills tokens on next tick. + assert SCHEMA_VERSION == 44 def test_v37_marketplace_curator_columns(tmp_path): diff --git a/tests/test_home_stats.py b/tests/test_home_stats.py new file mode 100644 index 0000000..b1ea1b8 --- /dev/null +++ b/tests/test_home_stats.py @@ -0,0 +1,319 @@ +"""Homepage status frame — schema v44, endpoint shapes, manifest stamp, +operator visibility flag. + +Covers: +- v44 ALTERs land users.last_pull_at + 4 token columns idempotently on + both fresh installs and upgrades from a v43-shaped DB. +- compute_home_stats returns the right counters for 24h vs 7d windows + and clamps unknown windows to 24h. +- GET /api/sync/manifest bumps users.last_pull_at as a side effect. +- get_home_status_frame_visibility honors the env var + yaml override + and defaults true. +""" +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone + +import duckdb +import pytest + +from src.db import ( + SCHEMA_VERSION, + _SYSTEM_SCHEMA, + _ensure_schema, + _v43_to_v44, +) + + +# --------------------------------------------------------------------------- +# Schema v44 +# --------------------------------------------------------------------------- + + +def test_v44_fresh_install_has_token_columns_and_last_pull(tmp_path): + """Fresh install reaches v44 with all new columns declared in + _SYSTEM_SCHEMA (the migration function is a no-op on fresh install + but the columns must exist regardless).""" + db_path = tmp_path / "system.duckdb" + conn = duckdb.connect(str(db_path)) + _ensure_schema(conn) + + user_cols = {r[1] for r in conn.execute("PRAGMA table_info(users)").fetchall()} + assert "last_pull_at" in user_cols + + sess_cols = { + r[1] + for r in conn.execute("PRAGMA table_info(usage_session_summary)").fetchall() + } + for col in ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_creation_tokens", + ): + assert col in sess_cols, f"missing {col}" + + +def test_v43_to_v44_upgrade_is_idempotent(tmp_path): + """Running _v43_to_v44 on a hand-rolled pre-v44 DB lands the four + new columns; a second call is a no-op (IF NOT EXISTS guards).""" + db_path = tmp_path / "system.duckdb" + conn = duckdb.connect(str(db_path)) + # Hand-roll v43-shaped tables (no last_pull_at, no token cols). + conn.execute( + "CREATE TABLE users (id VARCHAR PRIMARY KEY, email VARCHAR, " + "onboarded BOOLEAN DEFAULT FALSE)" + ) + conn.execute( + """ + CREATE TABLE usage_session_summary ( + session_file VARCHAR PRIMARY KEY, + session_id VARCHAR NOT NULL, + username VARCHAR NOT NULL, + started_at TIMESTAMP, + ended_at TIMESTAMP, + user_messages INTEGER DEFAULT 0, + processor_version INTEGER NOT NULL + ) + """ + ) + + _v43_to_v44(conn) + _v43_to_v44(conn) # idempotent + + assert "last_pull_at" in { + r[1] for r in conn.execute("PRAGMA table_info(users)").fetchall() + } + tok_cols = { + r[1] + for r in conn.execute("PRAGMA table_info(usage_session_summary)").fetchall() + if "token" in r[1] + } + assert tok_cols == { + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_creation_tokens", + } + + +def test_schema_version_constant_is_44(): + """Belt + suspenders against schema_version regressions.""" + assert SCHEMA_VERSION == 44 + + +# --------------------------------------------------------------------------- +# compute_home_stats / GET /api/me/home-stats +# --------------------------------------------------------------------------- + + +@pytest.fixture +def stats_conn(tmp_path): + db_path = tmp_path / "system.duckdb" + conn = duckdb.connect(str(db_path)) + conn.execute(_SYSTEM_SCHEMA) + return conn + + +def _seed_user(conn, *, uid="u1", email="alice@example.com"): + conn.execute( + "INSERT INTO users (id, email, active, onboarded, last_pull_at) " + "VALUES (?, ?, TRUE, TRUE, current_timestamp)", + [uid, email], + ) + + +def _seed_session(conn, *, session_file, username, started_sql, prompts=0, + input_tokens=0, output_tokens=0, + cache_read=0, cache_creation=0): + conn.execute( + f""" + INSERT INTO usage_session_summary + (session_file, session_id, username, started_at, ended_at, + user_messages, input_tokens, output_tokens, cache_read_tokens, + cache_creation_tokens, processor_version) + VALUES (?, ?, ?, {started_sql}, current_timestamp, + ?, ?, ?, ?, ?, 2) + """, + [session_file, session_file, username, prompts, + input_tokens, output_tokens, cache_read, cache_creation], + ) + + +def _seed_event(conn, *, ev_id, session_file, username, cwd, occurred_sql): + conn.execute( + f""" + INSERT INTO usage_events + (id, session_id, session_file, username, event_type, source, + cwd, occurred_at, processor_version) + VALUES (?, ?, ?, ?, 'tool_use', 'builtin', ?, {occurred_sql}, 2) + """, + [ev_id, session_file, session_file, username, cwd], + ) + + +def test_compute_home_stats_24h_vs_7d_windowing(stats_conn): + """A session 1h ago shows in both windows; a session 3 days ago + only in 7d; a session 30 days ago in neither.""" + from app.api.me import compute_home_stats + + _seed_user(stats_conn) + _seed_session(stats_conn, session_file="a.jsonl", username="alice", + started_sql="current_timestamp - INTERVAL 1 HOUR", + prompts=5, input_tokens=100, output_tokens=50, + cache_read=800, cache_creation=25) + _seed_session(stats_conn, session_file="b.jsonl", username="alice", + started_sql="current_timestamp - INTERVAL 3 DAY", + prompts=5, input_tokens=100, output_tokens=50, + cache_read=800, cache_creation=25) + _seed_session(stats_conn, session_file="c.jsonl", username="alice", + started_sql="current_timestamp - INTERVAL 30 DAY", + prompts=99) + + _seed_event(stats_conn, ev_id="e1", session_file="a.jsonl", + username="alice", cwd="/proj/alpha", + occurred_sql="current_timestamp - INTERVAL 1 HOUR") + _seed_event(stats_conn, ev_id="e2", session_file="a.jsonl", + username="alice", cwd="/proj/beta", + occurred_sql="current_timestamp - INTERVAL 2 HOUR") + _seed_event(stats_conn, ev_id="e3", session_file="b.jsonl", + username="alice", cwd="/proj/gamma", + occurred_sql="current_timestamp - INTERVAL 3 DAY") + + user = {"id": "u1", "email": "alice@example.com"} + + s24 = compute_home_stats(stats_conn, user, "24h") + assert s24["window"] == "24h" + assert s24["sessions"] == 1 + assert s24["prompts"] == 5 + assert s24["projects"] == 2 + assert s24["tokens"]["total"] == 100 + 50 + 800 + 25 + assert s24["last_pull_at"] is not None + + s7 = compute_home_stats(stats_conn, user, "7d") + assert s7["window"] == "7d" + assert s7["sessions"] == 2 + assert s7["prompts"] == 10 + assert s7["projects"] == 3 + assert s7["tokens"]["total"] == 2 * (100 + 50 + 800 + 25) + + +def test_compute_home_stats_unknown_window_clamps_to_24h(stats_conn): + """Out-of-band window values clamp to 24h rather than 400-ing.""" + from app.api.me import compute_home_stats + + _seed_user(stats_conn) + s = compute_home_stats(stats_conn, {"id": "u1", "email": "alice@example.com"}, "bogus") + assert s["window"] == "24h" + + +def test_compute_home_stats_empty_user_returns_zeros(stats_conn): + """Brand-new user with no sessions / events surfaces zeros, not 500.""" + from app.api.me import compute_home_stats + + _seed_user(stats_conn, uid="u_empty", email="nobody@example.com") + s = compute_home_stats( + stats_conn, + {"id": "u_empty", "email": "nobody@example.com"}, + "24h", + ) + assert s["sessions"] == 0 + assert s["prompts"] == 0 + assert s["projects"] == 0 + assert s["tokens"]["total"] == 0 + # Seeded user row carries a last_pull_at from the helper, so this + # asserts the column travels through the join correctly. + assert s["last_pull_at"] is not None + + +def test_compute_home_stats_missing_users_row_returns_zeros(stats_conn): + """If the users row is missing entirely (race during deletion), the + helper returns a zeroed payload instead of crashing.""" + from app.api.me import compute_home_stats + + s = compute_home_stats( + stats_conn, + {"id": "ghost", "email": "ghost@example.com"}, + "24h", + ) + assert s == { + "window": "24h", + "last_pull_at": None, + "sessions": 0, + "prompts": 0, + "tokens": { + "input": 0, "output": 0, + "cache_read": 0, "cache_creation": 0, + "total": 0, + }, + "projects": 0, + } + + +# --------------------------------------------------------------------------- +# GET /api/sync/manifest bumps users.last_pull_at +# --------------------------------------------------------------------------- + + +def test_sync_manifest_bumps_last_pull_at(stats_conn, monkeypatch, tmp_path): + """The manifest endpoint records the user's pull timestamp so the + /home status frame's 'Last sync' card stays current.""" + from app.api.sync import sync_manifest + + # data_dir for asset hashing; we don't seed docs/profiles so the + # assets dict will be empty (manifest still returns ok). + monkeypatch.setenv("DATA_DIR", str(tmp_path)) + + _seed_user(stats_conn, uid="u_pull", email="puller@example.com") + # Wipe seeded last_pull_at so we can detect the bump. + stats_conn.execute( + "UPDATE users SET last_pull_at = NULL WHERE id = ?", ["u_pull"] + ) + + asyncio.run( + sync_manifest( + user={"id": "u_pull", "email": "puller@example.com"}, + conn=stats_conn, + ) + ) + row = stats_conn.execute( + "SELECT last_pull_at FROM users WHERE id = ?", ["u_pull"] + ).fetchone() + # Don't compare against `datetime.now(utc)` — DuckDB's + # ``current_timestamp`` returns the session's wall-clock time which + # may be naive-local-or-utc depending on the environment, so a + # delta-based assertion would tz-skew. The semantic the test cares + # about is "the column flipped from NULL", which is what the home + # status card reads. + assert row[0] is not None + + +# --------------------------------------------------------------------------- +# Operator visibility flag — get_home_status_frame_visibility +# --------------------------------------------------------------------------- + + +def test_status_frame_default_is_visible(monkeypatch): + """Absent both env var and yaml entry, the flag returns True.""" + monkeypatch.delenv("AGNES_HOME_SHOW_STATUS_FRAME", raising=False) + from app.instance_config import get_home_status_frame_visibility + assert get_home_status_frame_visibility() is True + + +def test_status_frame_env_var_off(monkeypatch): + """AGNES_HOME_SHOW_STATUS_FRAME=0 hides the frame.""" + monkeypatch.setenv("AGNES_HOME_SHOW_STATUS_FRAME", "0") + from app.instance_config import get_home_status_frame_visibility + assert get_home_status_frame_visibility() is False + + +def test_status_frame_env_var_falsey_values(monkeypatch): + """Each of {0, false, no, off, ''} hides the frame; anything else shows.""" + from app.instance_config import get_home_status_frame_visibility + for val in ("0", "false", "False", "FALSE", "no", "off", ""): + monkeypatch.setenv("AGNES_HOME_SHOW_STATUS_FRAME", val) + assert get_home_status_frame_visibility() is False, f"{val!r} should hide" + for val in ("1", "true", "yes", "on", "anything"): + monkeypatch.setenv("AGNES_HOME_SHOW_STATUS_FRAME", val) + assert get_home_status_frame_visibility() is True, f"{val!r} should show" diff --git a/tests/test_schema_v42_migration.py b/tests/test_schema_v42_migration.py index ba695ff..eb791ae 100644 --- a/tests/test_schema_v42_migration.py +++ b/tests/test_schema_v42_migration.py @@ -5,7 +5,12 @@ from src.db import _ensure_schema as init_database, SCHEMA_VERSION def test_schema_version_is_42(): - assert SCHEMA_VERSION == 43 + # v44 bumped by PR #297 (homepage stats frame backing columns) — keep + # this assertion in lockstep with `_SYSTEM_SCHEMA` SCHEMA_VERSION + # constant. Test name preserved for git-blame continuity; the + # version-pinned tests in test_db_schema_version.py and + # test_home_stats.py carry the v44 commentary. + assert SCHEMA_VERSION == 44 def test_v42_tables_exist_after_init(tmp_path): @@ -46,7 +51,7 @@ def test_v41_to_v42_is_idempotent(tmp_path): conn = duckdb.connect(str(db_path)) init_database(conn) v = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()[0] - assert v == 43 + assert v == 44 conn.close() @@ -67,7 +72,7 @@ def test_v41_db_upgrades_cleanly(tmp_path): conn = duckdb.connect(str(db_path)) init_database(conn) v = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()[0] - assert v == 43 + assert v == 44 # All 7 new v41 tables exist after the v40→v41 upgrade tables = {row[0] for row in conn.execute( "SELECT table_name FROM information_schema.tables WHERE table_schema='main'" @@ -94,7 +99,7 @@ def test_v30_db_ladders_all_the_way_up(tmp_path): conn = duckdb.connect(str(db_path)) init_database(conn) v = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()[0] - assert v == 43 + assert v == 44 cnt = conn.execute("SELECT COUNT(*) FROM audit_log WHERE id='vintage'").fetchone()[0] assert cnt == 1 # New v41 table exists