agnes-the-ai-analyst/app/api/marketplaces.py

"""Admin endpoints for marketplace git repositories.

CRUD + on-demand "Sync now" mirroring the /api/users shape. Tokens supplied
through the admin UI are persisted to data/state/.env_overlay (same pattern
as /api/admin/configure for Keboola/BigQuery) — never stored in the DB.
"""

import logging
import os
from datetime import datetime
from pathlib import Path
from typing import Any, List, Optional

import duckdb
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel

from app.auth.access import require_admin
from app.auth.dependencies import _get_db
from app.resource_types import ResourceType
from src.marketplace import (
    MarketplaceNotFound,
    delete_marketplace_dir,
    is_valid_slug,
    sync_marketplaces,
    sync_one,
)
from src.repositories.audit import AuditRepository
from src.repositories.marketplace_plugins import MarketplacePluginsRepository
from src.repositories.marketplace_registry import MarketplaceRegistryRepository

logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/marketplaces", tags=["marketplaces"])


# ---------------------------------------------------------------------------
# Audit helper — same shape as app/api/users.py::_audit
# ---------------------------------------------------------------------------


def _audit(
    conn: duckdb.DuckDBPyConnection,
    actor_id: str,
    action: str,
    target_id: str,
    params: Optional[dict] = None,
) -> None:
    try:
        safe_params = None
        if params:
            safe_params = {}
            for k, v in params.items():
                if isinstance(v, datetime):
                    safe_params[k] = v.isoformat()
                else:
                    safe_params[k] = v
        AuditRepository(conn).log(
            user_id=actor_id,
            action=action,
            resource=f"marketplace:{target_id}",
            params=safe_params,
        )
    except Exception:
        pass


# ---------------------------------------------------------------------------
# Pydantic models
# ---------------------------------------------------------------------------


class CreateMarketplaceRequest(BaseModel):
    name: str
    slug: str
    url: str
    branch: Optional[str] = None
    description: Optional[str] = None
    token: Optional[str] = None


class UpdateMarketplaceRequest(BaseModel):
    name: Optional[str] = None
    url: Optional[str] = None
    branch: Optional[str] = None
    description: Optional[str] = None
    # None = leave untouched; empty string = clear token; non-empty = rotate
    token: Optional[str] = None


class MarketplaceResponse(BaseModel):
    id: str
    name: str
    url: str
    branch: Optional[str] = None
    description: Optional[str] = None
    registered_by: Optional[str] = None
    registered_at: Optional[str] = None
    last_synced_at: Optional[str] = None
    last_commit_sha: Optional[str] = None
    last_error: Optional[str] = None
    has_token: bool = False
    plugin_count: int = 0


def _to_response(row: dict, plugin_count: int = 0) -> MarketplaceResponse:
    token_env = row.get("token_env") or ""
    has_token = bool(token_env) and bool(os.environ.get(token_env, ""))
    return MarketplaceResponse(
        id=row["id"],
        name=row["name"],
        url=row["url"],
        branch=row.get("branch"),
        description=row.get("description"),
        registered_by=row.get("registered_by"),
        registered_at=str(row["registered_at"]) if row.get("registered_at") else None,
        last_synced_at=str(row["last_synced_at"]) if row.get("last_synced_at") else None,
        last_commit_sha=row.get("last_commit_sha"),
        last_error=row.get("last_error"),
        has_token=has_token,
        plugin_count=plugin_count,
    )


class PluginResponse(BaseModel):
    name: str
    description: Optional[str] = None
    version: Optional[str] = None
    author_name: Optional[str] = None
    homepage: Optional[str] = None
    category: Optional[str] = None
    source_type: Optional[str] = None
    source_spec: Optional[Any] = None


# ---------------------------------------------------------------------------
# Token persistence — mirrors app/api/admin.py::configure_instance
# ---------------------------------------------------------------------------


def _token_env_name(slug: str) -> str:
    """Derive a conventional env-var name from a slug.

    "foundry-ai" -> "AGNES_MARKETPLACE_FOUNDRY_AI_TOKEN"
    """
    normalized = slug.upper().replace("-", "_")
    return f"AGNES_MARKETPLACE_{normalized}_TOKEN"


def _persist_token(env_name: str, value: str) -> None:
    """Write (or update) a single key in ``${STATE_DIR}/.env_overlay`` and ``os.environ``.

    Path resolution matches ``app/main.py``'s startup-time read; without
    this alignment, marketplace PATs persisted under the flat-mount
    layout (``STATE_DIR=/data-state``) would land at
    ``/data/state/.env_overlay`` while the app reads from
    ``/data-state/.env_overlay``, silently dropping the token on the
    next restart.
    """
    from app.secrets import _state_dir
    overlay_path = _state_dir() / ".env_overlay"
    overlay_path.parent.mkdir(parents=True, exist_ok=True)

    existing: dict[str, str] = {}
    if overlay_path.exists():
        for line in overlay_path.read_text().splitlines():
            if "=" in line and not line.startswith("#"):
                k, v = line.split("=", 1)
                existing[k.strip()] = v.strip()

    if value:
        existing[env_name] = value
        os.environ[env_name] = value
    else:
        existing.pop(env_name, None)
        os.environ.pop(env_name, None)

    overlay_path.write_text(
        "\n".join(f"{k}={v}" for k, v in existing.items()) + ("\n" if existing else "")
    )
    try:
        overlay_path.chmod(0o600)
    except OSError:
        pass


# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------


@router.get("", response_model=List[MarketplaceResponse])
async def list_marketplaces(
    user: dict = Depends(require_admin),
    conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
    counts = MarketplacePluginsRepository(conn).count_by_marketplace()
    return [
        _to_response(row, counts.get(row["id"], 0))
        for row in MarketplaceRegistryRepository(conn).list_all()
    ]


@router.get("/{marketplace_id}/plugins", response_model=List[PluginResponse])
async def list_plugins(
    marketplace_id: str,
    user: dict = Depends(require_admin),
    conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
    """Return the cached plugin list for a marketplace.

    Rows come from `marketplace_plugins`, which is refreshed from
    `.claude-plugin/marketplace.json` on every successful sync. An
    unsynced marketplace will return an empty list.
    """
    if not MarketplaceRegistryRepository(conn).get(marketplace_id):
        raise HTTPException(status_code=404, detail="marketplace not found")
    rows = MarketplacePluginsRepository(conn).list_for_marketplace(marketplace_id)
    return [
        PluginResponse(
            name=r["name"],
            description=r.get("description"),
            version=r.get("version"),
            author_name=r.get("author_name"),
            homepage=r.get("homepage"),
            category=r.get("category"),
            source_type=r.get("source_type"),
            source_spec=r.get("source_spec"),
        )
        for r in rows
    ]


@router.post("", response_model=MarketplaceResponse, status_code=201)
async def create_marketplace(
    payload: CreateMarketplaceRequest,
    request: Request,
    user: dict = Depends(require_admin),
    conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
    slug = (payload.slug or "").strip().lower()
    if not is_valid_slug(slug):
        raise HTTPException(
            status_code=400,
            detail="slug must match [a-z0-9][a-z0-9_-]{0,63} (1-64 chars, start with alnum)",
        )
    if not (payload.url or "").strip().lower().startswith("https://"):
        raise HTTPException(status_code=400, detail="url must start with https://")
    if not (payload.name or "").strip():
        raise HTTPException(status_code=400, detail="name is required")

    repo = MarketplaceRegistryRepository(conn)
    if repo.get(slug):
        raise HTTPException(status_code=409, detail=f"marketplace '{slug}' already exists")

    token_env: Optional[str] = None
    if payload.token:
        token_env = _token_env_name(slug)
        _persist_token(token_env, payload.token)

    repo.register(
        id=slug,
        name=payload.name.strip(),
        url=payload.url.strip(),
        branch=(payload.branch or "").strip() or None,
        token_env=token_env,
        description=payload.description,
        registered_by=user.get("email"),
    )
    _audit(
        conn,
        user["id"],
        "marketplace.create",
        slug,
        {"url": payload.url, "has_token": bool(payload.token)},
    )
    return _to_response(repo.get(slug))


@router.patch("/{marketplace_id}", response_model=MarketplaceResponse)
async def update_marketplace(
    marketplace_id: str,
    payload: UpdateMarketplaceRequest,
    user: dict = Depends(require_admin),
    conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
    repo = MarketplaceRegistryRepository(conn)
    existing = repo.get(marketplace_id)
    if not existing:
        raise HTTPException(status_code=404, detail="marketplace not found")

    # Start with the existing row; override fields the caller provided.
    updated = {
        "id": existing["id"],
        "name": existing["name"],
        "url": existing["url"],
        "branch": existing.get("branch"),
        "token_env": existing.get("token_env"),
        "description": existing.get("description"),
        "registered_by": existing.get("registered_by"),
    }
    changed: dict = {}
    if payload.name is not None:
        if not payload.name.strip():
            raise HTTPException(status_code=400, detail="name cannot be empty")
        updated["name"] = payload.name.strip()
        changed["name"] = updated["name"]
    if payload.url is not None:
        url = payload.url.strip()
        if not url.lower().startswith("https://"):
            raise HTTPException(status_code=400, detail="url must start with https://")
        updated["url"] = url
        changed["url"] = url
    if payload.branch is not None:
        updated["branch"] = payload.branch.strip() or None
        changed["branch"] = updated["branch"]
    if payload.description is not None:
        updated["description"] = payload.description
        changed["description"] = payload.description

    if payload.token is not None:
        # None = untouched; "" = clear token_env binding; non-empty = rotate.
        if payload.token == "":
            if updated["token_env"]:
                _persist_token(updated["token_env"], "")
            updated["token_env"] = None
            changed["token"] = "cleared"
        else:
            env_name = _token_env_name(marketplace_id)
            _persist_token(env_name, payload.token)
            updated["token_env"] = env_name
            changed["token"] = "rotated"

    repo.register(
        id=updated["id"],
        name=updated["name"],
        url=updated["url"],
        branch=updated["branch"],
        token_env=updated["token_env"],
        description=updated["description"],
        registered_by=updated["registered_by"],
    )
    _audit(conn, user["id"], "marketplace.update", marketplace_id, changed)
    counts = MarketplacePluginsRepository(conn).count_by_marketplace()
    return _to_response(repo.get(marketplace_id), counts.get(marketplace_id, 0))


@router.delete("/{marketplace_id}", status_code=204)
async def delete_marketplace(
    marketplace_id: str,
    purge: bool = False,
    user: dict = Depends(require_admin),
    conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
    repo = MarketplaceRegistryRepository(conn)
    existing = repo.get(marketplace_id)
    if not existing:
        raise HTTPException(status_code=404, detail="marketplace not found")

    # Also clear any overlay token binding so a re-created marketplace of the
    # same slug doesn't accidentally inherit the old PAT.
    if existing.get("token_env"):
        _persist_token(existing["token_env"], "")

    repo.unregister(marketplace_id)
    # Drop cached plugin rows and any resource grants that reference plugins
    # from this marketplace. resource_grants stores resource_id as
    # "<marketplace_slug>/<plugin_name>" — match the slash-prefix via
    # starts_with(), not LIKE: marketplace slugs may contain '_' (validated
    # by [a-z0-9][a-z0-9_-]{0,63}) and LIKE would interpret it as a
    # single-char wildcard, silently dropping grants from sibling
    # marketplaces whose slug differs by exactly one character.
    try:
        conn.execute(
            "DELETE FROM marketplace_plugins WHERE marketplace_id = ?",
            [marketplace_id],
        )
        conn.execute(
            "DELETE FROM resource_grants "
            "WHERE resource_type = ? AND starts_with(resource_id, ? || '/')",
            [ResourceType.MARKETPLACE_PLUGIN.value, marketplace_id],
        )
        # Drop user subscriptions to plugins from this marketplace so a
        # re-registered slug doesn't inherit stale subscribe state.
        from src.repositories.user_curated_subscriptions import (
            UserCuratedSubscriptionsRepository,
        )
        UserCuratedSubscriptionsRepository(conn).delete_for_marketplace(
            marketplace_id
        )
    except Exception as e:
        logger.warning("cleanup for marketplace %s failed: %s", marketplace_id, e)
    purged = False
    if purge:
        try:
            purged = delete_marketplace_dir(marketplace_id)
        except Exception as e:
            logger.warning("delete_marketplace_dir(%s) failed: %s", marketplace_id, e)

    _audit(
        conn,
        user["id"],
        "marketplace.delete",
        marketplace_id,
        {"purged_disk": purged},
    )


@router.post("/{marketplace_id}/sync")
async def trigger_sync(
    marketplace_id: str,
    user: dict = Depends(require_admin),
    conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
    try:
        result = sync_one(marketplace_id)
    except MarketplaceNotFound:
        raise HTTPException(status_code=404, detail="marketplace not found")
    except (RuntimeError, ValueError) as e:
        _audit(conn, user["id"], "marketplace.sync_failed", marketplace_id, {"error": str(e)})
        raise HTTPException(status_code=500, detail=str(e))
    _audit(
        conn,
        user["id"],
        "marketplace.sync",
        marketplace_id,
        {"commit": result["commit"], "action": result["action"]},
    )
    return result


@router.post("/sync-all")
def trigger_sync_all(
    user: dict = Depends(require_admin),
    conn: duckdb.DuckDBPyConnection = Depends(_get_db),
):
    """Sync every registered marketplace.

    Wired up so the scheduler service can drive the nightly refresh over
    HTTP. The previous implementation called ``src.marketplace.sync_marketplaces``
    in-process from the scheduler container, which conflicted with the app's
    long-lived ``system.duckdb`` handle (DuckDB allows only one writer per
    file across processes). Routing through the app inherits the existing
    connection without contention.

    Declared ``def`` (not ``async def``) so FastAPI runs it in a thread
    pool — :func:`sync_marketplaces` does blocking I/O (subprocess git
    clones with ``GIT_TIMEOUT_SEC=300`` per repo, DuckDB writes, a
    process-wide threading.Lock) and would freeze the event loop for the
    duration of a bulk sync if it ran on the asyncio thread. Health
    checks, login redirects, and every other concurrent request keep
    serving while the bulk sync churns through the registry.

    One audit row per call summarises the outcome — per-marketplace details
    live in ``marketplace_registry`` and the per-call result payload below.
    """
    result = sync_marketplaces()
    # _audit appends "marketplace:" to the target id when writing the
    # resource column. "_all" produces "marketplace:_all" — a stable,
    # greppable sentinel for bulk-sync rows; the real per-marketplace
    # commit/error breakdown is in the params payload.
    _audit(
        conn,
        user["id"],
        "marketplace.sync_all",
        "_all",
        {
            "synced": [r.get("id") for r in result.get("synced", [])],
            "errors": [{"id": e.get("id"), "error": e.get("error")} for e in result.get("errors", [])],
        },
    )
    return result