b6543c9c55
1. .env_overlay write paths now match read path under STATE_DIR. app/main.py:343 reads via _state_dir() (post-PR #194), but two write sites still hardcoded ${DATA_DIR}/state/.env_overlay: - app/api/admin.py:2687 — configure endpoint secrets persistence - app/api/marketplaces.py:152 — marketplace PAT persistence Under flat-mount layout (STATE_DIR=/data-state) the admin UI wrote secrets to /data/state/.env_overlay while the app read from /data-state/.env_overlay, silently dropping the value on next restart. Both write sites now go through _state_dir(). 2. host-mount.yml: caddy inherits data:/srv:ro from base, but with no service populating the data: named volume (other services switched to direct /data binds), the inherited mount points at an empty Docker volume — try_files finds nothing, every parquet download falls through to uvicorn, defeating the v0.36.0 file_server bypass under the host-mount layout. Added a caddy override that restates all mounts including a direct /data:/srv:ro bind. Mirrors the comment + treatment already in flat-mount.yml.
463 lines
16 KiB
Python
463 lines
16 KiB
Python
"""Admin endpoints for marketplace git repositories.
|
|
|
|
CRUD + on-demand "Sync now" mirroring the /api/users shape. Tokens supplied
|
|
through the admin UI are persisted to data/state/.env_overlay (same pattern
|
|
as /api/admin/configure for Keboola/BigQuery) — never stored in the DB.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, List, Optional
|
|
|
|
import duckdb
|
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
|
from pydantic import BaseModel
|
|
|
|
from app.auth.access import require_admin
|
|
from app.auth.dependencies import _get_db
|
|
from app.resource_types import ResourceType
|
|
from src.marketplace import (
|
|
MarketplaceNotFound,
|
|
delete_marketplace_dir,
|
|
is_valid_slug,
|
|
sync_marketplaces,
|
|
sync_one,
|
|
)
|
|
from src.repositories.audit import AuditRepository
|
|
from src.repositories.marketplace_plugins import MarketplacePluginsRepository
|
|
from src.repositories.marketplace_registry import MarketplaceRegistryRepository
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter(prefix="/api/marketplaces", tags=["marketplaces"])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Audit helper — same shape as app/api/users.py::_audit
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _audit(
|
|
conn: duckdb.DuckDBPyConnection,
|
|
actor_id: str,
|
|
action: str,
|
|
target_id: str,
|
|
params: Optional[dict] = None,
|
|
) -> None:
|
|
try:
|
|
safe_params = None
|
|
if params:
|
|
safe_params = {}
|
|
for k, v in params.items():
|
|
if isinstance(v, datetime):
|
|
safe_params[k] = v.isoformat()
|
|
else:
|
|
safe_params[k] = v
|
|
AuditRepository(conn).log(
|
|
user_id=actor_id,
|
|
action=action,
|
|
resource=f"marketplace:{target_id}",
|
|
params=safe_params,
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pydantic models
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class CreateMarketplaceRequest(BaseModel):
|
|
name: str
|
|
slug: str
|
|
url: str
|
|
branch: Optional[str] = None
|
|
description: Optional[str] = None
|
|
token: Optional[str] = None
|
|
|
|
|
|
class UpdateMarketplaceRequest(BaseModel):
|
|
name: Optional[str] = None
|
|
url: Optional[str] = None
|
|
branch: Optional[str] = None
|
|
description: Optional[str] = None
|
|
# None = leave untouched; empty string = clear token; non-empty = rotate
|
|
token: Optional[str] = None
|
|
|
|
|
|
class MarketplaceResponse(BaseModel):
|
|
id: str
|
|
name: str
|
|
url: str
|
|
branch: Optional[str] = None
|
|
description: Optional[str] = None
|
|
registered_by: Optional[str] = None
|
|
registered_at: Optional[str] = None
|
|
last_synced_at: Optional[str] = None
|
|
last_commit_sha: Optional[str] = None
|
|
last_error: Optional[str] = None
|
|
has_token: bool = False
|
|
plugin_count: int = 0
|
|
|
|
|
|
def _to_response(row: dict, plugin_count: int = 0) -> MarketplaceResponse:
|
|
token_env = row.get("token_env") or ""
|
|
has_token = bool(token_env) and bool(os.environ.get(token_env, ""))
|
|
return MarketplaceResponse(
|
|
id=row["id"],
|
|
name=row["name"],
|
|
url=row["url"],
|
|
branch=row.get("branch"),
|
|
description=row.get("description"),
|
|
registered_by=row.get("registered_by"),
|
|
registered_at=str(row["registered_at"]) if row.get("registered_at") else None,
|
|
last_synced_at=str(row["last_synced_at"]) if row.get("last_synced_at") else None,
|
|
last_commit_sha=row.get("last_commit_sha"),
|
|
last_error=row.get("last_error"),
|
|
has_token=has_token,
|
|
plugin_count=plugin_count,
|
|
)
|
|
|
|
|
|
class PluginResponse(BaseModel):
|
|
name: str
|
|
description: Optional[str] = None
|
|
version: Optional[str] = None
|
|
author_name: Optional[str] = None
|
|
homepage: Optional[str] = None
|
|
category: Optional[str] = None
|
|
source_type: Optional[str] = None
|
|
source_spec: Optional[Any] = None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Token persistence — mirrors app/api/admin.py::configure_instance
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _token_env_name(slug: str) -> str:
|
|
"""Derive a conventional env-var name from a slug.
|
|
|
|
"foundry-ai" -> "AGNES_MARKETPLACE_FOUNDRY_AI_TOKEN"
|
|
"""
|
|
normalized = slug.upper().replace("-", "_")
|
|
return f"AGNES_MARKETPLACE_{normalized}_TOKEN"
|
|
|
|
|
|
def _persist_token(env_name: str, value: str) -> None:
|
|
"""Write (or update) a single key in ``${STATE_DIR}/.env_overlay`` and ``os.environ``.
|
|
|
|
Path resolution matches ``app/main.py``'s startup-time read; without
|
|
this alignment, marketplace PATs persisted under the flat-mount
|
|
layout (``STATE_DIR=/data-state``) would land at
|
|
``/data/state/.env_overlay`` while the app reads from
|
|
``/data-state/.env_overlay``, silently dropping the token on the
|
|
next restart.
|
|
"""
|
|
from app.secrets import _state_dir
|
|
overlay_path = _state_dir() / ".env_overlay"
|
|
overlay_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
existing: dict[str, str] = {}
|
|
if overlay_path.exists():
|
|
for line in overlay_path.read_text().splitlines():
|
|
if "=" in line and not line.startswith("#"):
|
|
k, v = line.split("=", 1)
|
|
existing[k.strip()] = v.strip()
|
|
|
|
if value:
|
|
existing[env_name] = value
|
|
os.environ[env_name] = value
|
|
else:
|
|
existing.pop(env_name, None)
|
|
os.environ.pop(env_name, None)
|
|
|
|
overlay_path.write_text(
|
|
"\n".join(f"{k}={v}" for k, v in existing.items()) + ("\n" if existing else "")
|
|
)
|
|
try:
|
|
overlay_path.chmod(0o600)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Endpoints
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@router.get("", response_model=List[MarketplaceResponse])
|
|
async def list_marketplaces(
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
counts = MarketplacePluginsRepository(conn).count_by_marketplace()
|
|
return [
|
|
_to_response(row, counts.get(row["id"], 0))
|
|
for row in MarketplaceRegistryRepository(conn).list_all()
|
|
]
|
|
|
|
|
|
@router.get("/{marketplace_id}/plugins", response_model=List[PluginResponse])
|
|
async def list_plugins(
|
|
marketplace_id: str,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
"""Return the cached plugin list for a marketplace.
|
|
|
|
Rows come from `marketplace_plugins`, which is refreshed from
|
|
`.claude-plugin/marketplace.json` on every successful sync. An
|
|
unsynced marketplace will return an empty list.
|
|
"""
|
|
if not MarketplaceRegistryRepository(conn).get(marketplace_id):
|
|
raise HTTPException(status_code=404, detail="marketplace not found")
|
|
rows = MarketplacePluginsRepository(conn).list_for_marketplace(marketplace_id)
|
|
return [
|
|
PluginResponse(
|
|
name=r["name"],
|
|
description=r.get("description"),
|
|
version=r.get("version"),
|
|
author_name=r.get("author_name"),
|
|
homepage=r.get("homepage"),
|
|
category=r.get("category"),
|
|
source_type=r.get("source_type"),
|
|
source_spec=r.get("source_spec"),
|
|
)
|
|
for r in rows
|
|
]
|
|
|
|
|
|
@router.post("", response_model=MarketplaceResponse, status_code=201)
|
|
async def create_marketplace(
|
|
payload: CreateMarketplaceRequest,
|
|
request: Request,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
slug = (payload.slug or "").strip().lower()
|
|
if not is_valid_slug(slug):
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="slug must match [a-z0-9][a-z0-9_-]{0,63} (1-64 chars, start with alnum)",
|
|
)
|
|
if not (payload.url or "").strip().lower().startswith("https://"):
|
|
raise HTTPException(status_code=400, detail="url must start with https://")
|
|
if not (payload.name or "").strip():
|
|
raise HTTPException(status_code=400, detail="name is required")
|
|
|
|
repo = MarketplaceRegistryRepository(conn)
|
|
if repo.get(slug):
|
|
raise HTTPException(status_code=409, detail=f"marketplace '{slug}' already exists")
|
|
|
|
token_env: Optional[str] = None
|
|
if payload.token:
|
|
token_env = _token_env_name(slug)
|
|
_persist_token(token_env, payload.token)
|
|
|
|
repo.register(
|
|
id=slug,
|
|
name=payload.name.strip(),
|
|
url=payload.url.strip(),
|
|
branch=(payload.branch or "").strip() or None,
|
|
token_env=token_env,
|
|
description=payload.description,
|
|
registered_by=user.get("email"),
|
|
)
|
|
_audit(
|
|
conn,
|
|
user["id"],
|
|
"marketplace.create",
|
|
slug,
|
|
{"url": payload.url, "has_token": bool(payload.token)},
|
|
)
|
|
return _to_response(repo.get(slug))
|
|
|
|
|
|
@router.patch("/{marketplace_id}", response_model=MarketplaceResponse)
|
|
async def update_marketplace(
|
|
marketplace_id: str,
|
|
payload: UpdateMarketplaceRequest,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
repo = MarketplaceRegistryRepository(conn)
|
|
existing = repo.get(marketplace_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="marketplace not found")
|
|
|
|
# Start with the existing row; override fields the caller provided.
|
|
updated = {
|
|
"id": existing["id"],
|
|
"name": existing["name"],
|
|
"url": existing["url"],
|
|
"branch": existing.get("branch"),
|
|
"token_env": existing.get("token_env"),
|
|
"description": existing.get("description"),
|
|
"registered_by": existing.get("registered_by"),
|
|
}
|
|
changed: dict = {}
|
|
if payload.name is not None:
|
|
if not payload.name.strip():
|
|
raise HTTPException(status_code=400, detail="name cannot be empty")
|
|
updated["name"] = payload.name.strip()
|
|
changed["name"] = updated["name"]
|
|
if payload.url is not None:
|
|
url = payload.url.strip()
|
|
if not url.lower().startswith("https://"):
|
|
raise HTTPException(status_code=400, detail="url must start with https://")
|
|
updated["url"] = url
|
|
changed["url"] = url
|
|
if payload.branch is not None:
|
|
updated["branch"] = payload.branch.strip() or None
|
|
changed["branch"] = updated["branch"]
|
|
if payload.description is not None:
|
|
updated["description"] = payload.description
|
|
changed["description"] = payload.description
|
|
|
|
if payload.token is not None:
|
|
# None = untouched; "" = clear token_env binding; non-empty = rotate.
|
|
if payload.token == "":
|
|
if updated["token_env"]:
|
|
_persist_token(updated["token_env"], "")
|
|
updated["token_env"] = None
|
|
changed["token"] = "cleared"
|
|
else:
|
|
env_name = _token_env_name(marketplace_id)
|
|
_persist_token(env_name, payload.token)
|
|
updated["token_env"] = env_name
|
|
changed["token"] = "rotated"
|
|
|
|
repo.register(
|
|
id=updated["id"],
|
|
name=updated["name"],
|
|
url=updated["url"],
|
|
branch=updated["branch"],
|
|
token_env=updated["token_env"],
|
|
description=updated["description"],
|
|
registered_by=updated["registered_by"],
|
|
)
|
|
_audit(conn, user["id"], "marketplace.update", marketplace_id, changed)
|
|
counts = MarketplacePluginsRepository(conn).count_by_marketplace()
|
|
return _to_response(repo.get(marketplace_id), counts.get(marketplace_id, 0))
|
|
|
|
|
|
@router.delete("/{marketplace_id}", status_code=204)
|
|
async def delete_marketplace(
|
|
marketplace_id: str,
|
|
purge: bool = False,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
repo = MarketplaceRegistryRepository(conn)
|
|
existing = repo.get(marketplace_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="marketplace not found")
|
|
|
|
# Also clear any overlay token binding so a re-created marketplace of the
|
|
# same slug doesn't accidentally inherit the old PAT.
|
|
if existing.get("token_env"):
|
|
_persist_token(existing["token_env"], "")
|
|
|
|
repo.unregister(marketplace_id)
|
|
# Drop cached plugin rows and any resource grants that reference plugins
|
|
# from this marketplace. resource_grants stores resource_id as
|
|
# "<marketplace_slug>/<plugin_name>" — match the slash-prefix via
|
|
# starts_with(), not LIKE: marketplace slugs may contain '_' (validated
|
|
# by [a-z0-9][a-z0-9_-]{0,63}) and LIKE would interpret it as a
|
|
# single-char wildcard, silently dropping grants from sibling
|
|
# marketplaces whose slug differs by exactly one character.
|
|
try:
|
|
conn.execute(
|
|
"DELETE FROM marketplace_plugins WHERE marketplace_id = ?",
|
|
[marketplace_id],
|
|
)
|
|
conn.execute(
|
|
"DELETE FROM resource_grants "
|
|
"WHERE resource_type = ? AND starts_with(resource_id, ? || '/')",
|
|
[ResourceType.MARKETPLACE_PLUGIN.value, marketplace_id],
|
|
)
|
|
except Exception as e:
|
|
logger.warning("cleanup for marketplace %s failed: %s", marketplace_id, e)
|
|
purged = False
|
|
if purge:
|
|
try:
|
|
purged = delete_marketplace_dir(marketplace_id)
|
|
except Exception as e:
|
|
logger.warning("delete_marketplace_dir(%s) failed: %s", marketplace_id, e)
|
|
|
|
_audit(
|
|
conn,
|
|
user["id"],
|
|
"marketplace.delete",
|
|
marketplace_id,
|
|
{"purged_disk": purged},
|
|
)
|
|
|
|
|
|
@router.post("/{marketplace_id}/sync")
|
|
async def trigger_sync(
|
|
marketplace_id: str,
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
try:
|
|
result = sync_one(marketplace_id)
|
|
except MarketplaceNotFound:
|
|
raise HTTPException(status_code=404, detail="marketplace not found")
|
|
except (RuntimeError, ValueError) as e:
|
|
_audit(conn, user["id"], "marketplace.sync_failed", marketplace_id, {"error": str(e)})
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
_audit(
|
|
conn,
|
|
user["id"],
|
|
"marketplace.sync",
|
|
marketplace_id,
|
|
{"commit": result["commit"], "action": result["action"]},
|
|
)
|
|
return result
|
|
|
|
|
|
@router.post("/sync-all")
|
|
def trigger_sync_all(
|
|
user: dict = Depends(require_admin),
|
|
conn: duckdb.DuckDBPyConnection = Depends(_get_db),
|
|
):
|
|
"""Sync every registered marketplace.
|
|
|
|
Wired up so the scheduler service can drive the nightly refresh over
|
|
HTTP. The previous implementation called ``src.marketplace.sync_marketplaces``
|
|
in-process from the scheduler container, which conflicted with the app's
|
|
long-lived ``system.duckdb`` handle (DuckDB allows only one writer per
|
|
file across processes). Routing through the app inherits the existing
|
|
connection without contention.
|
|
|
|
Declared ``def`` (not ``async def``) so FastAPI runs it in a thread
|
|
pool — :func:`sync_marketplaces` does blocking I/O (subprocess git
|
|
clones with ``GIT_TIMEOUT_SEC=300`` per repo, DuckDB writes, a
|
|
process-wide threading.Lock) and would freeze the event loop for the
|
|
duration of a bulk sync if it ran on the asyncio thread. Health
|
|
checks, login redirects, and every other concurrent request keep
|
|
serving while the bulk sync churns through the registry.
|
|
|
|
One audit row per call summarises the outcome — per-marketplace details
|
|
live in ``marketplace_registry`` and the per-call result payload below.
|
|
"""
|
|
result = sync_marketplaces()
|
|
# _audit appends "marketplace:" to the target id when writing the
|
|
# resource column. "_all" produces "marketplace:_all" — a stable,
|
|
# greppable sentinel for bulk-sync rows; the real per-marketplace
|
|
# commit/error breakdown is in the params payload.
|
|
_audit(
|
|
conn,
|
|
user["id"],
|
|
"marketplace.sync_all",
|
|
"_all",
|
|
{
|
|
"synced": [r.get("id") for r in result.get("synced", [])],
|
|
"errors": [{"id": e.get("id"), "error": e.get("error")} for e in result.get("errors", [])],
|
|
},
|
|
)
|
|
return result
|