chore: Docker prod config (Python 3.13, no reload), fix utcnow deprecation, update docs
This commit is contained in:
parent
05a1b452e9
commit
92fbb88c15
13 changed files with 29 additions and 1665 deletions
|
|
@ -53,7 +53,7 @@ docker compose --profile full up # Include telegram bot
|
||||||
├── scripts/ # Utility + migration scripts
|
├── scripts/ # Utility + migration scripts
|
||||||
├── config/ # Configuration templates (instance.yaml.example)
|
├── config/ # Configuration templates (instance.yaml.example)
|
||||||
├── docs/ # Documentation + metric YAML definitions
|
├── docs/ # Documentation + metric YAML definitions
|
||||||
└── tests/ # Test suite (704 tests)
|
└── tests/ # Test suite (633 tests)
|
||||||
```
|
```
|
||||||
|
|
||||||
## Architecture: extract.duckdb Contract
|
## Architecture: extract.duckdb Contract
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
FROM python:3.11-slim
|
FROM python:3.13-slim
|
||||||
|
|
||||||
# Install uv for fast dependency management
|
# Install uv for fast dependency management
|
||||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from fastapi import APIRouter, Request, Response
|
from fastapi import APIRouter, Request, Response
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
@ -52,7 +52,7 @@ def _log_webhook_event(event_data: dict) -> None:
|
||||||
"""Log webhook event to file for debugging/audit."""
|
"""Log webhook event to file for debugging/audit."""
|
||||||
try:
|
try:
|
||||||
WEBHOOK_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
WEBHOOK_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f")
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f")
|
||||||
event_type = event_data.get("webhookEvent", "unknown").replace(":", "_")
|
event_type = event_data.get("webhookEvent", "unknown").replace(":", "_")
|
||||||
filename = f"{timestamp}_{event_type}.json"
|
filename = f"{timestamp}_{event_type}.json"
|
||||||
filepath = WEBHOOK_LOG_DIR / filename
|
filepath = WEBHOOK_LOG_DIR / filename
|
||||||
|
|
|
||||||
|
|
@ -33,7 +33,7 @@ import sys
|
||||||
import time
|
import time
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Iterator
|
from typing import Iterator
|
||||||
|
|
||||||
|
|
@ -292,7 +292,7 @@ class JiraBackfill:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Add sync metadata
|
# Add sync metadata
|
||||||
issue_data["_synced_at"] = datetime.utcnow().isoformat()
|
issue_data["_synced_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
file_path = self.issues_dir / f"{issue_key}.json"
|
file_path = self.issues_dir / f"{issue_key}.json"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,7 @@ import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Iterator
|
from typing import Iterator
|
||||||
|
|
||||||
|
|
@ -151,8 +151,8 @@ class JiraConsistencyChecker:
|
||||||
Set of issue keys from Jira API (ground truth)
|
Set of issue keys from Jira API (ground truth)
|
||||||
"""
|
"""
|
||||||
# Calculate cutoff date with grace period
|
# Calculate cutoff date with grace period
|
||||||
cutoff = datetime.utcnow() - timedelta(days=max_age_days)
|
cutoff = datetime.now(timezone.utc) - timedelta(days=max_age_days)
|
||||||
grace_cutoff = datetime.utcnow() - timedelta(minutes=self.GRACE_PERIOD_MINUTES)
|
grace_cutoff = datetime.now(timezone.utc) - timedelta(minutes=self.GRACE_PERIOD_MINUTES)
|
||||||
|
|
||||||
# JQL: fetch issues created after cutoff, but not too recent (grace period)
|
# JQL: fetch issues created after cutoff, but not too recent (grace period)
|
||||||
jira_project = os.environ.get("JIRA_PROJECT", "")
|
jira_project = os.environ.get("JIRA_PROJECT", "")
|
||||||
|
|
@ -533,7 +533,7 @@ class JiraConsistencyChecker:
|
||||||
|
|
||||||
# Build report
|
# Build report
|
||||||
report = {
|
report = {
|
||||||
"timestamp": datetime.utcnow().isoformat(),
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||||
"check_type": "incremental" if max_age_days <= 90 else "deep",
|
"check_type": "incremental" if max_age_days <= 90 else "deep",
|
||||||
"max_age_days": max_age_days,
|
"max_age_days": max_age_days,
|
||||||
"duration_seconds": round(duration, 2),
|
"duration_seconds": round(duration, 2),
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
@ -259,7 +259,7 @@ class JiraService:
|
||||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# Add metadata
|
# Add metadata
|
||||||
issue_data["_synced_at"] = datetime.utcnow().isoformat()
|
issue_data["_synced_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
# Fetch and embed remote links for Parquet transform
|
# Fetch and embed remote links for Parquet transform
|
||||||
issue_key_for_links = issue_data.get("key")
|
issue_key_for_links = issue_data.get("key")
|
||||||
|
|
@ -525,7 +525,7 @@ class JiraService:
|
||||||
with issue_json_lock(issues_dir, issue_key):
|
with issue_json_lock(issues_dir, issue_key):
|
||||||
with open(file_path) as f:
|
with open(file_path) as f:
|
||||||
data = json.load(f)
|
data = json.load(f)
|
||||||
data["_deleted_at"] = datetime.utcnow().isoformat()
|
data["_deleted_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
# Atomic write: temp file + replace
|
# Atomic write: temp file + replace
|
||||||
fd, tmp_path = tempfile.mkstemp(
|
fd, tmp_path = tempfile.mkstemp(
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
@ -549,7 +549,7 @@ def transform_remote_links(raw_issue: dict) -> list[dict]:
|
||||||
def get_month_key(dt: datetime | None) -> str:
|
def get_month_key(dt: datetime | None) -> str:
|
||||||
"""Get month key (YYYY-MM) from datetime, defaulting to current month."""
|
"""Get month key (YYYY-MM) from datetime, defaulting to current month."""
|
||||||
if dt is None:
|
if dt is None:
|
||||||
dt = datetime.utcnow()
|
dt = datetime.now(timezone.utc)
|
||||||
return dt.strftime("%Y-%m")
|
return dt.strftime("%Y-%m")
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from flask import Blueprint, abort, jsonify, request
|
from flask import Blueprint, abort, jsonify, request
|
||||||
|
|
||||||
|
|
@ -69,7 +69,7 @@ def log_webhook_event(event_data: dict) -> None:
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
WEBHOOK_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
WEBHOOK_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f")
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f")
|
||||||
event_type = event_data.get("webhookEvent", "unknown").replace(":", "_")
|
event_type = event_data.get("webhookEvent", "unknown").replace(":", "_")
|
||||||
filename = f"{timestamp}_{event_type}.json"
|
filename = f"{timestamp}_{event_type}.json"
|
||||||
filepath = WEBHOOK_LOG_DIR / filename
|
filepath = WEBHOOK_LOG_DIR / filename
|
||||||
|
|
|
||||||
8
docker-compose.override.yml
Normal file
8
docker-compose.override.yml
Normal file
|
|
@ -0,0 +1,8 @@
|
||||||
|
# Development overrides — auto-reload + source mount
|
||||||
|
# This file is auto-loaded by `docker compose up` when present
|
||||||
|
services:
|
||||||
|
app:
|
||||||
|
command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
|
||||||
|
volumes:
|
||||||
|
- .:/app
|
||||||
|
- data:/data
|
||||||
|
|
@ -1,11 +1,10 @@
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
build: .
|
build: .
|
||||||
command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
|
command: uvicorn app.main:app --host 0.0.0.0 --port 8000
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "8000:8000"
|
||||||
volumes:
|
volumes:
|
||||||
- .:/app
|
|
||||||
- data:/data
|
- data:/data
|
||||||
env_file: .env
|
env_file: .env
|
||||||
environment:
|
environment:
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,69 +0,0 @@
|
||||||
# Complete System Implementation Plan
|
|
||||||
|
|
||||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans.
|
|
||||||
|
|
||||||
**Goal:** Make the new FastAPI system feature-complete with the old Flask system. Every route, every service function, every template — replicated with the new DuckDB-backed architecture.
|
|
||||||
|
|
||||||
**Status:** Infrastructure done (DuckDB, repos, FastAPI skeleton, CLI, Docker). Missing: business logic wiring, web UI, auth providers, 18 routes, 38 service functions.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Part A: Wire sync trigger to DataSyncManager
|
|
||||||
|
|
||||||
Files:
|
|
||||||
- Modify: `app/api/sync.py` (replace stub with real sync)
|
|
||||||
- Modify: `app/main.py` (add instance config loading)
|
|
||||||
|
|
||||||
## Part B: Instance config integration
|
|
||||||
|
|
||||||
Files:
|
|
||||||
- Create: `app/instance_config.py` (load instance.yaml, expose to FastAPI)
|
|
||||||
- Modify: `app/main.py` (lifespan event loads config)
|
|
||||||
- Modify: `app/api/health.py` (include data source info)
|
|
||||||
|
|
||||||
## Part C: Web UI — Jinja2 templates in FastAPI
|
|
||||||
|
|
||||||
Files:
|
|
||||||
- Create: `app/web/router.py` (ALL web routes: /, /dashboard, /catalog, /login, /corporate-memory, /admin/tables, etc.)
|
|
||||||
- Copy: `webapp/templates/` → `app/web/templates/` (adapt for FastAPI)
|
|
||||||
- Copy: `webapp/static/` → `app/web/static/`
|
|
||||||
- Modify: `app/main.py` (mount templates + static)
|
|
||||||
|
|
||||||
## Part D: Auth providers (Google OAuth + Email + Password)
|
|
||||||
|
|
||||||
Files:
|
|
||||||
- Create: `app/auth/providers/google.py`
|
|
||||||
- Create: `app/auth/providers/email.py`
|
|
||||||
- Create: `app/auth/providers/password.py`
|
|
||||||
- Modify: `app/auth/router.py` (OAuth callback, magic link, password verify)
|
|
||||||
|
|
||||||
## Part E: Missing API endpoints (18 routes)
|
|
||||||
|
|
||||||
Files:
|
|
||||||
- Create: `app/api/catalog.py` (profile, metrics)
|
|
||||||
- Create: `app/api/telegram.py` (verify, unlink, status)
|
|
||||||
- Create: `app/api/desktop.py` (scripts, run)
|
|
||||||
- Create: `app/api/admin.py` (tables discover, registry CRUD)
|
|
||||||
- Modify: `app/api/memory.py` (add 10 admin governance endpoints)
|
|
||||||
- Modify: `app/api/sync.py` (add sync-settings, table-subscriptions)
|
|
||||||
|
|
||||||
## Part F: Service logic rewiring
|
|
||||||
|
|
||||||
Files:
|
|
||||||
- Rewrite all old service calls to use DuckDB repositories
|
|
||||||
- Bridge: old corporate_memory_service → KnowledgeRepository
|
|
||||||
- Bridge: old sync_settings_service → SyncSettingsRepository
|
|
||||||
- Bridge: old telegram_service → TelegramRepository
|
|
||||||
|
|
||||||
## Part G: CLI missing commands + old test fixes
|
|
||||||
|
|
||||||
Files:
|
|
||||||
- Create: `cli/commands/setup.py`
|
|
||||||
- Create: `cli/commands/server.py`
|
|
||||||
- Create: `cli/commands/explore.py`
|
|
||||||
- Fix: old tests to work with new code
|
|
||||||
|
|
||||||
## Part H: Full test coverage
|
|
||||||
|
|
||||||
- Integration tests for all 40 routes
|
|
||||||
- E2E Docker test
|
|
||||||
|
|
@ -15,7 +15,7 @@ import math
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import tempfile
|
import tempfile
|
||||||
from datetime import datetime
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
|
@ -1210,7 +1210,7 @@ def profile_changed_tables(table_names: list[str]) -> dict:
|
||||||
|
|
||||||
# Write atomically
|
# Write atomically
|
||||||
output = {
|
output = {
|
||||||
"generated_at": datetime.utcnow().isoformat() + "Z",
|
"generated_at": datetime.now(timezone.utc).isoformat() + "Z",
|
||||||
"version": "1.0",
|
"version": "1.0",
|
||||||
"tables": merged,
|
"tables": merged,
|
||||||
}
|
}
|
||||||
|
|
@ -1376,7 +1376,7 @@ def main() -> None:
|
||||||
|
|
||||||
# Build output
|
# Build output
|
||||||
output = {
|
output = {
|
||||||
"generated_at": datetime.utcnow().isoformat() + "Z",
|
"generated_at": datetime.now(timezone.utc).isoformat() + "Z",
|
||||||
"version": "1.0",
|
"version": "1.0",
|
||||||
"tables": profiles,
|
"tables": profiles,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue