{{ provider.login_button.subtitle }}
+{% endif %} +{% endfor %} +``` + +### Session Contract + +All auth providers MUST set the same session structure: +```python +session["user"] = { + "email": "user@acme.com", # Required - unique identifier + "name": "John Doe", # Optional - display name + "picture": "https://...", # Optional - avatar URL +} +``` + +## Implementation Phases + +### Phase 1: Move services to services/ (git mv + fix imports) + +**Files moved:** +- `server/telegram_bot/` -> `services/telegram_bot/` +- `server/ws_gateway/` -> `services/ws_gateway/` +- `server/corporate_memory/` -> `services/corporate_memory/` +- `server/session_collector.py` -> `services/session_collector/collector.py` +- Service files from `server/*.service` -> `services/*/systemd/` +- Timer files from `server/*.timer` -> `services/*/systemd/` + +**Import fixes:** +- `from server.telegram_bot.X` -> `from services.telegram_bot.X` (in webapp/app.py) +- `python -m server.X` -> `python -m services.X` (in systemd files, bin/ scripts) +- Internal imports within services stay as relative imports + +**Config updates:** +- `server/deploy.sh` - discover services from `services/*/systemd/` +- `server/bin/collect-knowledge` - update module path +- `server/bin/collect-sessions` - update module path + +### Phase 2: Extract auth providers to auth/ + +**Files moved:** +- `webapp/auth.py` -> `auth/google/provider.py` (OAuth logic) +- `webapp/password_auth.py` -> `auth/password/provider.py` +- `webapp/desktop_auth.py` -> `auth/desktop/provider.py` + +**What stays in webapp/auth.py:** +- `login_required` decorator (used everywhere) +- `/logout` route +- Session management utils + +**New files:** +- `auth/__init__.py` - AuthProvider ABC + discover_providers() +- `auth/google/__init__.py` +- `auth/password/__init__.py` +- `auth/desktop/__init__.py` + +**webapp/app.py changes:** +- Replace hardcoded blueprint imports with `discover_providers()` +- Pass `auth_providers` to login template context +- Remove try/except blocks for individual auth modules + +### Phase 3: Update deploy.sh service discovery + +**deploy.sh changes:** +- Auto-discover and install `services/*/systemd/*.service` and `*.timer` +- Remove hardcoded service file paths +- Add enable/disable per instance.yaml config + +### Phase 4: Cleanup + +- Merge `dev_scripts/` into `scripts/` +- Sanitize `dev_docs/` (replace real IPs, hostnames, usernames with placeholders) +- Update CLAUDE.md, README.md, ARCHITECTURE.md +- Update MEMORY.md + +## Verification + +```bash +# 1. All tests pass +pytest tests/ connectors/ -v + +# 2. No server.telegram_bot imports remain +grep -rn "from server\.\(telegram_bot\|ws_gateway\|corporate_memory\)" . + +# 3. No hardcoded auth imports in app.py +grep -n "from.*auth import\|from.*password_auth" webapp/app.py + +# 4. Import smoke tests +python -c "from auth import discover_providers; print(f'{len(discover_providers())} providers')" +python -c "from services.telegram_bot.bot import TelegramBot; print('OK')" + +# 5. Service files discoverable +ls services/*/systemd/*.service services/*/systemd/*.timer +``` diff --git a/server/bin/collect-knowledge b/server/bin/collect-knowledge index dd85432..d10494b 100644 --- a/server/bin/collect-knowledge +++ b/server/bin/collect-knowledge @@ -28,4 +28,4 @@ if [[ -f "${REPO_DIR}/.env" ]]; then fi # Run the collector -exec "$VENV_PYTHON" -m server.corporate_memory.collector "$@" +exec "$VENV_PYTHON" -m services.corporate_memory "$@" diff --git a/server/bin/collect-sessions b/server/bin/collect-sessions index d41a1cf..9501a1f 100755 --- a/server/bin/collect-sessions +++ b/server/bin/collect-sessions @@ -13,4 +13,4 @@ cd "$REPO_DIR" # No environment variables needed - pure file operations # Run the collector -exec "$VENV_PYTHON" -m server.session_collector "$@" +exec "$VENV_PYTHON" -m services.session_collector "$@" diff --git a/server/deploy.sh b/server/deploy.sh index 2b5c424..7410d99 100755 --- a/server/deploy.sh +++ b/server/deploy.sh @@ -195,58 +195,30 @@ if command -v setfacl &>/dev/null; then log " ACL set for data-private group on private parquet directory" fi -# Deploy notification bot systemd service -log "Deploying notify-bot service..." -if [[ -f "${REPO_DIR}/server/notify-bot.service" ]]; then - sudo /usr/bin/cp "${REPO_DIR}/server/notify-bot.service" /etc/systemd/system/notify-bot.service +# Deploy systemd service and timer files from services/ and connectors/ +log "Deploying systemd service and timer files..." +SYSTEMD_CHANGED=false +for unit_file in "${REPO_DIR}"/services/*/systemd/*.service "${REPO_DIR}"/services/*/systemd/*.timer \ + "${REPO_DIR}"/connectors/*/systemd/*.service "${REPO_DIR}"/connectors/*/systemd/*.timer; do + if [[ -f "$unit_file" ]]; then + unit_name=$(basename "$unit_file") + sudo /usr/bin/cp "$unit_file" "/etc/systemd/system/${unit_name}" + log " Installed /etc/systemd/system/${unit_name}" + SYSTEMD_CHANGED=true + fi +done +if [[ "$SYSTEMD_CHANGED" == "true" ]]; then sudo /usr/bin/systemctl daemon-reload + log " systemd daemon-reload completed" fi -# Deploy WebSocket gateway systemd service -log "Deploying ws-gateway service..." -if [[ -f "${REPO_DIR}/server/ws-gateway.service" ]]; then - sudo /usr/bin/cp "${REPO_DIR}/server/ws-gateway.service" /etc/systemd/system/ws-gateway.service - sudo /usr/bin/systemctl daemon-reload -fi - -# Deploy corporate memory systemd service and timer -log "Deploying corporate-memory service and timer..." -if [[ -f "${REPO_DIR}/server/corporate-memory.service" ]]; then - sudo /usr/bin/cp "${REPO_DIR}/server/corporate-memory.service" /etc/systemd/system/corporate-memory.service - sudo /usr/bin/cp "${REPO_DIR}/server/corporate-memory.timer" /etc/systemd/system/corporate-memory.timer - sudo /usr/bin/systemctl daemon-reload -fi - -# Deploy Jira SLA polling systemd service and timer -log "Deploying jira-sla-poll service and timer..." -if [[ -f "${REPO_DIR}/connectors/jira/systemd/jira-sla-poll.service" ]]; then - sudo /usr/bin/cp "${REPO_DIR}/connectors/jira/systemd/jira-sla-poll.service" /etc/systemd/system/jira-sla-poll.service - sudo /usr/bin/cp "${REPO_DIR}/connectors/jira/systemd/jira-sla-poll.timer" /etc/systemd/system/jira-sla-poll.timer - sudo /usr/bin/systemctl daemon-reload -fi - -# Deploy Jira consistency monitoring systemd service and timers -log "Deploying jira-consistency service and timers..." -if [[ -f "${REPO_DIR}/connectors/jira/systemd/jira-consistency.service" ]]; then - sudo /usr/bin/cp "${REPO_DIR}/connectors/jira/systemd/jira-consistency.service" /etc/systemd/system/jira-consistency.service - sudo /usr/bin/cp "${REPO_DIR}/connectors/jira/systemd/jira-consistency.timer" /etc/systemd/system/jira-consistency.timer - sudo /usr/bin/cp "${REPO_DIR}/connectors/jira/systemd/jira-consistency-deep.timer" /etc/systemd/system/jira-consistency-deep.timer - sudo /usr/bin/systemctl daemon-reload - - # Create log file with correct permissions +# Post-install hooks for specific services +if [[ -f "/etc/systemd/system/jira-consistency.service" ]]; then sudo /usr/bin/touch /opt/data-analyst/logs/jira-consistency.log sudo /usr/bin/chown root:data-ops /opt/data-analyst/logs/jira-consistency.log sudo /usr/bin/chmod 664 /opt/data-analyst/logs/jira-consistency.log fi -# Deploy session collector systemd service and timer -log "Deploying session-collector service and timer..." -if [[ -f "${REPO_DIR}/server/session-collector.service" ]]; then - sudo /usr/bin/cp "${REPO_DIR}/server/session-collector.service" /etc/systemd/system/session-collector.service - sudo /usr/bin/cp "${REPO_DIR}/server/session-collector.timer" /etc/systemd/system/session-collector.timer - sudo /usr/bin/systemctl daemon-reload -fi - # Deploy example notification scripts to /data/examples log "Deploying example notification scripts..." sudo /usr/bin/mkdir -p /data/examples/notifications diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 0000000..56c396d --- /dev/null +++ b/services/__init__.py @@ -0,0 +1,13 @@ +""" +Services package - standalone optional services. + +Each service is a self-contained module with its own systemd unit files, +configuration, and README. Services are auto-discovered by deploy.sh +from services/*/systemd/*.service and *.timer. + +Available services: +- telegram_bot: Telegram notification bot +- ws_gateway: WebSocket real-time notification gateway +- corporate_memory: AI knowledge extraction from analyst insights +- session_collector: User session log collection +""" diff --git a/server/corporate_memory/__init__.py b/services/corporate_memory/__init__.py similarity index 100% rename from server/corporate_memory/__init__.py rename to services/corporate_memory/__init__.py diff --git a/services/corporate_memory/__main__.py b/services/corporate_memory/__main__.py new file mode 100644 index 0000000..b2a1e8f --- /dev/null +++ b/services/corporate_memory/__main__.py @@ -0,0 +1,7 @@ +"""Entry point: python -m services.corporate_memory""" + +import sys + +from .collector import main + +sys.exit(main()) diff --git a/server/corporate_memory/collector.py b/services/corporate_memory/collector.py similarity index 100% rename from server/corporate_memory/collector.py rename to services/corporate_memory/collector.py diff --git a/server/corporate_memory/prompts.py b/services/corporate_memory/prompts.py similarity index 100% rename from server/corporate_memory/prompts.py rename to services/corporate_memory/prompts.py diff --git a/server/corporate-memory.service b/services/corporate_memory/systemd/corporate-memory.service similarity index 100% rename from server/corporate-memory.service rename to services/corporate_memory/systemd/corporate-memory.service diff --git a/server/corporate-memory.timer b/services/corporate_memory/systemd/corporate-memory.timer similarity index 100% rename from server/corporate-memory.timer rename to services/corporate_memory/systemd/corporate-memory.timer diff --git a/services/session_collector/__init__.py b/services/session_collector/__init__.py new file mode 100644 index 0000000..b9c3c90 --- /dev/null +++ b/services/session_collector/__init__.py @@ -0,0 +1 @@ +"""Session collector service - collects user session logs.""" diff --git a/services/session_collector/__main__.py b/services/session_collector/__main__.py new file mode 100644 index 0000000..d2ab145 --- /dev/null +++ b/services/session_collector/__main__.py @@ -0,0 +1,7 @@ +"""Entry point: python -m services.session_collector""" + +import sys + +from .collector import main + +sys.exit(main()) diff --git a/server/session_collector.py b/services/session_collector/collector.py similarity index 100% rename from server/session_collector.py rename to services/session_collector/collector.py diff --git a/server/session-collector.service b/services/session_collector/systemd/session-collector.service similarity index 100% rename from server/session-collector.service rename to services/session_collector/systemd/session-collector.service diff --git a/server/session-collector.timer b/services/session_collector/systemd/session-collector.timer similarity index 100% rename from server/session-collector.timer rename to services/session_collector/systemd/session-collector.timer diff --git a/server/telegram_bot/__init__.py b/services/telegram_bot/__init__.py similarity index 100% rename from server/telegram_bot/__init__.py rename to services/telegram_bot/__init__.py diff --git a/server/telegram_bot/__main__.py b/services/telegram_bot/__main__.py similarity index 54% rename from server/telegram_bot/__main__.py rename to services/telegram_bot/__main__.py index 93795db..d0e4ef6 100644 --- a/server/telegram_bot/__main__.py +++ b/services/telegram_bot/__main__.py @@ -1,4 +1,4 @@ -"""Allow running as: python -m telegram_bot""" +"""Entry point: python -m services.telegram_bot""" import asyncio diff --git a/server/telegram_bot/bot.py b/services/telegram_bot/bot.py similarity index 100% rename from server/telegram_bot/bot.py rename to services/telegram_bot/bot.py diff --git a/server/telegram_bot/config.py b/services/telegram_bot/config.py similarity index 100% rename from server/telegram_bot/config.py rename to services/telegram_bot/config.py diff --git a/server/telegram_bot/dispatch.py b/services/telegram_bot/dispatch.py similarity index 100% rename from server/telegram_bot/dispatch.py rename to services/telegram_bot/dispatch.py diff --git a/server/telegram_bot/runner.py b/services/telegram_bot/runner.py similarity index 100% rename from server/telegram_bot/runner.py rename to services/telegram_bot/runner.py diff --git a/server/telegram_bot/sender.py b/services/telegram_bot/sender.py similarity index 100% rename from server/telegram_bot/sender.py rename to services/telegram_bot/sender.py diff --git a/server/telegram_bot/status.py b/services/telegram_bot/status.py similarity index 100% rename from server/telegram_bot/status.py rename to services/telegram_bot/status.py diff --git a/server/telegram_bot/storage.py b/services/telegram_bot/storage.py similarity index 100% rename from server/telegram_bot/storage.py rename to services/telegram_bot/storage.py diff --git a/server/notify-bot.service b/services/telegram_bot/systemd/notify-bot.service similarity index 91% rename from server/notify-bot.service rename to services/telegram_bot/systemd/notify-bot.service index 99df2be..6d6e82e 100644 --- a/server/notify-bot.service +++ b/services/telegram_bot/systemd/notify-bot.service @@ -8,7 +8,7 @@ Type=simple User=deploy Group=data-ops WorkingDirectory=/opt/data-analyst/repo -ExecStart=/opt/data-analyst/.venv/bin/python -m server.telegram_bot.bot +ExecStart=/opt/data-analyst/.venv/bin/python -m services.telegram_bot Restart=always RestartSec=10 diff --git a/server/telegram_bot/test_report.py b/services/telegram_bot/test_report.py similarity index 100% rename from server/telegram_bot/test_report.py rename to services/telegram_bot/test_report.py diff --git a/server/ws_gateway/__init__.py b/services/ws_gateway/__init__.py similarity index 100% rename from server/ws_gateway/__init__.py rename to services/ws_gateway/__init__.py diff --git a/server/ws_gateway/__main__.py b/services/ws_gateway/__main__.py similarity index 100% rename from server/ws_gateway/__main__.py rename to services/ws_gateway/__main__.py diff --git a/server/ws_gateway/auth.py b/services/ws_gateway/auth.py similarity index 100% rename from server/ws_gateway/auth.py rename to services/ws_gateway/auth.py diff --git a/server/ws_gateway/config.py b/services/ws_gateway/config.py similarity index 100% rename from server/ws_gateway/config.py rename to services/ws_gateway/config.py diff --git a/server/ws_gateway/gateway.py b/services/ws_gateway/gateway.py similarity index 100% rename from server/ws_gateway/gateway.py rename to services/ws_gateway/gateway.py diff --git a/server/ws-gateway.service b/services/ws_gateway/systemd/ws-gateway.service similarity index 86% rename from server/ws-gateway.service rename to services/ws_gateway/systemd/ws-gateway.service index 077d96a..19e812e 100644 --- a/server/ws-gateway.service +++ b/services/ws_gateway/systemd/ws-gateway.service @@ -7,7 +7,7 @@ Type=simple User=deploy Group=data-ops WorkingDirectory=/opt/data-analyst/repo -ExecStart=/opt/data-analyst/.venv/bin/python -m server.ws_gateway +ExecStart=/opt/data-analyst/.venv/bin/python -m services.ws_gateway Restart=always RestartSec=5 diff --git a/webapp/app.py b/webapp/app.py index e65f3bf..f20235a 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -636,7 +636,7 @@ def register_routes(app: Flask) -> None: def desktop_scripts(): """List notification scripts for the authenticated desktop user.""" username = require_desktop_auth() - from server.telegram_bot.status import get_script_list_structured + from services.telegram_bot.status import get_script_list_structured scripts = get_script_list_structured(username) return jsonify(scripts) @@ -649,8 +649,8 @@ def register_routes(app: Flask) -> None: if not script_name: return jsonify({"error": "Missing 'name' field"}), 400 - from server.telegram_bot.runner import run_user_script - from server.telegram_bot.dispatch import dispatch_to_ws_gateway + from services.telegram_bot.runner import run_user_script + from services.telegram_bot.dispatch import dispatch_to_ws_gateway output = run_user_script(username, script_name) if output is None: