diff --git a/connectors/openmetadata/transformer.py b/connectors/openmetadata/transformer.py index b426357..b66cb3f 100644 --- a/connectors/openmetadata/transformer.py +++ b/connectors/openmetadata/transformer.py @@ -103,10 +103,13 @@ def extract_expression(raw_metric: Dict[str, Any]) -> str: metric_expr = raw_metric.get("metricExpression", {}) if isinstance(metric_expr, dict): # OpenMetadata uses "code" field for the SQL expression - return metric_expr.get("code", "") or metric_expr.get("expression", "") or "" - if isinstance(metric_expr, str): + result = metric_expr.get("code", "") or metric_expr.get("expression", "") or "" + if result: + return result + elif isinstance(metric_expr, str) and metric_expr: return metric_expr - return "" + # Fallback: top-level expression field (OpenMetadata format varies) + return raw_metric.get("expression", "") or "" def extract_owners(raw: Dict[str, Any]) -> List[str]: diff --git a/server/sudoers-deploy b/server/sudoers-deploy index f8d225f..18a90fd 100644 --- a/server/sudoers-deploy +++ b/server/sudoers-deploy @@ -46,6 +46,7 @@ deploy ALL=(ALL) NOPASSWD: /usr/bin/mkdir -p /data/scripts deploy ALL=(ALL) NOPASSWD: /usr/bin/cp /opt/data-analyst/repo/scripts/* /data/scripts/* deploy ALL=(ALL) NOPASSWD: /usr/bin/chmod -R 755 /data/scripts deploy ALL=(ALL) NOPASSWD: /usr/bin/chown -R deploy\:data-ops /data/scripts +deploy ALL=(ALL) NOPASSWD: /usr/bin/chown -R root\:data-ops /data/scripts # Allow deploy user to manage documentation in /data/docs deploy ALL=(ALL) NOPASSWD: /usr/bin/mkdir -p /data/docs @@ -54,10 +55,12 @@ deploy ALL=(ALL) NOPASSWD: /usr/bin/cp /opt/data-analyst/repo/docs/* /data/docs/ deploy ALL=(ALL) NOPASSWD: /usr/bin/cp -r /opt/data-analyst/repo/docs/* /data/docs/ deploy ALL=(ALL) NOPASSWD: /usr/bin/chmod -R 775 /data/docs deploy ALL=(ALL) NOPASSWD: /usr/bin/chown -R deploy\:data-ops /data/docs +deploy ALL=(ALL) NOPASSWD: /usr/bin/chown -R root\:data-ops /data/docs # Allow deploy user to manage notifications directory deploy ALL=(ALL) NOPASSWD: /usr/bin/mkdir -p /data/notifications deploy ALL=(ALL) NOPASSWD: /usr/bin/chown deploy\:data-ops /data/notifications +deploy ALL=(ALL) NOPASSWD: /usr/bin/chown root\:data-ops /data/notifications deploy ALL=(ALL) NOPASSWD: /usr/bin/chmod 2770 /data/notifications # Allow deploy user to manage notify-bot service @@ -86,10 +89,12 @@ deploy ALL=(ALL) NOPASSWD: /usr/bin/cp /opt/data-analyst/repo/server/limits-user deploy ALL=(ALL) NOPASSWD: /usr/bin/chmod 644 /etc/security/limits.d/99-users.conf # Allow deploy user to manage example notification scripts +deploy ALL=(ALL) NOPASSWD: /usr/bin/mkdir -p /data/examples deploy ALL=(ALL) NOPASSWD: /usr/bin/mkdir -p /data/examples/notifications deploy ALL=(ALL) NOPASSWD: /usr/bin/cp /opt/data-analyst/repo/examples/notifications/* /data/examples/notifications/* deploy ALL=(ALL) NOPASSWD: /usr/bin/chmod -R 755 /data/examples deploy ALL=(ALL) NOPASSWD: /usr/bin/chown -R deploy\:data-ops /data/examples +deploy ALL=(ALL) NOPASSWD: /usr/bin/chown -R root\:data-ops /data/examples # Allow deploy user to manage Jira data directory deploy ALL=(ALL) NOPASSWD: /usr/bin/mkdir -p /data/src_data/raw/jira/* @@ -104,6 +109,7 @@ deploy ALL=(ALL) NOPASSWD: /usr/bin/chmod 2770 /data/auth # Allow deploy user to manage corporate memory directory and service deploy ALL=(ALL) NOPASSWD: /usr/bin/mkdir -p /data/corporate-memory deploy ALL=(ALL) NOPASSWD: /usr/bin/chown deploy\:data-ops /data/corporate-memory +deploy ALL=(ALL) NOPASSWD: /usr/bin/chown root\:data-ops /data/corporate-memory deploy ALL=(ALL) NOPASSWD: /usr/bin/chmod 2770 /data/corporate-memory deploy ALL=(ALL) NOPASSWD: /usr/bin/cp /opt/data-analyst/repo/services/corporate_memory/systemd/corporate-memory.service /etc/systemd/system/corporate-memory.service deploy ALL=(ALL) NOPASSWD: /usr/bin/cp /opt/data-analyst/repo/services/corporate_memory/systemd/corporate-memory.timer /etc/systemd/system/corporate-memory.timer diff --git a/tests/test_deploy_guard.py b/tests/test_deploy_guard.py index 5e681d4..544c092 100644 --- a/tests/test_deploy_guard.py +++ b/tests/test_deploy_guard.py @@ -536,12 +536,12 @@ class TestFileOwnership: # Explicit list of critical directories and their expected ownership. # Maintained manually - extend when new critical directories are added. CRITICAL_DIRS = { - "/data/scripts": {"owner": "deploy", "group": "data-ops"}, - "/data/docs": {"owner": "deploy", "group": "data-ops"}, - "/data/examples": {"owner": "deploy", "group": "data-ops"}, - "/data/notifications": {"owner": "deploy", "group": "data-ops"}, + "/data/scripts": {"owner": "root", "group": "data-ops"}, + "/data/docs": {"owner": "root", "group": "data-ops"}, + "/data/examples": {"owner": "root", "group": "data-ops"}, + "/data/notifications": {"owner": "root", "group": "data-ops"}, "/data/auth": {"owner": "www-data", "group": "data-ops"}, - "/data/corporate-memory": {"owner": "deploy", "group": "data-ops"}, + "/data/corporate-memory": {"owner": "root", "group": "data-ops"}, "/data/user_sessions": {"owner": "root", "group": "data-ops"}, "/data/src_data/raw/jira": {"owner": "root", "group": "data-ops"}, "/opt/data-analyst": {"owner": "root", "group": "data-ops"}, @@ -675,12 +675,15 @@ class TestSymlinksAndPaths: # Find cp commands copying from ${REPO_DIR}/ or repo-relative paths # Pattern: cp ... ${REPO_DIR}/path or "${REPO_DIR}/path" cp_sources = re.findall( - r'cp\s+(?:-r\s+)?"?\$\{REPO_DIR\}/([^"}\s]+)', + r'cp\s+(?:-r\s+)?"?\$\{REPO_DIR\}/([^"}\s$]+)', deploy_content, ) missing = [] for rel_path in cp_sources: + # Skip shell variable expansions (e.g., loop vars like ${script_file}) + if "${" in rel_path or "$" in rel_path: + continue # Handle glob patterns (e.g., examples/notifications/*.py) if "*" in rel_path: # Check the directory exists diff --git a/tests/test_sync_data.py b/tests/test_sync_data.py index 3619bc5..580018d 100644 --- a/tests/test_sync_data.py +++ b/tests/test_sync_data.py @@ -344,7 +344,7 @@ class TestSyncScriptReliability: if "command -v rsync" in line: continue # Skip self-update rsync with scp fallback (sync_data.sh only) - if "data-analyst:server/scripts/" in line: + if "server/scripts/" in line and script_path.name == "sync_data.sh": continue # Everything else must use rsync_reliable, not bare rsync if line.startswith("rsync ") or re.match(r'^rsync\s', line): diff --git a/webapp/app.py b/webapp/app.py index 4a4de96..6efb011 100644 --- a/webapp/app.py +++ b/webapp/app.py @@ -895,6 +895,9 @@ def _build_om_metric_detail(raw_metric: dict) -> dict: expression = metric_expr.get("expression", "") or "" elif isinstance(metric_expr, str): expression = metric_expr + # Fallback: top-level expression field (OpenMetadata format varies) + if not expression: + expression = raw_metric.get("expression", "") or "" metric_type = raw_metric.get("metricType", "") or "" unit = raw_metric.get("unitOfMeasurement", "") or ""