AI-Cognitive-Leap/agnes-the-ai-analyst
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix data sync crash: make primary_key and sync_strategy optional

Browse source 
Remote-only tables (query_mode="remote") like daily_deal_traffic
don't need primary_key or sync_strategy. The parser used hard
lookups (table_data["primary_key"]) causing KeyError and breaking
all data sync since 2026-03-21.

Changes:
- TableConfig: default primary_key="" and sync_strategy="none"
- Parser: use .get() with defaults instead of [] lookups
- Validator: add "none" as valid sync_strategy
This commit is contained in:
Petr 2026-03-25 14:43:22 +01:00
parent 74ecf66f80
commit 4ebb3fc7b2
1 changed files with 8 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/config.py
View file

@ -75,8 +75,8 @@ class TableConfig:
id: Full table ID in Keboola (e.g., "in.c-sfdc.company") id: Full table ID in Keboola (e.g., "in.c-sfdc.company")
name: Short table name (e.g., "company") name: Short table name (e.g., "company")
description: Table description description: Table description
primary_key: Primary key column name primary_key: Primary key column name (optional for remote-only tables)
sync_strategy: "full_refresh", "incremental", or "partitioned" sync_strategy: "full_refresh", "incremental", "partitioned", or "none" (for remote-only tables)
incremental_window_days: Number of days to backtrack for incremental sync incremental_window_days: Number of days to backtrack for incremental sync
partition_by: Column name to partition by (for incremental/partitioned with partitions) partition_by: Column name to partition by (for incremental/partitioned with partitions)
partition_granularity: Partition granularity: "month", "day", or "year" partition_granularity: Partition granularity: "month", "day", or "year"
@ -92,8 +92,8 @@ class TableConfig:
id: str id: str
name: str name: str
description: str description: str
primary_key: str primary_key: str = "" # Optional for remote-only tables
sync_strategy: str # "full_refresh", "incremental", or "partitioned" sync_strategy: str = "none" # "full_refresh", "incremental", "partitioned", or "none" (remote-only)
incremental_window_days: Optional[int] = None incremental_window_days: Optional[int] = None
partition_by: Optional[str] = None partition_by: Optional[str] = None
partition_granularity: Optional[str] = None # "month", "day", "year" partition_granularity: Optional[str] = None # "month", "day", "year"
@ -132,10 +132,10 @@ class TableConfig:
) )
# Validate sync_strategy # Validate sync_strategy
if self.sync_strategy not in ["full_refresh", "incremental", "partitioned"]: if self.sync_strategy not in ["full_refresh", "incremental", "partitioned", "none"]:
raise ValueError( raise ValueError(
f"Invalid sync_strategy '{self.sync_strategy}' for table {self.id}. " f"Invalid sync_strategy '{self.sync_strategy}' for table {self.id}. "
f"Allowed values: 'full_refresh', 'incremental', 'partitioned'" f"Allowed values: 'full_refresh', 'incremental', 'partitioned', 'none'"
) )
# For incremental strategy: # For incremental strategy:
@ -467,8 +467,8 @@ class Config:
id=table_data["id"], id=table_data["id"],
name=table_data["name"], name=table_data["name"],
description=table_data["description"], description=table_data["description"],
primary_key=table_data["primary_key"], primary_key=table_data.get("primary_key", ""),
sync_strategy=table_data["sync_strategy"], sync_strategy=table_data.get("sync_strategy", "none"),
incremental_window_days=table_data.get("incremental_window_days"), incremental_window_days=table_data.get("incremental_window_days"),
partition_by=table_data.get("partition_by"), partition_by=table_data.get("partition_by"),
partition_granularity=table_data.get("partition_granularity"), partition_granularity=table_data.get("partition_granularity"),