From e3f67d38f839be829033f5abd960e2fe37190fa0 Mon Sep 17 00:00:00 2001 From: qixinbo Date: Fri, 20 Mar 2026 16:02:22 +0800 Subject: [PATCH] feat: Add relationships to ER --- backend/app/api/semantic.py | 9 ++- backend/app/connectors/postgres.py | 59 +++++++++--------- backend/app/schemas/mdl.py | 2 +- backend/app/services/mdl.py | 55 ++++++++++++++-- backend/dataclaw.db | Bin 36864 -> 36864 bytes .../src/components/modeling/TableNode.tsx | 52 +++++++++++---- 6 files changed, 131 insertions(+), 46 deletions(-) diff --git a/backend/app/api/semantic.py b/backend/app/api/semantic.py index 3abebab..7963d7c 100644 --- a/backend/app/api/semantic.py +++ b/backend/app/api/semantic.py @@ -41,7 +41,14 @@ def get_semantic_schema(datasource_id: int, db: Session = Depends(get_db)): raise HTTPException(status_code=404, detail="DataSource not found") try: - return MDLService.get_raw_schema(ds) + raw_schema = MDLService.get_raw_schema(ds) + result = {} + for table, data in raw_schema.items(): + if isinstance(data, dict) and "columns" in data: + result[table] = data["columns"] + elif isinstance(data, list): + result[table] = data + return result except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/app/connectors/postgres.py b/backend/app/connectors/postgres.py index d406cdd..a89bbfd 100644 --- a/backend/app/connectors/postgres.py +++ b/backend/app/connectors/postgres.py @@ -22,41 +22,44 @@ class PostgresConnector: return [dict(row._mapping) for row in result] def get_schema(self): - if self.engine.dialect.name == "sqlite": - return self._get_sqlite_schema() - - query = """ - SELECT table_name, column_name, data_type - FROM information_schema.columns - WHERE table_schema = 'public' - ORDER BY table_name, ordinal_position; - """ - try: - results = self.execute_query(query) - schema = {} - for row in results: - table = row['table_name'] - if table not in schema: - schema[table] = [] - schema[table].append({"name": row['column_name'], "type": row['data_type']}) - return schema - except Exception as e: - print(f"Error getting schema: {e}") - return {} - - def _get_sqlite_schema(self): try: from sqlalchemy import inspect inspector = inspect(self.engine) schema = {} - for table_name in inspector.get_table_names(): + # Default schema for postgres is 'public', sqlite is None + schema_name = 'public' if self.engine.dialect.name == 'postgresql' else None + + for table_name in inspector.get_table_names(schema=schema_name): columns = [] - for col in inspector.get_columns(table_name): - columns.append({"name": col['name'], "type": str(col['type'])}) - schema[table_name] = columns + # get columns + for col in inspector.get_columns(table_name, schema=schema_name): + columns.append({ + "name": col['name'], + "type": str(col['type']) + }) + + # get primary key + pk_constraint = inspector.get_pk_constraint(table_name, schema=schema_name) + pks = pk_constraint.get('constrained_columns', []) if pk_constraint else [] + + # get foreign keys + fks = inspector.get_foreign_keys(table_name, schema=schema_name) + foreign_keys = [] + for fk in fks: + foreign_keys.append({ + "constrained_columns": fk['constrained_columns'], + "referred_table": fk['referred_table'], + "referred_columns": fk['referred_columns'] + }) + + schema[table_name] = { + "columns": columns, + "primary_keys": pks, + "foreign_keys": foreign_keys + } return schema except Exception as e: - print(f"Error getting SQLite schema: {e}") + print(f"Error getting schema: {e}") return {} def test_connection(self) -> bool: diff --git a/backend/app/schemas/mdl.py b/backend/app/schemas/mdl.py index 1f9dbe5..e593bf6 100644 --- a/backend/app/schemas/mdl.py +++ b/backend/app/schemas/mdl.py @@ -34,7 +34,7 @@ class Column(BaseModel): expression: Optional[str] = None isHidden: bool = False columnLevelAccessControl: Optional[ColumnAccessControl] = None - properties: Dict[str, str] = Field(default_factory=dict) + properties: Dict[str, Any] = Field(default_factory=dict) # Model Definitions class TableReference(BaseModel): diff --git a/backend/app/services/mdl.py b/backend/app/services/mdl.py index e43d5c0..8151560 100644 --- a/backend/app/services/mdl.py +++ b/backend/app/services/mdl.py @@ -34,10 +34,24 @@ class MDLService: raw_schema = MDLService.get_raw_schema(datasource) models = [] - for table_name, columns in raw_schema.items(): + relationships = [] + from app.schemas.mdl import Relationship + + # Helper to get columns for a table from the raw schema (which could be a list or a dict) + def get_table_info(t_name): + data = raw_schema.get(t_name, []) + if isinstance(data, dict) and "columns" in data: + return data + return {"columns": data, "primary_keys": [], "foreign_keys": []} + + for table_name in raw_schema.keys(): if selected_tables is not None and table_name not in selected_tables: continue + table_info = get_table_info(table_name) + columns = table_info["columns"] + pks = table_info.get("primary_keys", []) + model_cols = [] for col_info in columns: if isinstance(col_info, dict): @@ -65,7 +79,8 @@ class MDLService: if allowed and name not in allowed: continue - model_cols.append(Column(name=name, type=type_)) + is_pk = name in pks + model_cols.append(Column(name=name, type=type_, properties={"is_primary_key": is_pk})) if not model_cols: continue @@ -73,14 +88,46 @@ class MDLService: models.append(Model( name=table_name, tableReference=TableReference(table=table_name), - columns=model_cols + columns=model_cols, + primaryKey=pks[0] if pks else None )) + # Extract relationships from foreign keys + fks = table_info.get("foreign_keys", []) + for fk in fks: + referred_table = fk.get("referred_table") + if not referred_table: + continue + # Skip if the referred table is not selected + if selected_tables is not None and referred_table not in selected_tables: + continue + + constrained_cols = fk.get("constrained_columns", []) + referred_cols = fk.get("referred_columns", []) + + if len(constrained_cols) == 1 and len(referred_cols) == 1: + # Update column properties for FK + fk_col_name = constrained_cols[0] + for col in model_cols: + if col.name == fk_col_name: + col.properties["is_foreign_key"] = True + + # Simple single-column foreign key + condition = f"{table_name}.{constrained_cols[0]} = {referred_table}.{referred_cols[0]}" + rel_name = f"{table_name}_{constrained_cols[0]}_to_{referred_table}" + relationships.append(Relationship( + name=rel_name, + models=[table_name, referred_table], + joinType="MANY_TO_ONE", # typically a foreign key represents many-to-one + condition=condition + )) + return MDLManifest( catalog="default", schema="public", # Default schema, might need adjustment based on datasource config dataSource=datasource.type.upper(), - models=models + models=models, + relationships=relationships ) @staticmethod diff --git a/backend/dataclaw.db b/backend/dataclaw.db index 3a2895fb34a42e3f2f53c6238f36a8ba7b938e25..d4f54d4796bc6bc950078d58f2cd8bff7d47e602 100644 GIT binary patch delta 193 zcmZozz|^pSX@WGP*hCp;MzM_v3;ntID;b#hQW*F?^QCMSREXdcYvf_(WKfiBR5v!} zOinC{FD}fZ*9xoe{VqtD%mTYXIYhapap=)BEYNl(MVrmQ|Owv+}EG*3vjSO|2 zfqGmLOA?b3i&OPdl9XzV42;Zl4UBb-3=|A3tPBjSOw6zdP5vC$#=^kB&_1~~zJ;6r b4+9hb2L}Fc{2w+8COqMv{J~yeQGo&g*vvMq delta 57 zcmV-90LK4-paOuP0+1U46p { return ( - + { -
- {data.columns.map((col) => ( -
- {col.name} - {/* 类型列已被隐藏 */} -
- ))} +
+ + + {data.columns.map((col) => { + const isPk = col.properties?.is_primary_key; + const isFk = col.properties?.is_foreign_key; + let keyText = ""; + if (isPk && isFk) keyText = "PK, FK"; + else if (isPk) keyText = "PK"; + else if (isFk) keyText = "FK"; + + // Simplify type display, e.g., INTEGER -> int, CHARACTER VARYING -> string + let displayType = (col.type || "string").toLowerCase(); + if (displayType.includes("int")) displayType = "int"; + else if (displayType.includes("char") || displayType.includes("text")) displayType = "string"; + else if (displayType.includes("time") || displayType.includes("date")) displayType = "date"; + else if (displayType.includes("bool")) displayType = "boolean"; + else if (displayType.includes("float") || displayType.includes("double") || displayType.includes("numeric") || displayType.includes("decimal")) displayType = "float"; + + return ( + + + + + + ); + })} + +
{displayType}{col.name} + {keyText} +