First build

backend/app/connectors/clickhouse.py

@@ -0,0 +1,50 @@
+from clickhouse_driver import Client
+import os
+
+class ClickHouseConnector:
+    def __init__(self, host: str = None, port: int = 9000, user: str = 'default', password: str = '', database: str = 'default'):
+        self.host = host or os.getenv("CLICKHOUSE_HOST", "localhost")
+        self.port = port or int(os.getenv("CLICKHOUSE_PORT", 9000))
+        self.user = user or os.getenv("CLICKHOUSE_USER", "default")
+        self.password = password or os.getenv("CLICKHOUSE_PASSWORD", "")
+        self.database = database or os.getenv("CLICKHOUSE_DB", "default")
+        
+        self.client = Client(
+            host=self.host, 
+            port=self.port, 
+            user=self.user, 
+            password=self.password, 
+            database=self.database
+        )
+
+    def execute_query(self, query: str):
+        try:
+            return self.client.execute(query)
+        except Exception as e:
+            print(f"ClickHouse Query Error: {e}")
+            raise e
+
+    def get_schema(self):
+        query = "SELECT table, name, type FROM system.columns WHERE database = currentDatabase()"
+        try:
+            results = self.client.execute(query)
+            schema = {}
+            for row in results:
+                table = row[0]
+                if table not in schema:
+                    schema[table] = []
+                schema[table].append(f"{row[1]} ({row[2]})")
+            return schema
+        except Exception as e:
+            print(f"Error getting schema: {e}")
+            return {}
+
+    def test_connection(self) -> bool:
+        try:
+            self.client.execute("SELECT 1")
+            return True
+        except Exception as e:
+            print(f"ClickHouse Connection Error: {e}")
+            return False
+
+clickhouse_connector = ClickHouseConnector()

backend/app/connectors/minio.py

@@ -0,0 +1,51 @@
+from minio import Minio
+from minio.error import S3Error
+import os
+from typing import BinaryIO
+
+class MinioConnector:
+    def __init__(self):
+        self.endpoint = os.getenv("MINIO_ENDPOINT", "localhost:9000")
+        self.access_key = os.getenv("MINIO_ACCESS_KEY", "minioadmin")
+        self.secret_key = os.getenv("MINIO_SECRET_KEY", "minioadmin")
+        self.secure = os.getenv("MINIO_SECURE", "False").lower() == "true"
+        self.bucket_name = os.getenv("MINIO_BUCKET", "dataclaw")
+
+        self.client = Minio(
+            self.endpoint,
+            access_key=self.access_key,
+            secret_key=self.secret_key,
+            secure=self.secure
+        )
+        self._ensure_bucket_exists()
+
+    def _ensure_bucket_exists(self):
+        try:
+            if not self.client.bucket_exists(self.bucket_name):
+                self.client.make_bucket(self.bucket_name)
+        except S3Error as e:
+            print(f"MinIO Bucket Error: {e}")
+
+    def upload_file(self, object_name: str, file_data: BinaryIO, length: int, content_type: str = "application/octet-stream"):
+        try:
+            self.client.put_object(
+                self.bucket_name,
+                object_name,
+                file_data,
+                length,
+                content_type=content_type
+            )
+            return f"http{'s' if self.secure else ''}://{self.endpoint}/{self.bucket_name}/{object_name}"
+        except S3Error as e:
+            print(f"MinIO Upload Error: {e}")
+            raise e
+
+    def test_connection(self) -> bool:
+        try:
+            self.client.list_buckets()
+            return True
+        except Exception as e:
+            print(f"MinIO Connection Error: {e}")
+            return False
+
+minio_connector = MinioConnector()

backend/app/connectors/postgres.py

@@ -0,0 +1,53 @@
+from sqlalchemy import create_engine, text
+from sqlalchemy.orm import sessionmaker
+from typing import Generator
+import os
+
+class PostgresConnector:
+    def __init__(self, db_url: str = None):
+        self.db_url = db_url or os.getenv("POSTGRES_URL", "postgresql://user:password@localhost:5432/dbname")
+        self.engine = create_engine(self.db_url)
+        self.SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=self.engine)
+
+    def get_db(self) -> Generator:
+        db = self.SessionLocal()
+        try:
+            yield db
+        finally:
+            db.close()
+
+    def execute_query(self, query: str):
+        with self.engine.connect() as connection:
+            result = connection.execute(text(query))
+            return [dict(row._mapping) for row in result]
+
+    def get_schema(self):
+        query = """
+        SELECT table_name, column_name, data_type
+        FROM information_schema.columns
+        WHERE table_schema = 'public'
+        ORDER BY table_name, ordinal_position;
+        """
+        try:
+            results = self.execute_query(query)
+            schema = {}
+            for row in results:
+                table = row['table_name']
+                if table not in schema:
+                    schema[table] = []
+                schema[table].append(f"{row['column_name']} ({row['data_type']})")
+            return schema
+        except Exception as e:
+            print(f"Error getting schema: {e}")
+            return {}
+
+    def test_connection(self) -> bool:
+        try:
+            with self.engine.connect() as connection:
+                connection.execute(text("SELECT 1"))
+            return True
+        except Exception as e:
+            print(f"PostgreSQL Connection Error: {e}")
+            return False
+
+postgres_connector = PostgresConnector()