WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

skills/preloaded/data-processor/scripts/analyze.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""
+数据分析脚本 - 用于分析 RAG 检索结果和知识库数据
+
+支持功能:
+- 基本统计（计数、求和、平均值、最大/最小值）
+- 数值分布分析
+- 文本统计（词频、字符数）
+
+用法:
+    # 通过 stdin 传入 JSON 数据
+    echo '{"items": [1, 2, 3, 4, 5]}' | python analyze.py
+    
+    # 通过参数传入文件
+    python analyze.py --file data.json
+    
+    # 指定分析类型
+    echo '{"items": [1, 2, 3]}' | python analyze.py --type numeric
+"""
+
+import sys
+import json
+import argparse
+from collections import Counter
+
+
+def analyze_numeric(data: list) -> dict:
+    """分析数值数据"""
+    if not data:
+        return {"error": "空数据集"}
+    
+    # 过滤出数值
+    numbers = [x for x in data if isinstance(x, (int, float))]
+    if not numbers:
+        return {"error": "无有效数值数据"}
+    
+    numbers.sort()
+    n = len(numbers)
+    
+    result = {
+        "count": n,
+        "sum": sum(numbers),
+        "mean": sum(numbers) / n,
+        "min": min(numbers),
+        "max": max(numbers),
+        "median": numbers[n // 2] if n % 2 == 1 else (numbers[n // 2 - 1] + numbers[n // 2]) / 2,
+    }
+    
+    # 计算标准差
+    mean = result["mean"]
+    variance = sum((x - mean) ** 2 for x in numbers) / n
+    result["std_dev"] = variance ** 0.5
+    
+    # 分布统计
+    if n >= 5:
+        result["quartiles"] = {
+            "q1": numbers[n // 4],
+            "q2": result["median"],
+            "q3": numbers[3 * n // 4]
+        }
+    
+    return result
+
+
+def analyze_text(data: list) -> dict:
+    """分析文本数据"""
+    if not data:
+        return {"error": "空数据集"}
+    
+    texts = [str(x) for x in data if x]
+    
+    # 基本统计
+    total_chars = sum(len(t) for t in texts)
+    total_words = sum(len(t.split()) for t in texts)
+    
+    # 词频统计（简单分词）
+    all_words = []
+    for text in texts:
+        words = text.split()
+        all_words.extend(w.strip('.,!?;:""\'()[]{}') for w in words if w.strip())
+    
+    word_freq = Counter(all_words)
+    
+    result = {
+        "count": len(texts),
+        "total_chars": total_chars,
+        "total_words": total_words,
+        "avg_chars_per_item": total_chars / len(texts) if texts else 0,
+        "avg_words_per_item": total_words / len(texts) if texts else 0,
+        "top_words": dict(word_freq.most_common(10)),
+        "unique_words": len(word_freq)
+    }
+    
+    return result
+
+
+def analyze_mixed(data: list) -> dict:
+    """分析混合数据"""
+    if not data:
+        return {"error": "空数据集"}
+    
+    # 类型统计
+    type_counts = Counter(type(x).__name__ for x in data)
+    
+    result = {
+        "total_items": len(data),
+        "type_distribution": dict(type_counts),
+    }
+    
+    # 分别分析数值和文本
+    numbers = [x for x in data if isinstance(x, (int, float))]
+    texts = [x for x in data if isinstance(x, str)]
+    
+    if numbers:
+        result["numeric_analysis"] = analyze_numeric(numbers)
+    if texts:
+        result["text_analysis"] = analyze_text(texts)
+    
+    return result
+
+
+def analyze_dict_list(data: list) -> dict:
+    """分析字典列表（如数据库查询结果）"""
+    if not data:
+        return {"error": "空数据集"}
+    
+    if not all(isinstance(x, dict) for x in data):
+        return {"error": "数据格式不正确，需要字典列表"}
+    
+    result = {
+        "record_count": len(data),
+        "fields": {},
+    }
+    
+    # 获取所有字段
+    all_keys = set()
+    for item in data:
+        all_keys.update(item.keys())
+    
+    # 分析每个字段
+    for key in all_keys:
+        values = [item.get(key) for item in data if key in item]
+        
+        # 判断字段类型
+        non_null_values = [v for v in values if v is not None]
+        if not non_null_values:
+            result["fields"][key] = {"type": "all_null", "null_count": len(values)}
+            continue
+        
+        sample = non_null_values[0]
+        if isinstance(sample, (int, float)):
+            field_analysis = analyze_numeric(non_null_values)
+            field_analysis["type"] = "numeric"
+        elif isinstance(sample, str):
+            field_analysis = analyze_text(non_null_values)
+            field_analysis["type"] = "text"
+        else:
+            field_analysis = {"type": type(sample).__name__, "count": len(non_null_values)}
+        
+        field_analysis["null_count"] = len(values) - len(non_null_values)
+        result["fields"][key] = field_analysis
+    
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description="数据分析工具")
+    parser.add_argument("--file", "-f", help="输入文件路径")
+    parser.add_argument("--type", "-t", choices=["numeric", "text", "mixed", "auto"],
+                       default="auto", help="分析类型")
+    parser.add_argument("--pretty", "-p", action="store_true", help="格式化输出")
+    args = parser.parse_args()
+    
+    # 读取输入
+    try:
+        if args.file:
+            with open(args.file, 'r', encoding='utf-8') as f:
+                raw_data = f.read()
+        else:
+            raw_data = sys.stdin.read()
+        
+        if not raw_data.strip():
+            print(json.dumps({"error": "空输入"}))
+            return
+        
+        data = json.loads(raw_data)
+    except json.JSONDecodeError as e:
+        print(json.dumps({"error": f"JSON 解析错误: {str(e)}"}))
+        return
+    except FileNotFoundError:
+        print(json.dumps({"error": f"文件未找到: {args.file}"}))
+        return
+    except Exception as e:
+        print(json.dumps({"error": f"读取错误: {str(e)}"}))
+        return
+    
+    # 提取数据
+    items = None
+    if isinstance(data, dict):
+        if "items" in data:
+            items = data["items"]
+        elif "data" in data:
+            items = data["data"]
+        elif "results" in data:
+            items = data["results"]
+        else:
+            # 假设整个 dict 是单条记录，包装成列表
+            items = [data]
+    elif isinstance(data, list):
+        items = data
+    else:
+        print(json.dumps({"error": "不支持的数据格式，需要列表或包含 items/data/results 的字典"}))
+        return
+    
+    # 根据类型分析
+    if args.type == "auto":
+        # 自动检测
+        if items and all(isinstance(x, dict) for x in items):
+            result = analyze_dict_list(items)
+        elif items and all(isinstance(x, (int, float)) for x in items):
+            result = analyze_numeric(items)
+        elif items and all(isinstance(x, str) for x in items):
+            result = analyze_text(items)
+        else:
+            result = analyze_mixed(items)
+    elif args.type == "numeric":
+        result = analyze_numeric(items)
+    elif args.type == "text":
+        result = analyze_text(items)
+    else:
+        result = analyze_mixed(items)
+    
+    # 添加元信息
+    output = {
+        "success": True,
+        "analysis": result,
+        "metadata": {
+            "input_type": type(data).__name__,
+            "item_count": len(items) if items else 0,
+            "analysis_type": args.type
+        }
+    }
+    
+    # 输出
+    indent = 2 if args.pretty else None
+    print(json.dumps(output, ensure_ascii=False, indent=indent))
+
+
+if __name__ == "__main__":
+    main()

skills/preloaded/data-processor/scripts/extract_info.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""
+信息提取脚本 - 从文本中提取结构化信息
+
+提取内容:
+- 数字
+- 日期
+- 百分比
+- 金额
+- 邮箱
+- URL
+- 电话号码
+
+用法:
+    echo "2024年销售额为100万元，同比增长15%" | python extract_info.py
+    
+    # 指定提取类型
+    echo "联系我: test@example.com 或 13800138000" | python extract_info.py --types email,phone
+"""
+
+import sys
+import json
+import argparse
+import re
+
+
+def extract_numbers(text: str) -> list:
+    """提取数字"""
+    # 匹配整数和小数
+    pattern = r'-?\d+(?:\.\d+)?'
+    numbers = re.findall(pattern, text)
+    # 转换为数值
+    result = []
+    for n in numbers:
+        try:
+            if '.' in n:
+                result.append(float(n))
+            else:
+                result.append(int(n))
+        except ValueError:
+            result.append(n)
+    return result
+
+
+def extract_dates(text: str) -> list:
+    """提取日期"""
+    patterns = [
+        r'\d{4}[-/年]\d{1,2}[-/月]\d{1,2}[日]?',  # 2024-01-01 或 2024年1月1日
+        r'\d{4}[-/年]\d{1,2}[月]?',                 # 2024-01 或 2024年1月
+        r'\d{4}年',                                  # 2024年
+        r'\d{1,2}[-/月]\d{1,2}[日]?',              # 01-01 或 1月1日
+    ]
+    
+    dates = []
+    for pattern in patterns:
+        matches = re.findall(pattern, text)
+        dates.extend(matches)
+    
+    return list(set(dates))
+
+
+def extract_percentages(text: str) -> list:
+    """提取百分比"""
+    pattern = r'-?\d+(?:\.\d+)?%'
+    return re.findall(pattern, text)
+
+
+def extract_amounts(text: str) -> list:
+    """提取金额"""
+    patterns = [
+        r'[¥$€£]\s*\d+(?:,\d{3})*(?:\.\d+)?',      # ¥100.00
+        r'\d+(?:,\d{3})*(?:\.\d+)?\s*[元万亿美金]', # 100万元
+        r'\d+(?:\.\d+)?[百千万亿]+[元]?',            # 100万
+    ]
+    
+    amounts = []
+    for pattern in patterns:
+        matches = re.findall(pattern, text)
+        amounts.extend(matches)
+    
+    return list(set(amounts))
+
+
+def extract_emails(text: str) -> list:
+    """提取邮箱"""
+    pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
+    return re.findall(pattern, text)
+
+
+def extract_urls(text: str) -> list:
+    """提取 URL"""
+    pattern = r'https?://[^\s<>"{}|\\^`\[\]]+'
+    return re.findall(pattern, text)
+
+
+def extract_phones(text: str) -> list:
+    """提取电话号码"""
+    patterns = [
+        r'1[3-9]\d{9}',                           # 手机号
+        r'\d{3,4}[-\s]?\d{7,8}',                  # 固话
+        r'\+\d{1,3}[-\s]?\d{10,12}',             # 国际号码
+    ]
+    
+    phones = []
+    for pattern in patterns:
+        matches = re.findall(pattern, text)
+        phones.extend(matches)
+    
+    return list(set(phones))
+
+
+def extract_keywords(text: str, min_len: int = 2) -> list:
+    """提取关键词（中文和英文）"""
+    # 中文关键词
+    chinese_pattern = r'[\u4e00-\u9fa5]{2,}'
+    chinese_words = re.findall(chinese_pattern, text)
+    
+    # 英文关键词
+    english_pattern = r'[a-zA-Z]{3,}'
+    english_words = re.findall(english_pattern, text)
+    
+    # 统计词频
+    from collections import Counter
+    words = chinese_words + [w.lower() for w in english_words]
+    
+    # 过滤停用词
+    stopwords = {'的', '是', '在', '了', '和', '与', '或', '为', '有', '这', '那', '等',
+                 'the', 'is', 'are', 'was', 'were', 'and', 'or', 'for', 'with', 'this'}
+    words = [w for w in words if w not in stopwords and len(w) >= min_len]
+    
+    word_freq = Counter(words)
+    return [{"word": w, "count": c} for w, c in word_freq.most_common(20)]
+
+
+def main():
+    parser = argparse.ArgumentParser(description="信息提取工具")
+    parser.add_argument("--types", "-t", 
+                       help="要提取的类型，逗号分隔 (numbers,dates,percentages,amounts,emails,urls,phones,keywords)")
+    parser.add_argument("--pretty", "-p", action="store_true", help="格式化输出")
+    args = parser.parse_args()
+    
+    # 读取输入
+    try:
+        text = sys.stdin.read()
+        if not text.strip():
+            print(json.dumps({"error": "空输入"}))
+            return
+    except Exception as e:
+        print(json.dumps({"error": f"读取错误: {str(e)}"}))
+        return
+    
+    # 确定要提取的类型
+    all_types = ["numbers", "dates", "percentages", "amounts", "emails", "urls", "phones", "keywords"]
+    if args.types:
+        extract_types = [t.strip().lower() for t in args.types.split(",")]
+    else:
+        extract_types = all_types
+    
+    # 提取信息
+    result = {
+        "success": True,
+        "text_length": len(text),
+        "extracted": {}
+    }
+    
+    extractors = {
+        "numbers": extract_numbers,
+        "dates": extract_dates,
+        "percentages": extract_percentages,
+        "amounts": extract_amounts,
+        "emails": extract_emails,
+        "urls": extract_urls,
+        "phones": extract_phones,
+        "keywords": extract_keywords,
+    }
+    
+    for ext_type in extract_types:
+        if ext_type in extractors:
+            try:
+                extracted = extractors[ext_type](text)
+                if extracted:
+                    result["extracted"][ext_type] = extracted
+            except Exception as e:
+                result["extracted"][ext_type] = {"error": str(e)}
+    
+    # 统计
+    result["summary"] = {
+        "total_extractions": sum(len(v) if isinstance(v, list) else 0 
+                                  for v in result["extracted"].values()),
+        "types_found": list(result["extracted"].keys())
+    }
+    
+    # 输出
+    indent = 2 if args.pretty else None
+    print(json.dumps(result, ensure_ascii=False, indent=indent))
+
+
+if __name__ == "__main__":
+    main()

skills/preloaded/data-processor/scripts/format_converter.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+格式转换脚本 - JSON/CSV/Markdown 相互转换
+
+用法:
+    # JSON 转 CSV
+    echo '[{"name": "A", "value": 1}]' | python format_converter.py --to csv
+    
+    # JSON 转 Markdown 表格
+    echo '[{"name": "A", "value": 1}]' | python format_converter.py --to markdown
+    
+    # CSV 转 JSON
+    cat data.csv | python format_converter.py --from csv --to json
+"""
+
+import sys
+import json
+import argparse
+import csv
+import io
+
+
+def json_to_csv(data: list) -> str:
+    """将 JSON 列表转换为 CSV"""
+    if not data:
+        return ""
+    
+    if not all(isinstance(x, dict) for x in data):
+        raise ValueError("JSON 数据必须是字典列表")
+    
+    # 获取所有字段
+    fieldnames = []
+    for item in data:
+        for key in item.keys():
+            if key not in fieldnames:
+                fieldnames.append(key)
+    
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=fieldnames)
+    writer.writeheader()
+    writer.writerows(data)
+    
+    return output.getvalue()
+
+
+def csv_to_json(csv_text: str) -> list:
+    """将 CSV 转换为 JSON 列表"""
+    reader = csv.DictReader(io.StringIO(csv_text))
+    return list(reader)
+
+
+def json_to_markdown(data: list) -> str:
+    """将 JSON 列表转换为 Markdown 表格"""
+    if not data:
+        return ""
+    
+    if not all(isinstance(x, dict) for x in data):
+        raise ValueError("JSON 数据必须是字典列表")
+    
+    # 获取所有字段
+    fieldnames = []
+    for item in data:
+        for key in item.keys():
+            if key not in fieldnames:
+                fieldnames.append(key)
+    
+    # 构建表头
+    lines = []
+    lines.append("| " + " | ".join(fieldnames) + " |")
+    lines.append("| " + " | ".join(["---"] * len(fieldnames)) + " |")
+    
+    # 构建数据行
+    for item in data:
+        row = []
+        for field in fieldnames:
+            value = item.get(field, "")
+            # 转义 Markdown 特殊字符
+            str_value = str(value) if value is not None else ""
+            str_value = str_value.replace("|", "\\|")
+            row.append(str_value)
+        lines.append("| " + " | ".join(row) + " |")
+    
+    return "\n".join(lines)
+
+
+def markdown_to_json(md_text: str) -> list:
+    """将 Markdown 表格转换为 JSON 列表"""
+    lines = [line.strip() for line in md_text.strip().split("\n") if line.strip()]
+    
+    if len(lines) < 2:
+        raise ValueError("无效的 Markdown 表格")
+    
+    # 解析表头
+    header_line = lines[0]
+    if not header_line.startswith("|"):
+        raise ValueError("无效的 Markdown 表格格式")
+    
+    headers = [h.strip() for h in header_line.strip("|").split("|")]
+    
+    # 跳过分隔行
+    data_lines = lines[2:] if len(lines) > 2 else []
+    
+    # 解析数据
+    result = []
+    for line in data_lines:
+        if not line.startswith("|"):
+            continue
+        values = [v.strip() for v in line.strip("|").split("|")]
+        item = {}
+        for i, header in enumerate(headers):
+            if i < len(values):
+                item[header] = values[i]
+        result.append(item)
+    
+    return result
+
+
+def detect_format(text: str) -> str:
+    """自动检测输入格式"""
+    text = text.strip()
+    
+    if text.startswith("[") or text.startswith("{"):
+        return "json"
+    elif text.startswith("|"):
+        return "markdown"
+    elif "," in text.split("\n")[0]:
+        return "csv"
+    else:
+        return "unknown"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="数据格式转换工具")
+    parser.add_argument("--from", "-f", dest="from_format", 
+                       choices=["json", "csv", "markdown", "auto"],
+                       default="auto", help="输入格式")
+    parser.add_argument("--to", "-t", dest="to_format",
+                       choices=["json", "csv", "markdown"],
+                       required=True, help="输出格式")
+    parser.add_argument("--pretty", "-p", action="store_true", help="格式化输出")
+    args = parser.parse_args()
+    
+    # 读取输入
+    try:
+        raw_input = sys.stdin.read()
+        if not raw_input.strip():
+            print(json.dumps({"error": "空输入"}))
+            return
+    except Exception as e:
+        print(json.dumps({"error": f"读取错误: {str(e)}"}))
+        return
+    
+    # 检测输入格式
+    from_format = args.from_format
+    if from_format == "auto":
+        from_format = detect_format(raw_input)
+        if from_format == "unknown":
+            print(json.dumps({"error": "无法自动检测输入格式"}))
+            return
+    
+    # 转换为中间格式（JSON 列表）
+    try:
+        if from_format == "json":
+            data = json.loads(raw_input)
+            if isinstance(data, dict):
+                # 尝试提取列表
+                if "items" in data:
+                    data = data["items"]
+                elif "data" in data:
+                    data = data["data"]
+                else:
+                    data = [data]
+            if not isinstance(data, list):
+                data = [data]
+        elif from_format == "csv":
+            data = csv_to_json(raw_input)
+        elif from_format == "markdown":
+            data = markdown_to_json(raw_input)
+        else:
+            print(json.dumps({"error": f"不支持的输入格式: {from_format}"}))
+            return
+    except Exception as e:
+        print(json.dumps({"error": f"解析输入失败: {str(e)}"}))
+        return
+    
+    # 转换为目标格式
+    try:
+        if args.to_format == "json":
+            indent = 2 if args.pretty else None
+            output = json.dumps(data, ensure_ascii=False, indent=indent)
+        elif args.to_format == "csv":
+            output = json_to_csv(data)
+        elif args.to_format == "markdown":
+            output = json_to_markdown(data)
+        else:
+            print(json.dumps({"error": f"不支持的输出格式: {args.to_format}"}))
+            return
+        
+        print(output)
+    except Exception as e:
+        print(json.dumps({"error": f"转换失败: {str(e)}"}))
+        return
+
+
+if __name__ == "__main__":
+    main()