diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e69de29 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..6b035ee --- /dev/null +++ b/.env.example @@ -0,0 +1,210 @@ +# 使用说明 +# 1. 复制此文件为 .env +# 2. 替换所有占位符为实际值 +# 3. 确保 .env 文件不会被提交到版本控制系统 + +# gin mod +# 可选值: debug(开发模式，有详细日志), release(生产模式) +GIN_MODE=debug + +# Ollama 服务的基准 URL，用于连接本地/其他服务器上运行的 Ollama 服务 +OLLAMA_BASE_URL=http://host.docker.internal:11434 + +# 存储配置 +# 主数据库类型(postgres/mysql) +DB_DRIVER=postgres + +# 向量存储类型(postgres/elasticsearch_v7/elasticsearch_v8) +RETRIEVE_DRIVER=postgres + +# 文件存储类型(local/minio/cos) +STORAGE_TYPE=local + +# 流处理后端(memory/redis) +STREAM_MANAGER_TYPE=redis + +# 主数据库配置 +# 数据库端口，默认为5432 +DB_PORT=5432 + +# 应用服务端口，默认为8080 +APP_PORT=8080 + +# 前端服务端口，默认为80 +FRONTEND_PORT=80 + +# 文档解析模块端口，默认为50051 +DOCREADER_PORT=50051 + +# 数据库用户名 +DB_USER=postgres + +# 数据库密码 +DB_PASSWORD=postgres123!@# + +# 数据库名称 +DB_NAME=WeKnora + +# 如果使用 redis 作为流处理后端，需要配置以下参数 +# Redis端口，默认为6379 +REDIS_PORT=6379 + +# Redis密码，如果没有设置密码，可以留空 +REDIS_PASSWORD=redis123!@# + +# Redis数据库索引，默认为0 +REDIS_DB=0 + +# Redis key的前缀，用于命名空间隔离 +REDIS_PREFIX=stream: + +# 当使用本地存储时，文件保存的基础目录路径 +LOCAL_STORAGE_BASE_DIR=./data/files + +TENANT_AES_KEY=weknorarag-api-key-secret-secret + +# 是否开启知识图谱构建和检索（构建阶段需调用大模型，耗时较长） +ENABLE_GRAPH_RAG=false + +MINIO_PORT=9000 + +MINIO_CONSOLE_PORT=9001 + +# Embedding并发数，出现429错误时，可调小此参数 +CONCURRENCY_POOL_SIZE=5 + +# 如果使用ElasticSearch作为向量存储，需要配置以下参数 +# ElasticSearch地址，例如 http://localhost:9200 +# ELASTICSEARCH_ADDR=your_elasticsearch_addr + +# ElasticSearch用户名，如果需要身份验证 +# ELASTICSEARCH_USERNAME=your_elasticsearch_username + +# ElasticSearch密码，如果需要身份验证 +# ELASTICSEARCH_PASSWORD=your_elasticsearch_password + +# ElasticSearch索引名称，用于存储向量数据 +# ELASTICSEARCH_INDEX=WeKnora + +# 如果使用MinIO作为文件存储，需要配置以下参数 +# MinIO访问密钥 +# MINIO_ACCESS_KEY_ID=your_minio_access_key + +# MinIO密钥 +# MINIO_SECRET_ACCESS_KEY=your_minio_secret_key + +# MinIO桶名称，用于存储文件 +# MINIO_BUCKET_NAME=your_minio_bucket_name + +# 如果使用腾讯云COS作为文件存储，需要配置以下参数 +# 腾讯云COS的访问密钥ID +# COS_SECRET_ID=your_cos_secret_id + +# 腾讯云COS的密钥 +# COS_SECRET_KEY=your_cos_secret_key + +# 腾讯云COS的区域，例如 ap-guangzhou +# COS_REGION=your_cos_region + +# 腾讯云COS的桶名称 +# COS_BUCKET_NAME=your_cos_bucket_name + +# 腾讯云COS的应用ID +# COS_APP_ID=your_cos_app_id + +# 腾讯云COS的路径前缀，用于存储文件 +# COS_PATH_PREFIX=your_cos_path_prefix + +# COS_ENABLE_OLD_DOMAIN=true 表示启用旧的域名格式，默认为 true +COS_ENABLE_OLD_DOMAIN=true + +# 如果解析网络连接使用Web代理，需要配置以下参数 +# WEB_PROXY=your_web_proxy + +# Neo4j 开关 +# NEO4J_ENABLE=false + +# Neo4j的访问地址 +# NEO4J_URI=neo4j://neo4j:7687 + +# Neo4j的用户名和密码 +# NEO4J_USERNAME=neo4j + +# Neo4j的密码 +# NEO4J_PASSWORD=password + +############################################################## + +###### 注意： 以下配置不再生效，已在Web“配置初始化”阶段完成 ######### + + +# # 初始化默认租户与知识库 +# # 租户ID，通常是一个字符串 +# INIT_TEST_TENANT_ID=1 + +# # 知识库ID，通常是一个字符串 +# INIT_TEST_KNOWLEDGE_BASE_ID=kb-00000001 + +# # LLM Model +# # 使用的LLM模型名称 +# # 默认使用 Ollama 的 Qwen3 8B 模型，ollama 会自动处理模型下载和加载 +# # 如果需要使用其他模型，请替换为实际的模型名称 +# INIT_LLM_MODEL_NAME=qwen3:8b + +# # LLM模型的访问地址 +# # 支持第三方模型服务的URL +# # 如果使用 Ollama 的本地服务，可以留空，ollama 会自动处理 +# # INIT_LLM_MODEL_BASE_URL=your_llm_model_base_url + +# # LLM模型的API密钥，如果需要身份验证，可以设置 +# # 支持第三方模型服务的API密钥 +# # 如果使用 Ollama 的本地服务，可以留空，ollama 会自动处理 +# # INIT_LLM_MODEL_API_KEY=your_llm_model_api_key + +# # Embedding Model +# # 使用的Embedding模型名称 +# # 默认使用 nomic-embed-text 模型，支持文本嵌入 +# # 如果需要使用其他模型，请替换为实际的模型名称 +# INIT_EMBEDDING_MODEL_NAME=nomic-embed-text + +# # Embedding模型向量维度 +# INIT_EMBEDDING_MODEL_DIMENSION=768 + +# # Embedding模型的ID，通常是一个字符串 +# INIT_EMBEDDING_MODEL_ID=builtin:nomic-embed-text:768 + +# # Embedding模型的访问地址 +# # 支持第三方模型服务的URL +# # 如果使用 Ollama 的本地服务，可以留空，ollama 会自动处理 +# # INIT_EMBEDDING_MODEL_BASE_URL=your_embedding_model_base_url + +# # Embedding模型的API密钥，如果需要身份验证，可以设置 +# # 支持第三方模型服务的API密钥 +# # 如果使用 Ollama 的本地服务，可以留空，ollama 会自动处理 +# # INIT_EMBEDDING_MODEL_API_KEY=your_embedding_model_api_key + +# # Rerank Model(可选) +# # 对于rag来说，使用Rerank模型对提升文档搜索的准确度有着重要作用 +# # 目前 ollama 暂不支持运行 Rerank 模型 +# # 使用的Rerank模型名称 +# # INIT_RERANK_MODEL_NAME=your_rerank_model_name + +# # Rerank模型的访问地址 +# # 支持第三方模型服务的URL +# # INIT_RERANK_MODEL_BASE_URL=your_rerank_model_base_url + +# # Rerank模型的API密钥，如果需要身份验证，可以设置 +# # 支持第三方模型服务的API密钥 +# # INIT_RERANK_MODEL_API_KEY=your_rerank_model_api_key + +# # VLM_MODEL_NAME 使用的多模态模型名称 +# # 用于解析图片数据 +# # VLM_MODEL_NAME=your_vlm_model_name + +# # VLM_MODEL_BASE_URL 使用的多模态模型访问地址 +# # 支持第三方模型服务的URL +# # VLM_MODEL_BASE_URL=your_vlm_model_base_url + +# # VLM_MODEL_API_KEY 使用的多模态模型API密钥 +# # 支持第三方模型服务的API密钥 +# # VLM_MODEL_API_KEY=your_vlm_model_api_key \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..526c8a3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3fb44ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +# 忽略.env文件和其他包含敏感信息的配置文件 +.env +# 但不忽略示例文件 +!.env.example +*.pem +*_key +*_secret +*.key +*.crt + +# IDE和编辑器文件 +.idea/ +.vscode/ +*.swp +*.swo + +# 构建和依赖文件 +node_modules/ +/dist/ +/build/ +*.log + +# 临时文件 +tmp/ +temp/ + +WeKnora +/models/ +services/docreader/src/proto/__pycache__ +test/data/mswag.txt +data/files/ + +.python-version +.venv/ + +### macOS +# General +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0fe4684 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,103 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [0.1.4] - 2025-09-17 + +### 🚀 Major Features +- **NEW**: Multi-knowledgebases operation support + - Added comprehensive multi-knowledgebase management functionality + - Implemented multi-data source search engine configuration and optimization logic + - Enhanced knowledge base switching and management in UI +- **NEW**: Enhanced tenant information management + - Added dedicated tenant information page + - Improved user and tenant management capabilities + +### 🎨 UI/UX Improvements +- **REDESIGNED**: Settings page with improved layout and functionality +- **ENHANCED**: Menu component with multi-knowledgebase support +- **IMPROVED**: Initialization configuration page structure +- **OPTIMIZED**: Login page and authentication flow + +### 🔒 Security Fixes +- **FIXED**: XSS attack vulnerabilities in thinking component +- **FIXED**: Content Security Policy (CSP) errors +- **ENHANCED**: Frontend security measures and input sanitization + +### 🐛 Bug Fixes +- **FIXED**: Login direct page navigation issues +- **FIXED**: App LLM model check logic +- **FIXED**: Version script functionality +- **FIXED**: File download content errors +- **IMPROVED**: Document content component display + +### 🧹 Code Cleanup +- **REMOVED**: Test data functionality and related APIs +- **SIMPLIFIED**: Initialization configuration components +- **CLEANED**: Redundant UI components and unused code + + +## [0.1.3] - 2025-09-16 + +### 🔒 Security Features +- **NEW**: Added login authentication functionality to enhance system security +- Implemented user authentication and authorization mechanisms +- Added session management and access control +- Fixed XSS attack vulnerabilities in frontend components + +### 📚 Documentation Updates +- Added security notices in all README files (English, Chinese, Japanese) +- Updated deployment recommendations emphasizing internal/private network deployment +- Enhanced security guidelines to prevent information leakage risks +- Fixed documentation spelling issues + +### 🛡️ Security Improvements +- Hide API keys in UI for security purposes +- Enhanced input sanitization and XSS protection +- Added comprehensive security utilities + +### 🐛 Bug Fixes +- Fixed OCR AVX support issues +- Improved frontend health check dependencies +- Enhanced Docker binary downloads for target architecture +- Fixed COS file service initialization parameters and URL processing logic + +### 🚀 Features & Enhancements +- Improved application and docreader log output +- Enhanced frontend routing and authentication flow +- Added comprehensive user management system +- Improved initialization configuration handling + +### 🛡️ Security Recommendations +- Deploy WeKnora services in internal/private network environments +- Avoid direct exposure to public internet +- Configure proper firewall rules and access controls +- Regular updates for security patches and improvements + +## [0.1.2] - 2025-09-10 + +- Fixed health check implementation for docreader service +- Improved query handling for empty queries +- Enhanced knowledge base column value update methods +- Optimized logging throughout the application +- Added process parsing documentation for markdown files +- Fixed OCR model pre-fetching in Docker containers +- Resolved image parser concurrency errors +- Added support for modifying listening port configuration + +## [0.1.0] - 2025-09-08 + +- Initial public release of WeKnora. +- Web UI for knowledge upload, chat, configuration, and settings. +- RAG pipeline with chunking, embedding, retrieval, reranking, and generation. +- Initialization wizard for configuring models (LLM, embedding, rerank, retriever). +- Support for local Ollama and remote API models. +- Vector backends: PostgreSQL (pgvector), Elasticsearch; GraphRAG support. +- End-to-end evaluation utilities and metrics. +- Docker Compose for quick startup and service orchestration. +- MCP server support for integrating with MCP-compatible clients. + +[0.1.4]: https://github.com/Tencent/WeKnora/tree/v0.1.4 +[0.1.3]: https://github.com/Tencent/WeKnora/tree/v0.1.3 +[0.1.2]: https://github.com/Tencent/WeKnora/tree/v0.1.2 +[0.1.0]: https://github.com/Tencent/WeKnora/tree/v0.1.0 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6c0d385 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,310 @@ +# WeKnora - 项目分析与开发指南 + +> 本文档由 Claude Code 自动生成，基于对 WeKnora 项目的全面分析 + +## ️ 项目架构概览 + +WeKnora 是一个基于大语言模型的文档理解与语义检索框架，采用模块化架构构建完整的 RAG 流水线。 + +### 核心技术栈 + +- **后端**: Go 1.24 + Gin 框架 + Uber Dig 依赖注入 +- **前端**: Vue 3 + TypeScript + TDesign UI + Vite +- **文档解析**: Python gRPC 微服务 +- **基础设施**: Docker + Docker Compose +- **数据库**: PostgreSQL (pgvector) + Redis + Neo4j +- **存储**: MinIO (对象存储) +- **监控**: Jaeger (分布式追踪) + +## 📁 项目目录结构 + +``` +WeKnora/ +├── cmd/server/ # Go 应用入口 +│ └── main.go # 主程序入口，依赖注入容器初始化 +├── config/ # 配置文件 +│ └── config.yaml # 服务配置（对话配置、提示词模板等） +├── docker/ # Docker 构建文件 +│ ├── Dockerfile.app # 后端应用镜像 +│ └── Dockerfile.docreader # 文档解析服务镜像 +├── frontend/ # 前端应用 +│ ├── src/ # 源码目录 +│ │ ├── views/ # 页面组件 +│ │ │ ├── auth/ # 认证页面 +│ │ │ ├── chat/ # 聊天界面 +│ │ │ ├── knowledge/ # 知识库管理 +│ │ │ └── settings/ # 系统设置 +│ │ ├── components/ # 通用组件 +│ │ ├── stores/ # Pinia 状态管理 +│ │ ├── api/ # API 接口定义 +│ │ └── router/ # 路由配置 +│ ├── package.json # 前端依赖管理 +│ └── vite.config.ts # Vite 构建配置 +├── internal/ # Go 内部包（核心业务逻辑） +│ ├── application/ # 应用层 +│ │ ├── repository/ # 数据访问层 +│ │ ├── service/ # 业务服务层 +│ │ └── chat_pipline/ # 聊天流水线插件系统 +│ ├── config/ # 配置管理 +│ ├── handler/ # HTTP 请求处理器 +│ ├── middleware/ # Gin 中间件 +│ ├── models/ # 模型定义（LLM、Embedding 等） +│ ├── router/ # 路由定义 +│ ├── types/ # 类型定义和接口 +│ └── utils/ # 工具函数 +├── migrations/ # 数据库迁移脚本 +├── scripts/ # 构建和部署脚本 +├── services/ # 微服务实现 +│ └── docreader/ # Python 文档解析服务 +└── docs/ # 项目文档 +``` + +## 🔧 核心模块分析 + +### 后端架构 (Go) + +#### 1. 依赖注入系统 (`internal/container/`) +- 使用 Uber Dig 实现依赖注入 +- 统一管理所有服务的生命周期 +- 支持接口隔离和模块化测试 + +#### 2. 聊天流水线系统 (`internal/application/service/chat_pipline/`) +- 插件化架构，支持事件驱动处理 +- 主要事件类型：预处理、检索、重排、生成等 +- 支持熔断机制和错误处理 + +#### 3. 检索引擎 (`internal/application/service/retriever/`) +- 混合检索策略：向量检索 + 关键词检索 +- 支持多种向量数据库后端 +- 基于接口的插件化设计 + +#### 4. API 路由 (`internal/router/`) +- RESTful API 设计 +- 认证中间件和权限控制 +- 一致的错误处理机制 + +### 前端架构 (Vue 3) + +#### 1. 路由配置 (`frontend/src/router/index.ts`) +- 基于文件的路由配置 +- 路由守卫实现认证检查 +- 支持知识库级别的路由嵌套 + +#### 2. 状态管理 (`frontend/src/stores/`) +- 使用 Pinia 进行状态管理 +- 模块化状态设计：认证、知识库、菜单等 +- 类型安全的 TypeScript 支持 + +#### 3. 组件设计 +- 基于 TDesign Vue Next 组件库 +- 模块化组件设计，职责单一 +- 支持响应式布局和主题定制 + +### 文档解析微服务 (Python) + +#### 1. gRPC 服务设计 (`services/docreader/src/server/server.py`) +- 基于 gRPC 的高性能文档解析 +- 支持多格式文档：PDF、Word、图片、网页等 +- 多模态处理支持 OCR 和图像字幕 + +#### 2. 解析器架构 (`services/docreader/src/parser/`) +- 插件化文档解析器 +- 支持分块配置和自定义分隔符 +- 集成视觉语言模型 (VLM) 处理图像 + +## 🚀 开发环境配置 + +### 1. 环境要求 +```bash +# 必备工具 +Docker & Docker Compose +Git +Go 1.24+ (后端开发) +Node.js 18+ (前端开发) +Python 3.8+ (文档解析服务开发) +``` + +### 2. 快速启动 +```bash +# 克隆项目 +git clone https://github.com/Tencent/WeKnora.git +cd WeKnora + +# 配置环境变量 +cp .env.example .env +# 编辑 .env 文件设置必要参数 + +# 启动完整服务栈 +./scripts/start_all.sh + +# 或使用 Makefile +make start-all +``` + +### 3. 开发模式启动 +```bash +# 仅启动基础设施服务 +docker-compose up postgres redis minio neo4j jaeger + +# 后端开发模式 (热重载) +cd cmd/server +go run main.go + +# 前端开发模式 +cd frontend +npm install +npm run dev + +# 文档解析服务开发 +cd services/docreader +poetry install +python src/server/server.py +``` + +## 🔌 关键配置文件 + +### 1. 后端配置 (`config/config.yaml`) +- 服务器端口和主机配置 +- 对话服务参数（重写、重排阈值等） +- 提示词模板配置 +- 知识库分块配置 + +### 2. Docker 编排 (`docker-compose.yml`) +- 微服务容器定义和网络配置 +- 环境变量注入和健康检查 +- 数据卷持久化配置 + +### 3. Makefile 构建系统 +- 多平台 Docker 镜像构建 +- 数据库迁移和清理 +- 开发工具链集成 + +## 📊 数据模型概览 + +### 核心实体关系 +``` +Tenant (租户) + ↓ +KnowledgeBase (知识库) → EmbeddingModel (嵌入模型) + ↓ +Knowledge (知识) → Chunk (分块) + ↓ +Session (会话) → Message (消息) +``` + +### 主要数据库表 +- `knowledge_bases`: 知识库元数据 +- `knowledge`: 知识文档信息 +- `chunks`: 文档分块内容（向量化存储） +- `sessions`: 聊天会话 +- `messages`: 对话消息历史 + +## 🔍 API 接口概览 + +### 认证相关 (`/api/v1/auth/*`) +- `POST /auth/login` - 用户登录 +- `POST /auth/register` - 用户注册 +- `GET /auth/me` - 获取当前用户信息 + +### 知识库管理 (`/api/v1/knowledge-bases/*`) +- `POST /knowledge-bases` - 创建知识库 +- `GET /knowledge-bases` - 获取知识库列表 +- `GET /knowledge-bases/:id/hybrid-search` - 混合搜索 + +### 文档管理 (`/api/v1/knowledge/*`) +- `POST /knowledge-bases/:id/knowledge/file` - 文件上传 +- `POST /knowledge-bases/:id/knowledge/url` - URL 导入 +- `GET /knowledge/:id` - 获取知识详情 + +### 对话接口 (`/api/v1/knowledge-chat/*`) +- `POST /knowledge-chat/:session_id` - 知识问答 +- `POST /knowledge-search` - 知识检索 + +## 🛠️ 开发扩展指南 + +### 1. 添加新的检索策略 +1. 在 `internal/types/interfaces/retriever.go` 定义接口 +2. 在 `internal/application/service/retriever/` 实现检索器 +3. 在 `internal/application/service/retriever/registry.go` 注册 +4. 配置租户的检索器引擎 + +### 2. 扩展文档解析格式 +1. 在 `services/docreader/src/parser/` 添加新的解析器 +2. 实现 `parse_file` 方法 +3. 在 `services/docreader/src/parser/parser.py` 注册 + +### 3. 定制聊天流水线 +1. 在 `internal/application/service/chat_pipline/` 添加插件 +2. 实现 `Plugin` 接口 +3. 注册到事件管理器 +4. 配置事件处理链 + +## 调试和监控 + +### 1. 日志系统 +- 后端使用 structured logging +- 请求 ID 追踪支持 +- 日志级别可配置 + +### 2. 分布式追踪 +- 集成 OpenTelemetry +- Jaeger UI: http://localhost:16686 +- 完整的请求链路追踪 + +### 3. 健康检查 +- 服务健康检查端点: `/health` +- 数据库连接状态监控 +- gRPC 健康检查支持 + +## 🔒 安全考虑 + +### 1. 认证授权 +- JWT Token 认证 +- 租户级别的数据隔离 +- API Key 访问控制 + +### 2. 数据安全 +- 敏感信息环境变量配置 +- 文件上传安全检查 +- SQL 注入防护 + +### 3. 部署安全 +- 内网部署推荐 +- 防火墙规则配置 +- 定期安全更新 + +## 📈 性能优化建议 + +### 1. 数据库优化 +- 向量索引优化 +- 批量查询优化 +- 连接池配置 + +### 2. 检索优化 +- 分块策略调优 +- 缓存策略实现 +- 异步处理优化 + +### 3. 前端优化 +- 代码分割和懒加载 +- 图片优化和压缩 +- API 请求合并 + +--- + +## 📚 进一步阅读 + +- [官方文档](./docs/WeKnora.md) - 完整的功能说明 +- [API 文档](./docs/API.md) - 详细的接口文档 +- [QA 文档](./docs/QA.md) - 常见问题解答 + +## 🤝 贡献指南 + +项目采用标准的 Git 工作流： +1. Fork 项目到个人账户 +2. 创建功能分支 (`feature/feature-name`) +3. 提交代码变更 +4. 创建 Pull Request +5. 代码审查和合并 + +遵循 Conventional Commits 规范提交消息。 \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..75b8af5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,2692 @@ +Tencent is pleased to support the open source community by making this project available. + +Copyright (C) 2025 Tencent. All rights reserved. + +This project is licensed under the MIT License except for the third-party components listed below, which is licensed under different terms. Tencent does not impose any additional limitations beyond what is outlined in the respective licenses of these third-party components. Users must comply with all terms and conditions of original licenses of these third-party components and must ensure that the usage of the third party components adheres to all relevant laws and regulations. + + +Terms of the MIT License: +-------------------------------------------------------------------- +Copyright (C) 2025 Tencent. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the " Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------- +Other third-party components: + +Open Source Software Licensed under the Apache-2.0: +-------------------------------------------------------------------- +1. otel-1.37.0 +Copyright (c) 2025 opentelemetry-go original author and authors + +2. stdr-1.2.2 +Copyright (c) 2021 stdr original author and authors + +3. parquet-go-0.25.0 +Copyright 2023 Twilio, Inc. + +4. rpc-0.0.0-20250603155806-513f23925822 +Copyright (c) 2025 go-genproto original author and authors + +5. api-0.0.0-20250603155806-513f23925822 +Copyright (c) 2025 go-genproto original author and authors + +6. grpc-1.73.0 +Copyright (c) 2025 grpc-go original author and authors + +7. base64x-0.1.5 +Copyright (c) 2025 base64x original author and authors + +8. trace-1.37.0 +Copyright (c) 2025 opentelemetry-go original author and authors + +9. loader-0.2.4 +Copyright 2025 ByteDance Inc. + +10. paddlepaddle-gpu-3.0.0 +Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved + +11. otlp-1.7.0 +Copyright (c) 2025 opentelemetry-proto original author and authors + +12. logr-1.4.3 +Copyright (c) 2025 logr original author and authors + +13. otlptrace-1.37.0 +Copyright (c) 2025 opentelemetry-go original author and authors + +14. metric-1.37.0 +Copyright (c) 2025 opentelemetry-go original author and authors + +15. sonic-1.13.2 +Copyright (c) 2025 sonic original author and authors + +16. go-openai-1.40.5 +Copyright (c) 2025 go-openai original author and authors + +17. crc64nvme-1.0.1 +Copyright (c) 2025 crc64nvme original author and authors + +18. concurrent-0.0.0-20180306012644-bacd9c7ef1dd +Copyright (c) 2018 concurrent original author and authors + +19. stdouttrace-1.35.0 +Copyright (c) 2025 opentelemetry-go original author and authors + +20. otlptracegrpc-1.37.0 +Copyright (c) 2025 opentelemetry-go original author and authors + +21. paddleocr-3.0.0 +Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + + + +Terms of the Apache-2.0: +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Open Source Software Licensed under the BSD-2-Clause: +-------------------------------------------------------------------- +1. esrecurse-4.3.0 +Copyright (C) 2014 [Yusuke Suzuki](https://github.com/Constellation) (twitter: [@Constellation](https://twitter.com/Constellation)) and other contributors. + +2. glob-to-regexp-0.4.1 +Copyright (c) 2013, Nick Fitzgerald + +3. eslint-scope-5.1.1 +Copyright(c) 2025 eslint-scope original author and authors + + + +Terms of the BSD-2-Clause: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Open Source Software Licensed under the BSD-3-Clause: +-------------------------------------------------------------------- +1. text-0.26.0 +Copyright 2009 The Go Authors. + +2. compress-1.18.0 +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. +Copyright 2016-2017 The New York Times Company +Copyright (c) 2015 Klaus Post +Copyright 2016 The filepath Authors + +3. net-0.41.0 +Copyright 2009 The Go Authors. + +4. fsnotify-1.8.0 +Copyright © 2012 The Go Authors. All rights reserved. +Copyright © fsnotify Authors. All rights reserved. + +5. sys-0.33.0 +Copyright 2009 The Go Authors. + +6. protobuf-1.36.6 +Copyright (c) 2018 The Go Authors. All rights reserved. + +7. arch-0.15.0 +Copyright 2015 The Go Authors. + + + +Terms of the BSD-3-Clause: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the ORGANIZATION nor the names of its contributors may be +used to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Open Source Software Licensed under the CC-BY-4.0: +-------------------------------------------------------------------- +1. caniuse-lite-1.0.30001727 +Copyright(c) 2025 Ben Briggs + + + +Terms of the CC-BY-4.0: +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + + +Open Source Software Licensed under the ISC: +-------------------------------------------------------------------- +1. electron-to-chromium-1.5.183 +Copyright 2018 Kilian Valkhof + + + +Terms of the ISC: +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +Open Source Software Licensed under the MIT: +-------------------------------------------------------------------- +1. birpc-2.5.0 +Copyright (c) 2021 Anthony Fu + +2. @babel/generator-7.28.0 +Copyright (c) 2014-present Sebastian Mckenzie and other contributors + +3. neo-async-2.6.2 +Copyright (c) 2014-2018 Suguru Motegi + +4. parse-node-version-1.0.1 +Copyright (c) 2018 Blaine Bublitz and Eric Schoffstall + +5. cors-1.7.5 +Copyright (c) 2016 Gin-Gonic + +6. vue-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +7. copy-anything-2.0.6 +Copyright (c) 2018 Luca Ban - Mesqueeb + +8. yaml.v3-3.0.1 +Copyright (c) 2006-2010 Kirill Simonov +Copyright (c) 2006-2011 Kirill Simonov +Copyright (c) 2011-2019 Canonical Ltd + +9. gojieba-1.4.5 +Copyright (c) The project creators and maintainers + +10. codec-1.2.12 +Copyright (c) 2012-2020 Ugorji Nwoke. + +11. vue-tsc-2.2.12 +Copyright(c) 2025 vue-tsc original author and authors + +12. follow-redirects-1.15.9 +Copyright 2017-present Oliviver Lalonde , James Talmage , Ruben Verborgh + +13. @vue/server-renderer-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +14. pgvector-go-0.3.0 +Copyright (c) 2021-2025 Andrew Kane + +15. @babel/traverse-7.28.0 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +16. @webassemblyjs/helper-api-error-1.13.2 +Copyright (c) 2018 Sven Sauleau + +17. estree-walker-2.0.2 +Copyright (c) 2015-20 [these people](https://github.com/Rich-Harris/estree-walker/graphs/contributors) + +18. @types/node-22.16.3 +Copyright(c) 2025 @types/node original author and authors + +19. magic-string-0.30.17 +Copyright 2018 Rich Harris + +20. pinia-3.0.3 +Copyright (c) 2019-present Eduardo San Martin Morote + +21. @types/lodash-4.17.20 +Copyright(c) 2025 @types/lodash original author and authors + +22. es-set-tostringtag-2.1.0 +Copyright (c) 2022 ECMAScript Shims + +23. @vue/runtime-dom-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +24. @vue/reactivity-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +25. de-indent-1.0.2 +Copyright(c) 2025 Evan You + +26. @babel/template-7.27.2 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +27. tdesign-icons-vue-next-0.3.6 +Copyright (c) 2021-present TDesign + +28. webpack-5.100.1 +Copyright JS Foundation and other contributors + +29. asynq-0.25.1 +Copyright (c) 2019 Kentaro Hibino + +30. shell-quote-1.8.3 +Copyright (c) 2013 James Halliday (mail@substack.net) + +31. acorn-8.15.0 +Copyright(c) 2025 acorn original author and authors + +32. esbuild-0.25.6 +Copyright (c) 2020 Evan Wallace + +33. tdesign-vue-next-1.14.2 +Copyright (c) 2021-present TDesign + +34. @vue/runtime-core-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +35. validator-13.15.15 +Q29weXJpZ2h0IChjKSAyMDE4IENocmlzIE8nSGFyYSA8Y29oYXJhODdAZ21haWwuY29tPg== + +36. postcss-8.5.6 +Copyright 2013 Andrey Sitnik + +37. mimetype-1.4.8 +Copyright (c) 2018 Gabriel Vasile + +38. @babel/helper-globals-7.28.0 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +39. @babel/helper-create-class-features-plugin-7.27.1 +Copyright (c) 2014-present Sebastian McKenzieg and other contributors + +40. @babel/helper-replace-supers-7.27.1 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +41. is-what-3.14.1 +Copyright (c) 2018 Luca Ban - Mesqueeb + +42. @rolldown/pluginutils-1.0.0-beta.27 +Copyright (c) 2024-present VoidZero Inc. & Contributors + +43. ollama-0.9.6 +Copyright (c) Ollama + +44. locafero-0.7.0 +Copyright (c) 2023 Márk Sági-Kazar + +45. stylus->=0.54.8 +Copyright (c) Automatttic + +46. @vue/language-core-2.2.12 +Copyright (c) 2021-present Johnson Chu + +47. @babel/parser-7.28.0 +Copyright (c) 2014-present Sebastian MacKenzie and other contributors + +48. cos-go-sdk-v5-0.7.65 +Copyright (c) 2017 mozillazg + +49. @rolldown/pluginutils-1.0.0-beta.19 +Copyright (c) 2024-present VoidZero Inc. & Contributors + +50. @vue/compiler-core-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +51. @vue/shared-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +52. @vitejs/plugin-vue-jsx-5.0.1 +Copyright (c) 2019-present, Yuxi +(Evan) You and Vite contributors + +53. fdir-6.4.6 +Copyright 2023 AbdullAh AttA + +54. go-rendezvous-0.0.0-20200823014737-9f7001d12a5f +Copyright (c) 2017-2020 Damian Gryski + +55. @babel/plugin-syntax-jsx-7.27.1 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +56. @vue/compiler-ssr-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +57. @babel/helper-member-expression-to-functions-7.27.1 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +58. es-module-lexer-1.7.0 +Copyright (C) 2018-2022 Guy Bedford + +59. vue-router-4.5.1 +Copyright (c) 2019-present Eduardo San Martin Morote + +60. @babel/helper-validator-option-7.27.1 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +61. @vue/compiler-sfc-3.5.17 +Copyright (c) 2018-present, Yuxi (Evan) You and Vue contributors + +62. @vue/devtools-shared-7.7.7 +Copyright (c) 2023 webfansplz + +63. gensync-1.0.0-beta.2 +Copyright 2018 Logan Smyth + +64. @rspack/core-0.x || 1.x +Copyright (c) 2022-present Bytsedance Inc. and its affiliates. + +65. jiti->=1.21.0 +Copyright (c) Pooya Parsa + +66. pgservicefile-0.0.0-20240606120523-5a60cdf6a761 +Copyright (c) 2020 Jack Christensens + +67. @types/validator-13.15.2 +Copyright(c) 2025 @types/validator original author and authors + +68. @vue/devtools-api-7.7.7 +Copyright (c) 2023 webfansplz + +69. tapable-2.2.2 +Copyright JS Foundation and +other contributors + +70. @vitejs/plugin-vue-6.0.0 +Copyright (c) 2019-present, Yuxhi (Evan) You and Vite contributors + +71. @babel/core-7.28.0 +Copyright (c) 2014-present Sebastian McInerney and other contributors + +72. @babel/types-7.28.1 +Copyright (c) 2014-present Sebastian McPkenzie and other contributors + +73. @babel/compat-data-7.28.0 +Copyright (c) 2014-present Sebastian MacKenzie and other contributors + +74. @volar/typescript-2.4.15 +Copyright (c) 2021-present Johnson Chu + +75. @babel/helper-annotate-as-pure-7.27.3 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +76. vite-7.0.4 +Copyright (c) 2019-present, VoidZero Inc. and Vite contributors + +77. @volar/source-map-2.4.15 +Copyright (c) 2021-present Johnson Chu + +78. json-parse-even-better-errors-2.3.1 +Copyright 2017 Kat Marchán +Copyright npm, Inc. + +79. @vue/compiler-dom-3.5.17 +Copyright (c) 2018-present, Yuxia (Evan) You and Vue contributors + +80. @volar/language-core-2.4.15 +Copyright (c) 2021-present Johnson Chu + +81. @vue/devtools-kit-7.7.7 +Copyright (c) 2023 webfansplz + +82. @babel/helper-skip-transparent-expression-wrappers-7.27.1 +Copyright (c) 2014-present Sebastian MacKenzie and other contributors + +83. mitt-3.0.1 +Copyright (c) 2021 Jason Miller + +84. acorn-import-phases-1.0.4 +Copyright (c) 2025 Nicolò Ribaudo + +85. @babel/helper-string-parser-7.27.1 +Copyright (c) 2014-present Sebastian McKeenzied and other contributors + +86. watchpack-2.4.4 +Copyright JS Foundation and other contributors + +87. @babel/runtime-7.27.6 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +88. @babel/plugin-transform-typescript-7.28.0 +Copyright (c) 2014-present Sebastian McKenzie and other contributors + +89. jest-worker-27.5.1 +Copyright (c) jest-worker authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/jest-worker. + + + +Terms of the MIT: +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Open Source Software Licensed under the MIT AND ISC AND 0BSD: +-------------------------------------------------------------------- +1. rollup-4.45.0 +Copyright(c) 2025 Rich Harris + + + +Terms of the MIT AND ISC AND 0BSD: +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Open Source Software Licensed under the MIT-CMU: +-------------------------------------------------------------------- +1. Pillow-default +Copyright © 1997-2011 by Secret Labs AB +Copyright © 1995-2011 by Fredrik Lundh and contributors +Copyright © 2010 by Jeffrey A. Clark and contributors + + + +Terms of the MIT-CMU: +Permission to use, copy, modify and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appears in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of CMU and The Regents of the University of California not be used in +advertising or publicity pertaining to distribution of the software without +specific written permission. + +CMU AND THE REGENTS OF THE UNIVERSITY OF CALIFORNIA DISCLAIM ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL CMU OR THE REGENTS OF THE UNIVERSITY OF CALIFORNIA BE +LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM THE LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +Open Source Software Licensed under the MPL-2.0: +-------------------------------------------------------------------- +1. lightningcss-1.21.0 +Copyright(c) 2025 lightningcss original author and authors + + + +Terms of the MPL-2.0: +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. + +Open Source Software Licensed under the apache-2.0: +-------------------------------------------------------------------- +1. @ampproject/remapping-2.3.0 +Copyright (c) @ampproject/remapping authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@ampproject/remapping. + +2. sync-0.15.0 +Copyright (c) sync authors. +You may obtain the source code and detailed information about this component at https://mvnrepository.com/artifact/com.aerospike/sync. + +3. @webassemblyjs/leb128-1.13.2 +Copyright (c) @webassemblyjs/leb128 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/leb128. + +4. sdk-1.37.0 +Copyright (c) sdk authors. +You may obtain the source code and detailed information about this component at https://mvnrepository.com/artifact/io.lakefs/sdk. + +5. crypto-0.39.0 +Copyright (c) crypto authors. +You may obtain the source code and detailed information about this component at https://github.com/smallstep/crypto. + +6. grpcio-default +Copyright (c) grpcio authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/grpcio. + +7. sdk-1.1.0 +Copyright (c) sdk authors. +You may obtain the source code and detailed information about this component at https://mvnrepository.com/artifact/com.jedlix/sdk. + +8. afero-1.12.0 +Copyright (c) afero authors. +You may obtain the source code and detailed information about this component at https://github.com/spf13/afero. + +9. @xtuc/long-4.2.2 +Copyright (c) @xtuc/long authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@xtuc/long. + +10. chrome-trace-event-1.0.4 +Copyright (c) chrome-trace-event authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/chrome-trace-event. + +11. reflect2-1.0.2 +Copyright (c) reflect2 authors. +You may obtain the source code and detailed information about this component at https://github.com/modern-go/reflect2. + +12. grpcio-tools-default +Copyright (c) grpcio-tools authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/grpcio-tools. + +13. playwright-default +Copyright (c) playwright authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/playwright. + +14. requests-default +Copyright (c) requests authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/requests. + +15. ini-1.67.0 +Copyright (c) ini authors. +You may obtain the source code and detailed information about this component at https://github.com/go-ini/ini. + +16. asyncio-default +Copyright (c) asyncio authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/asyncio. + +17. time-0.11.0 +Copyright (c) time authors. +You may obtain the source code and detailed information about this component at https://mvnrepository.com/artifact/org.occurrent/time. + +18. less-4.3.0 +Copyright (c) less authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/less. + +19. typescript-5.8.3 +Copyright (c) typescript authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/typescript. + + + +Terms of the apache-2.0: +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Open Source Software Licensed under the bsd-new: +-------------------------------------------------------------------- +1. lxml-default +Copyright (c) lxml authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/lxml. + +2. pypdf-default +Copyright (c) pypdf authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/pypdf. + +3. go-querystring-1.1.0 +Copyright (c) go-querystring authors. +You may obtain the source code and detailed information about this component at https://github.com/google/go-querystring. + +4. speakingurl-14.0.1 +Copyright (c) speakingurl authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/speakingurl. + +5. markdown-default +Copyright (c) markdown authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/markdown. + +6. source-map-0.6.1 +Copyright (c) source-map authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/source-map. + +7. serialize-javascript-6.0.2 +Copyright (c) serialize-javascript authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/serialize-javascript. + +8. golang-asm-0.15.1 +Copyright (c) golang-asm authors. +You may obtain the source code and detailed information about this component at https://github.com/twitchyliquid64/golang-asm. + +9. source-map-js-1.2.1 +Copyright (c) source-map-js authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/source-map-js. + +10. PyPDF2-default +Copyright (c) PyPDF2 authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/pypdf2. + +11. fast-uri-3.0.6 +Copyright (c) fast-uri authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/fast-uri. + +12. mxj-1.8.4 +Copyright (c) mxj authors. +You may obtain the source code and detailed information about this component at https://github.com/clbanning/mxj. + +13. protobuf-default +Copyright (c) protobuf authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/protobuf. + +14. @xtuc/ieee754-1.2.0 +Copyright (c) @xtuc/ieee754 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@xtuc/ieee754. + + + +Terms of the bsd-new: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the ORGANIZATION nor the names of its contributors may be +used to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Open Source Software Licensed under the bsd-simplified: +-------------------------------------------------------------------- +1. terser-5.43.1 +Copyright (c) terser authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/terser. + +2. estraverse-4.3.0 +Copyright (c) estraverse authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/estraverse. + +3. entities-4.5.0 +Copyright (c) entities authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/entities. + + + +Terms of the bsd-simplified: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list +of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Open Source Software Licensed under the bsd-zero: +-------------------------------------------------------------------- +1. tslib-2.8.1 +Copyright (c) tslib authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/tslib. + + + +Terms of the bsd-zero: +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Open Source Software Licensed under the isc: +-------------------------------------------------------------------- +1. graceful-fs-4.2.11 +Copyright (c) graceful-fs authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/graceful-fs. + +2. which-5.0.0 +Copyright (c) which authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/which. + +3. isexe-3.1.1 +Copyright (c) isexe authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/isexe. + +4. yaml-2.4.2 +Copyright (c) yaml authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/yaml. + +5. lru-cache-5.1.1 +Copyright (c) lru-cache authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/lru-cache. + +6. read-package-json-fast-4.0.0 +Copyright (c) read-package-json-fast authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/read-package-json-fast. + +7. minimatch-9.0.5 +Copyright (c) minimatch authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/minimatch. + +8. semver-6.3.1 +Copyright (c) semver authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/semver. + +9. npm-normalize-package-bin-4.0.0 +Copyright (c) npm-normalize-package-bin authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/npm-normalize-package-bin. + +10. picocolors-1.1.1 +Copyright (c) picocolors authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/picocolors. + + + +Terms of the isc: +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. + + +Open Source Software Licensed under the mit: +-------------------------------------------------------------------- +1. @tsconfig/node22-22.0.2 +Copyright (c) @tsconfig/node22 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@tsconfig/node22. + +2. tinyglobby-0.2.14 +Copyright (c) tinyglobby authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/tinyglobby. + +3. jsesc-3.1.0 +Copyright (c) jsesc authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/jsesc. + +4. es-errors-1.3.0 +Copyright (c) es-errors authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/es-errors. + +5. brotli-1.1.0 +Copyright (c) brotli authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/brotli. + +6. undici-types-6.21.0 +Copyright (c) undici-types authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/undici-types. + +7. dayjs-1.11.10 +Copyright (c) dayjs authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/dayjs. + +8. @jridgewell/gen-mapping-0.3.12 +Copyright (c) @jridgewell/gen-mapping authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@jridgewell/gen-mapping. + +9. events-3.3.0 +Copyright (c) events authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/events. + +10. @types/sortablejs-1.15.8 +Copyright (c) @types/sortablejs authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/sortablejs. + +11. js-tokens-4.0.0 +Copyright (c) js-tokens authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/js-tokens. + +12. @vue/babel-helper-vue-transform-on-1.4.0 +Copyright (c) @vue/babel-helper-vue-transform-on authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@vue/babel-helper-vue-transform-on. + +13. @types/marked-5.0.2 +Copyright (c) @types/marked authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/marked. + +14. @types/tinycolor2-1.4.6 +Copyright (c) @types/tinycolor2 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/tinycolor2. + +15. xid-1.6.0 +Copyright (c) xid authors. +You may obtain the source code and detailed information about this component at https://github.com/rs/xid. + +16. lodash-es-4.17.21 +Copyright (c) lodash-es authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/lodash-es. + +17. go-runewidth-0.0.15 +Copyright (c) go-runewidth authors. +You may obtain the source code and detailed information about this component at https://gitee.com/mirrors/go-runewidth. + +18. marked-5.1.2 +Copyright (c) marked authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/marked. + +19. cast-1.7.1 +Copyright (c) cast authors. +You may obtain the source code and detailed information about this component at https://github.com/spf13/cast. + +20. antiword-default +Copyright (c) antiword authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/antiword. + +21. cos-python-sdk-v5-default +Copyright (c) cos-python-sdk-v5 authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/cos-python-sdk-v5. + +22. loader-runner-4.3.0 +Copyright (c) loader-runner authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/loader-runner. + +23. alien-signals-1.0.13 +Copyright (c) alien-signals authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/alien-signals. + +24. @webassemblyjs/wasm-parser-1.14.1 +Copyright (c) @webassemblyjs/wasm-parser authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/wasm-parser. + +25. textract-default +Copyright (c) textract authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/textract. + +26. hasown-2.0.2 +Copyright (c) hasown authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/hasown. + +27. shebang-regex-3.0.0 +Copyright (c) shebang-regex authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/shebang-regex. + +28. @types/eslint-scope-3.7.7 +Copyright (c) @types/eslint-scope authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/eslint-scope. + +29. node-releases-2.0.19 +Copyright (c) node-releases authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/node-releases. + +30. combined-stream-1.0.8 +Copyright (c) combined-stream authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/combined-stream. + +31. ajv-formats-2.1.1 +Copyright (c) ajv-formats authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/ajv-formats. + +32. has-tostringtag-1.0.2 +Copyright (c) has-tostringtag authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/has-tostringtag. + +33. vscode-uri-3.1.0 +Copyright (c) vscode-uri authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/vscode-uri. + +34. npm-run-all2-7.0.2 +Copyright (c) npm-run-all2 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/npm-run-all2. + +35. tablewriter-0.0.5 +Copyright (c) tablewriter authors. +You may obtain the source code and detailed information about this component at https://github.com/olekukonko/tablewriter. + +36. cross-spawn-7.0.6 +Copyright (c) cross-spawn authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/cross-spawn. + +37. gin-1.10.0 +Copyright (c) gin authors. +You may obtain the source code and detailed information about this component at https://github.com/donetkit/gin. + +38. he-1.2.0 +Copyright (c) he authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/he. + +39. @babel/helper-optimise-call-expression-7.27.1 +Copyright (c) @babel/helper-optimise-call-expression authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/helper-optimise-call-expression. + +40. debug-4.4.1 +Copyright (c) debug authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/debug. + +41. commander-2.20.3 +Copyright (c) commander authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/commander. + +42. mapstructure-1.4.3 +Copyright (c) mapstructure authors. +You may obtain the source code and detailed information about this component at https://github.com/go-viper/mapstructure. + +43. @vue/compiler-vue2-2.7.16 +Copyright (c) @vue/compiler-vue2 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@vue/compiler-vue2. + +44. escalade-3.2.0 +Copyright (c) escalade authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/escalade. + +45. conc-0.3.0 +Copyright (c) conc authors. +You may obtain the source code and detailed information about this component at https://github.com/sourcegraph/conc. + +46. @types/eslint-9.6.1 +Copyright (c) @types/eslint authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/eslint. + +47. go-httpheader-0.2.1 +Copyright (c) go-httpheader authors. +You may obtain the source code and detailed information about this component at https://github.com/mozillazg/go-httpheader. + +48. @webassemblyjs/wasm-opt-1.14.1 +Copyright (c) @webassemblyjs/wasm-opt authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/wasm-opt. + +49. @popperjs/core-2.11.8 +Copyright (c) @popperjs/core authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@popperjs/core. + +50. @webassemblyjs/wast-printer-1.14.1 +Copyright (c) @webassemblyjs/wast-printer authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/wast-printer. + +51. webpack-sources-3.3.3 +Copyright (c) webpack-sources authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/webpack-sources. + +52. @webassemblyjs/wasm-gen-1.14.1 +Copyright (c) @webassemblyjs/wasm-gen authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/wasm-gen. + +53. @babel/helper-validator-identifier-7.27.1 +Copyright (c) @babel/helper-validator-identifier authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/helper-validator-identifier. + +54. merge-stream-2.0.0 +Copyright (c) merge-stream authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/merge-stream. + +55. logrus-1.9.3 +Copyright (c) logrus authors. +You may obtain the source code and detailed information about this component at https://github.com/sirupsen/logrus. + +56. path-key-3.1.1 +Copyright (c) path-key authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/path-key. + +57. gorm-1.25.12 +Copyright (c) gorm authors. +You may obtain the source code and detailed information about this component at https://github.com/go-gorm/gorm. + +58. fast-deep-equal-3.1.3 +Copyright (c) fast-deep-equal authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/fast-deep-equal. + +59. @webassemblyjs/utf8-1.13.2 +Copyright (c) @webassemblyjs/utf8 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/utf8. + +60. @babel/plugin-syntax-typescript-7.27.1 +Copyright (c) @babel/plugin-syntax-typescript authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/plugin-syntax-typescript. + +61. mime-db-1.52.0 +Copyright (c) mime-db authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/mime-db. + +62. @webassemblyjs/floating-point-hex-parser-1.13.2 +Copyright (c) @webassemblyjs/floating-point-hex-parser authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/floating-point-hex-parser. + +63. hookable-5.5.3 +Copyright (c) hookable authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/hookable. + +64. pidtree-0.6.0 +Copyright (c) pidtree authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/pidtree. + +65. @types/lodash-es-4.17.12 +Copyright (c) @types/lodash-es authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/lodash-es. + +66. shebang-command-2.0.0 +Copyright (c) shebang-command authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/shebang-command. + +67. @webassemblyjs/wasm-edit-1.14.1 +Copyright (c) @webassemblyjs/wasm-edit authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/wasm-edit. + +68. convert-source-map-2.0.0 +Copyright (c) convert-source-map authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/convert-source-map. + +69. json-schema-traverse-1.0.0 +Copyright (c) json-schema-traverse authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/json-schema-traverse. + +70. @webassemblyjs/helper-buffer-1.14.1 +Copyright (c) @webassemblyjs/helper-buffer authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/helper-buffer. + +71. @babel/helper-module-imports-7.27.1 +Copyright (c) @babel/helper-module-imports authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/helper-module-imports. + +72. update-browserslist-db-1.1.3 +Copyright (c) update-browserslist-db authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/update-browserslist-db. + +73. sugarss-5.0.0 +Copyright (c) sugarss authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/sugarss. + +74. uniseg-0.4.7 +Copyright (c) uniseg authors. +You may obtain the source code and detailed information about this component at https://github.com/rivo/uniseg. + +75. buffer-from-1.1.2 +Copyright (c) buffer-from authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/buffer-from. + +76. @vue/babel-plugin-jsx-1.4.0 +Copyright (c) @vue/babel-plugin-jsx authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@vue/babel-plugin-jsx. + +77. @webassemblyjs/helper-wasm-section-1.14.1 +Copyright (c) @webassemblyjs/helper-wasm-section authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/helper-wasm-section. + +78. dig-1.18.1 +Copyright (c) dig authors. +You may obtain the source code and detailed information about this component at https://github.com/uber-go/dig. + +79. go-isatty-0.0.20 +Copyright (c) go-isatty authors. +You may obtain the source code and detailed information about this component at https://github.com/mattn/go-isatty. + +80. randombytes-2.1.0 +Copyright (c) randombytes authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/randombytes. + +81. mistletoe-default +Copyright (c) mistletoe authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/mistletoe. + +82. @jridgewell/trace-mapping-0.3.29 +Copyright (c) @jridgewell/trace-mapping authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@jridgewell/trace-mapping. + +83. @jridgewell/source-map-0.3.10 +Copyright (c) @jridgewell/source-map authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@jridgewell/source-map. + +84. ajv-keywords-5.1.0 +Copyright (c) ajv-keywords authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/ajv-keywords. + +85. memorystream-0.3.1 +Copyright (c) memorystream authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/memorystream. + +86. @vue/devtools-api-6.6.4 +Copyright (c) @vue/devtools-api authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@vue/devtools-api. + +87. go-1.1.12 +Copyright (c) go authors. +You may obtain the source code and detailed information about this component at https://github.com/json-iterator/go. + +88. supports-color-8.1.1 +Copyright (c) supports-color authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/supports-color. + +89. mime-types-2.1.35 +Copyright (c) mime-types authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/mime-types. + +90. enhanced-resolve-5.18.2 +Copyright (c) enhanced-resolve authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/enhanced-resolve. + +91. now-1.1.5 +Copyright (c) now authors. +You may obtain the source code and detailed information about this component at https://github.com/jinzhu/now. + +92. ajv-8.17.1 +Copyright (c) ajv authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/ajv. + +93. proxy-from-env-1.1.0 +Copyright (c) proxy-from-env authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/proxy-from-env. + +94. chrome-trace-event-1.0.4 +Copyright (c) chrome-trace-event authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/chrome-trace-event. + +95. axios-1.10.0 +Copyright (c) axios authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/axios. + +96. gotenv-1.6.0 +Copyright (c) gotenv authors. +You may obtain the source code and detailed information about this component at https://github.com/subosito/gotenv. + +97. brace-expansion-2.0.2 +Copyright (c) brace-expansion authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/brace-expansion. + +98. function-bind-1.1.2 +Copyright (c) function-bind authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/function-bind. + +99. @microsoft/fetch-event-source-2.0.1 +Copyright (c) @microsoft/fetch-event-source authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@microsoft/fetch-event-source. + +100. mxj-1.8.4 +Copyright (c) mxj authors. +You may obtain the source code and detailed information about this component at https://github.com/clbanning/mxj. + +101. schema-utils-4.3.2 +Copyright (c) schema-utils authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/schema-utils. + +102. muggle-string-0.4.1 +Copyright (c) muggle-string authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/muggle-string. + +103. go-urn-1.4.0 +Copyright (c) go-urn authors. +You may obtain the source code and detailed information about this component at https://github.com/leodido/go-urn. + +104. form-data-4.0.3 +Copyright (c) form-data authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/form-data. + +105. locales-0.14.1 +Copyright (c) locales authors. +You may obtain the source code and detailed information about this component at https://gitee.com/mirrors/locales. + +106. python-docx-default +Copyright (c) python-docx authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/python-docx. + +107. less-loader-12.3.0 +Copyright (c) less-loader authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/less-loader. + +108. @jridgewell/sourcemap-codec-1.5.4 +Copyright (c) @jridgewell/sourcemap-codec authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@jridgewell/sourcemap-codec. + +109. universal-translator-0.18.1 +Copyright (c) universal-translator authors. +You may obtain the source code and detailed information about this component at https://github.com/go-playground/universal-translator. + +110. inflection-1.0.0 +Copyright (c) inflection authors. +You may obtain the source code and detailed information about this component at https://github.com/jinzhu/inflection. + +111. sass-embedded-1.70.0 +Copyright (c) sass-embedded authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/sass-embedded. + +112. @jridgewell/resolve-uri-3.1.2 +Copyright (c) @jridgewell/resolve-uri authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@jridgewell/resolve-uri. + +113. tinycolor2-1.6.0 +Copyright (c) tinycolor2 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/tinycolor2. + +114. pagefind-1.3.0 +Copyright (c) pagefind authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/pagefind. + +115. viper-1.20.1 +Copyright (c) viper authors. +You may obtain the source code and detailed information about this component at https://github.com/spf13/viper. + +116. delayed-stream-1.0.0 +Copyright (c) delayed-stream authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/delayed-stream. + +117. @webassemblyjs/helper-numbers-1.13.2 +Copyright (c) @webassemblyjs/helper-numbers authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/helper-numbers. + +118. nanoid-3.3.11 +Copyright (c) nanoid authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/nanoid. + +119. @babel/helper-compilation-targets-7.27.2 +Copyright (c) @babel/helper-compilation-targets authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/helper-compilation-targets. + +120. @webassemblyjs/ast-1.14.1 +Copyright (c) @webassemblyjs/ast authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/ast. + +121. @types/estree-1.0.8 +Copyright (c) @types/estree authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/estree. + +122. uuid-1.6.0 +Copyright (c) uuid authors. +You may obtain the source code and detailed information about this component at https://kojipkgs.fedoraproject.org//vol/fedora_koji_archive00/packages/uuid/1.6.0/2.fc8/src/uuid-1.6.0-2.fc8.src.rpm. + +123. postgres-1.5.11 +Copyright (c) postgres authors. +You may obtain the source code and detailed information about this component at https://github.com/go-gorm/postgres. + +124. perfect-debounce-1.0.0 +Copyright (c) perfect-debounce authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/perfect-debounce. + +125. md5-simd-1.1.2 +Copyright (c) md5-simd authors. +You may obtain the source code and detailed information about this component at https://github.com/minio/md5-simd. + +126. ms-2.1.3 +Copyright (c) ms authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/ms. + +127. @babel/helper-module-transforms-7.27.3 +Copyright (c) @babel/helper-module-transforms authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/helper-module-transforms. + +128. @types/json-schema-7.0.15 +Copyright (c) @types/json-schema authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@types/json-schema. + +129. multierr-1.11.0 +Copyright (c) multierr authors. +You may obtain the source code and detailed information about this component at https://github.com/uber-go/multierr. + +130. pflag-1.0.6 +Copyright (c) pflag authors. +You may obtain the source code and detailed information about this component at https://github.com/php-toolkit/pflag. + +131. @webassemblyjs/ieee754-1.13.2 +Copyright (c) @webassemblyjs/ieee754 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/ieee754. + +132. browserslist-4.25.1 +Copyright (c) browserslist authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/browserslist. + +133. asynckit-0.4.0 +Copyright (c) asynckit authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/asynckit. + +134. sortablejs-1.15.6 +Copyright (c) sortablejs authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/sortablejs. + +135. get-intrinsic-1.3.0 +Copyright (c) get-intrinsic authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/get-intrinsic. + +136. markdownify-default +Copyright (c) markdownify authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/markdownify. + +137. go-json-0.10.5 +Copyright (c) go-json authors. +You may obtain the source code and detailed information about this component at https://github.com/goccy/go-json. + +138. path-browserify-1.0.1 +Copyright (c) path-browserify authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/path-browserify. + +139. terser-webpack-plugin-5.3.14 +Copyright (c) terser-webpack-plugin authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/terser-webpack-plugin. + +140. @babel/helpers-7.27.6 +Copyright (c) @babel/helpers authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/helpers. + +141. @vue/babel-plugin-resolve-type-1.4.0 +Copyright (c) @vue/babel-plugin-resolve-type authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@vue/babel-plugin-resolve-type. + +142. json-parse-even-better-errors-4.0.0 +Copyright (c) json-parse-even-better-errors authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/json-parse-even-better-errors. + +143. csstype-3.1.3 +Copyright (c) csstype authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/csstype. + +144. @babel/helper-plugin-utils-7.27.1 +Copyright (c) @babel/helper-plugin-utils authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/helper-plugin-utils. + +145. balanced-match-1.0.2 +Copyright (c) balanced-match authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/balanced-match. + +146. ansi-styles-6.2.1 +Copyright (c) ansi-styles authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/ansi-styles. + +147. go-humanize-1.0.1 +Copyright (c) go-humanize authors. +You may obtain the source code and detailed information about this component at https://gitee.com/mirrors/go-humanize. + +148. picomatch-4.0.2 +Copyright (c) picomatch authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/picomatch. + +149. @webassemblyjs/helper-wasm-bytecode-1.13.2 +Copyright (c) @webassemblyjs/helper-wasm-bytecode authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@webassemblyjs/helper-wasm-bytecode. + +150. @babel/code-frame-7.27.1 +Copyright (c) @babel/code-frame authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@babel/code-frame. + +151. tsx-4.8.1 +Copyright (c) tsx authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/tsx. + +152. superjson-2.2.2 +Copyright (c) superjson authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/superjson. + +153. require-from-string-2.0.2 +Copyright (c) require-from-string authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/require-from-string. + +154. json5-2.2.3 +Copyright (c) json5 authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/json5. + +155. sass-1.70.0 +Copyright (c) sass authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/sass. + +156. @vue/tsconfig-0.7.0 +Copyright (c) @vue/tsconfig authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/@vue/tsconfig. + +157. openai-default +Copyright (c) openai authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/openai. + +158. urllib3-default +Copyright (c) urllib3 authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/urllib3. + +159. beautifulsoup4-default +Copyright (c) beautifulsoup4 authors. +You may obtain the source code and detailed information about this component at https://pypi.org/project/beautifulsoup4. + +160. pgpassfile-1.0.0 +Copyright (c) pgpassfile authors. +You may obtain the source code and detailed information about this component at https://github.com/jackc/pgpassfile. + +161. source-map-support-0.5.21 +Copyright (c) source-map-support authors. +You may obtain the source code and detailed information about this component at https://www.npmjs.com/package/source-map-support. + + + +Terms of the mit: +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1ac93ab --- /dev/null +++ b/Makefile @@ -0,0 +1,215 @@ +.PHONY: help build run test clean docker-build-app docker-build-docreader docker-build-frontend docker-build-all docker-run migrate-up migrate-down docker-restart docker-stop start-all stop-all start-ollama stop-ollama build-images build-images-app build-images-docreader build-images-frontend clean-images check-env list-containers pull-images show-platform + +# Show help +help: + @echo "WeKnora Makefile 帮助" + @echo "" + @echo "基础命令:" + @echo " build 构建应用" + @echo " run 运行应用" + @echo " test 运行测试" + @echo " clean 清理构建文件" + @echo "" + @echo "Docker 命令:" + @echo " docker-build-app 构建应用 Docker 镜像 (wechatopenai/weknora-app)" + @echo " docker-build-docreader 构建文档读取器镜像 (wechatopenai/weknora-docreader)" + @echo " docker-build-frontend 构建前端镜像 (wechatopenai/weknora-ui)" + @echo " docker-build-all 构建所有 Docker 镜像" + @echo " docker-run 运行 Docker 容器" + @echo " docker-stop 停止 Docker 容器" + @echo " docker-restart 重启 Docker 容器" + @echo "" + @echo "服务管理:" + @echo " start-all 启动所有服务" + @echo " stop-all 停止所有服务" + @echo " start-ollama 仅启动 Ollama 服务" + @echo "" + @echo "镜像构建:" + @echo " build-images 从源码构建所有镜像" + @echo " build-images-app 从源码构建应用镜像" + @echo " build-images-docreader 从源码构建文档读取器镜像" + @echo " build-images-frontend 从源码构建前端镜像" + @echo " clean-images 清理本地镜像" + @echo "" + @echo "数据库:" + @echo " migrate-up 执行数据库迁移" + @echo " migrate-down 回滚数据库迁移" + @echo "" + @echo "开发工具:" + @echo " fmt 格式化代码" + @echo " lint 代码检查" + @echo " deps 安装依赖" + @echo " docs 生成 API 文档" + @echo "" + @echo "环境检查:" + @echo " check-env 检查环境配置" + @echo " list-containers 列出运行中的容器" + @echo " pull-images 拉取最新镜像" + @echo " show-platform 显示当前构建平台" + +# Go related variables +BINARY_NAME=WeKnora +MAIN_PATH=./cmd/server + +# Docker related variables +DOCKER_IMAGE=wechatopenai/weknora-app +DOCKER_TAG=latest + +# Platform detection +ifeq ($(shell uname -m),x86_64) + PLATFORM=linux/amd64 +else ifeq ($(shell uname -m),aarch64) + PLATFORM=linux/arm64 +else ifeq ($(shell uname -m),arm64) + PLATFORM=linux/arm64 +else + PLATFORM=linux/amd64 +endif + +# Build the application +build: + go build -o $(BINARY_NAME) $(MAIN_PATH) + +# Run the application +run: build + ./$(BINARY_NAME) + +# Run tests +test: + go test -v ./... + +# Clean build artifacts +clean: + go clean + rm -f $(BINARY_NAME) + +# Build Docker image +docker-build-app: + @echo "获取版本信息..." + @eval $$(./scripts/get_version.sh env); \ + ./scripts/get_version.sh info; \ + docker build --platform $(PLATFORM) \ + --build-arg VERSION_ARG="$$VERSION" \ + --build-arg COMMIT_ID_ARG="$$COMMIT_ID" \ + --build-arg BUILD_TIME_ARG="$$BUILD_TIME" \ + --build-arg GO_VERSION_ARG="$$GO_VERSION" \ + -f docker/Dockerfile.app -t $(DOCKER_IMAGE):$(DOCKER_TAG) . + +# Build docreader Docker image +docker-build-docreader: + docker build --platform $(PLATFORM) -f docker/Dockerfile.docreader -t wechatopenai/weknora-docreader:latest . + +# Build frontend Docker image +docker-build-frontend: + docker build --platform $(PLATFORM) -f frontend/Dockerfile -t wechatopenai/weknora-ui:latest frontend/ + +# Build all Docker images +docker-build-all: docker-build-app docker-build-docreader docker-build-frontend + +# Run Docker container (传统方式) +docker-run: + docker-compose up + +# 使用新脚本启动所有服务 +start-all: + ./scripts/start_all.sh + +# 使用新脚本仅启动Ollama服务 +start-ollama: + ./scripts/start_all.sh --ollama + +# 使用新脚本仅启动Docker容器 +start-docker: + ./scripts/start_all.sh --docker + +# 使用新脚本停止所有服务 +stop-all: + ./scripts/start_all.sh --stop + +# Stop Docker container (传统方式) +docker-stop: + docker-compose down + +# 从源码构建镜像相关命令 +build-images: + ./scripts/build_images.sh + +build-images-app: + ./scripts/build_images.sh --app + +build-images-docreader: + ./scripts/build_images.sh --docreader + +build-images-frontend: + ./scripts/build_images.sh --frontend + +clean-images: + ./scripts/build_images.sh --clean + +# Restart Docker container (stop, start) +docker-restart: + docker-compose stop -t 60 + docker-compose up + +# Database migrations +migrate-up: + ./scripts/migrate.sh up + +migrate-down: + ./scripts/migrate.sh down + +# Generate API documentation +docs: + swag init -g $(MAIN_PATH)/main.go -o ./docs + +# Format code +fmt: + go fmt ./... + +# Lint code +lint: + golangci-lint run + +# Install dependencies +deps: + go mod download + +# Build for production +build-prod: + VERSION=$${VERSION:-unknown}; \ + COMMIT_ID=$${COMMIT_ID:-unknown}; \ + BUILD_TIME=$${BUILD_TIME:-unknown}; \ + GO_VERSION=$${GO_VERSION:-unknown}; \ + LDFLAGS="-X 'github.com/Tencent/WeKnora/internal/handler.Version=$$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$$GO_VERSION'"; \ + go build -ldflags="-w -s $$LDFLAGS" -o $(BINARY_NAME) $(MAIN_PATH) + +clean-db: + @echo "Cleaning database..." + @if [ $$(docker volume ls -q -f name=weknora_postgres-data) ]; then \ + docker volume rm weknora_postgres-data; \ + fi + @if [ $$(docker volume ls -q -f name=weknora_minio_data) ]; then \ + docker volume rm weknora_minio_data; \ + fi + @if [ $$(docker volume ls -q -f name=weknora_redis_data) ]; then \ + docker volume rm weknora_redis_data; \ + fi + +# Environment check +check-env: + ./scripts/start_all.sh --check + +# List containers +list-containers: + ./scripts/start_all.sh --list + +# Pull latest images +pull-images: + ./scripts/start_all.sh --pull + +# Show current platform +show-platform: + @echo "当前系统架构: $(shell uname -m)" + @echo "Docker构建平台: $(PLATFORM)" + + diff --git a/README.md b/README.md index 8e94dba..2b0929f 100644 --- a/README.md +++ b/README.md @@ -1,164 +1,329 @@ -# C-Cube Intelligent Customer Service System +

+ + WeKnora Logo + +

-## Overview +

+ + 官方网站 + + + 微信对话开放平台 + + + License + + + Version + +

-C-Cube is an enterprise-level AI-powered customer service solution designed to help businesses improve customer service efficiency, reduce operational costs, and provide 24/7 intelligent customer support. +

+| English | 简体中文 | 日本語 | +

-## Key Features +

+

-- **Intelligent Dialogue Engine**: Natural language understanding, intent recognition, and context management -- **Knowledge Base Management**: Smart Q&A system with knowledge graph integration -- **Human-AI Collaboration**: Seamless handoff between AI and human agents -- **Multi-Channel Support**: Web, mobile, WeChat, Enterprise WeChat, DingTalk, phone, and email -- **Analytics & Reporting**: Real-time monitoring, intelligent analysis, and customizable reports -- **Continuous Learning**: Machine learning powered by human agent conversations + [Overview](#-overview) • [Architecture](#-architecture) • [Key Features](#-key-features) • [Getting Started](#-getting-started) • [API Reference](#-api-reference) • [Developer Guide](#-developer-guide) + +

+

-## Quick Start +# 💡 WeKnora - LLM-Powered Document Understanding & Retrieval Framework -### Installation +## 📌 Overview + +[**WeKnora**](https://weknora.weixin.qq.com) is an LLM-powered framework designed for deep document understanding and semantic retrieval, especially for handling complex, heterogeneous documents. + +It adopts a modular architecture that combines multimodal preprocessing, semantic vector indexing, intelligent retrieval, and large language model inference. At its core, WeKnora follows the **RAG (Retrieval-Augmented Generation)** paradigm, enabling high-quality, context-aware answers by combining relevant document chunks with model reasoning. + +**Website:** https://weknora.weixin.qq.com + +## 🔒 Security Notice + +**Important:** Starting from v0.1.3, WeKnora includes login authentication functionality to enhance system security. For production deployments, we strongly recommend: + +- Deploy WeKnora services in internal/private network environments rather than public internet +- Avoid exposing the service directly to public networks to prevent potential information leakage +- Configure proper firewall rules and access controls for your deployment environment +- Regularly update to the latest version for security patches and improvements + +## 🏗️ Architecture + +![weknora-pipeline.png](./docs/images/pipeline.jpg) + +WeKnora employs a modern modular design to build a complete document understanding and retrieval pipeline. The system primarily includes document parsing, vector processing, retrieval engine, and large model inference as core modules, with each component being flexibly configurable and extendable. + +## 🎯 Key Features + +- **🔍 Precise Understanding**: Structured content extraction from PDFs, Word documents, images and more into unified semantic views +- **🧠 Intelligent Reasoning**: Leverages LLMs to understand document context and user intent for accurate Q&A and multi-turn conversations +- **🔧 Flexible Extension**: All components from parsing and embedding to retrieval and generation are decoupled for easy customization +- **⚡ Efficient Retrieval**: Hybrid retrieval strategies combining keywords, vectors, and knowledge graphs +- **🎯 User-Friendly**: Intuitive web interface and standardized APIs for zero technical barriers +- **🔒 Secure & Controlled**: Support for local deployment and private cloud, ensuring complete data sovereignty + +## 📊 Application Scenarios + +| Scenario | Applications | Core Value | +|---------|----------|----------| +| **Enterprise Knowledge Management** | Internal document retrieval, policy Q&A, operation manual search | Improve knowledge discovery efficiency, reduce training costs | +| **Academic Research Analysis** | Paper retrieval, research report analysis, scholarly material organization | Accelerate literature review, assist research decisions | +| **Product Technical Support** | Product manual Q&A, technical documentation search, troubleshooting | Enhance customer service quality, reduce support burden | +| **Legal & Compliance Review** | Contract clause retrieval, regulatory policy search, case analysis | Improve compliance efficiency, reduce legal risks | +| **Medical Knowledge Assistance** | Medical literature retrieval, treatment guideline search, case analysis | Support clinical decisions, improve diagnosis quality | + +## 🧩 Feature Matrix + +| Module | Support | Description | +|---------|---------|------| +| Document Formats | ✅ PDF / Word / Txt / Markdown / Images (with OCR / Caption) | Support for structured and unstructured documents with text extraction from images | +| Embedding Models | ✅ Local models, BGE / GTE APIs, etc. | Customizable embedding models, compatible with local deployment and cloud vector generation APIs | +| Vector DB Integration | ✅ PostgreSQL (pgvector), Elasticsearch | Support for mainstream vector index backends, flexible switching for different retrieval scenarios | +| Retrieval Strategies | ✅ BM25 / Dense Retrieval / GraphRAG | Support for sparse/dense recall and knowledge graph-enhanced retrieval with customizable retrieve-rerank-generate pipelines | +| LLM Integration | ✅ Support for Qwen, DeepSeek, etc., with thinking/non-thinking mode switching | Compatible with local models (e.g., via Ollama) or external API services with flexible inference configuration | +| QA Capabilities | ✅ Context-aware, multi-turn dialogue, prompt templates | Support for complex semantic modeling, instruction control and chain-of-thought Q&A with configurable prompts and context windows | +| E2E Testing | ✅ Retrieval+generation process visualization and metric evaluation | End-to-end testing tools for evaluating recall hit rates, answer coverage, BLEU/ROUGE and other metrics | +| Deployment Modes | ✅ Support for local deployment / Docker images | Meets private, offline deployment and flexible operation requirements | +| User Interfaces | ✅ Web UI + RESTful API | Interactive interface and standard API endpoints, suitable for both developers and business users | + +## 🚀 Getting Started + +### 🛠 Prerequisites + +Make sure the following tools are installed on your system: + +* [Docker](https://www.docker.com/) +* [Docker Compose](https://docs.docker.com/compose/) +* [Git](https://git-scm.com/) + +### 📦 Installation + +#### ① Clone the repository + +```bash +# Clone the main repository +git clone https://github.com/Tencent/WeKnora.git +cd WeKnora +``` + +#### ② Configure environment variables + +```bash +# Copy example env file +cp .env.example .env + +# Edit .env and set required values +# All variables are documented in the .env.example comments +``` + +#### ③ Start the services ```bash -# Clone the repository -git clone https://github.com/Wintercom/c-cube.git +# Start all services (Ollama + backend containers) +./scripts/start_all.sh +# Or +make start-all +``` -# Install dependencies -cd c-cube -npm install +#### ③ Start the services (backup) + +```bash +# Start ollama services (Optional) +ollama serve > /dev/null 2>&1 & # Start the service -npm start +docker compose up -d +``` + +#### ④ Stop the services + +```bash +./scripts/start_all.sh --stop +# Or +make stop-all ``` -### Configuration +### 🌐 Access Services + +Once started, services will be available at: + +* Web UI: `http://localhost` +* Backend API: `http://localhost:8080` +* Jaeger Tracing: `http://localhost:16686` + +### 🔌 Using WeChat Dialog Open Platform + +WeKnora serves as the core technology framework for the [WeChat Dialog Open Platform](https://chatbot.weixin.qq.com), providing a more convenient usage approach: -Create a `.env` file in the root directory: +- **Zero-code Deployment**: Simply upload knowledge to quickly deploy intelligent Q&A services within the WeChat ecosystem, achieving an "ask and answer" experience +- **Efficient Question Management**: Support for categorized management of high-frequency questions, with rich data tools to ensure accurate, reliable, and easily maintainable answers +- **WeChat Ecosystem Integration**: Through the WeChat Dialog Open Platform, WeKnora's intelligent Q&A capabilities can be seamlessly integrated into WeChat Official Accounts, Mini Programs, and other WeChat scenarios, enhancing user interaction experiences -```env -BOT_ID=your_bot_id -API_KEY=your_api_key -DATABASE_URL=your_database_url +### 🔗 Access WeKnora via MCP Server + +#### 1️⃣ Clone the repository +``` +git clone https://github.com/Tencent/WeKnora ``` -### Basic Usage +#### 2️⃣ Configure MCP Server +Configure the MCP client to connect to the server: +```json +{ + "mcpServers": { + "weknora": { + "args": [ + "path/to/WeKnora/mcp-server/run_server.py" + ], + "command": "python", + "env":{ + "WEKNORA_API_KEY":"Enter your WeKnora instance, open developer tools, check the request header x-api-key starting with sk", + "WEKNORA_BASE_URL":"http(s)://your-weknora-address/api/v1" + } + } + } +} +``` -```javascript -// Initialize the chatbot -CCube.init({ - botId: 'your-bot-id', - apiKey: 'your-api-key', - position: 'right', - theme: 'light' -}); +Run directly using stdio command: +``` +pip install weknora-mcp-server +python -m weknora-mcp-server ``` -## Documentation +## 🔧 Initialization Configuration Guide -For complete product documentation in Chinese, please refer to [产品文档.md](./产品文档.md) +To help users quickly configure various models and reduce trial-and-error costs, we've improved the original configuration file initialization method by adding a Web UI interface for model configuration. Before using, please ensure the code is updated to the latest version. The specific steps are as follows: +If this is your first time using this project, you can skip steps ①② and go directly to steps ③④. -### Documentation Contents +### ① Stop the services -1. Product Overview -2. Core Features -3. Technical Architecture -4. Deployment Options -5. User Guide -6. API Documentation -7. Security & Compliance -8. FAQ +```bash +./scripts/start_all.sh --stop +``` -## Architecture +### ② Clear existing data tables (recommended when no important data exists) +```bash +make clean-db ``` -┌─────────────────────────────────────────────────────────────┐ -│ Access Layer │ -│ Web │ APP │ WeChat │ Enterprise WeChat │ Phone │ Email │ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ API Gateway │ -│ Load Balancing │ Auth │ Rate Limiting │ Logging │ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ Business Service Layer │ -│ Dialogue │ Session │ User │ Permission Management │ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ AI Service Layer │ -│ NLU │ Dialogue Management │ Knowledge Graph │ Recommendation│ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ Data Layer │ -│ MySQL │ Redis │ MongoDB │ Elasticsearch │ HDFS │ -└─────────────────────────────────────────────────────────────┘ + +### ③ Compile and start services + +```bash +./scripts/start_all.sh ``` -## Technology Stack +### ④ Access Web UI -### Backend -- Go, Python -- Gin, FastAPI -- Kafka, RabbitMQ -- Redis, MySQL, MongoDB +http://localhost -### Frontend -- React, Vue.js -- Ant Design, Element UI -- WebSocket, Socket.io +On first access, it will automatically redirect to the initialization configuration page. After configuration is complete, it will automatically redirect to the knowledge base page. Please follow the page instructions to complete model configuration. -### AI -- PyTorch, TensorFlow -- BERT, GPT, T5 -- Faiss, Milvus +![Configuration Page](./docs/images/config.png) -## API Reference +## 📱 Interface Showcase -### Send Message +### Web UI Interface -```http -POST /api/v1/chat/send -Authorization: Bearer YOUR_API_KEY -Content-Type: application/json + + + + + + + + +
Knowledge Upload
Knowledge Upload Interface		Q&A Entry
Q&A Entry Interface
Rich Text & Image Responses
Rich Answer Interface
-{ - "bot_id": "bot_123456", - "user_id": "user_789", - "message": "I want to check my order status", - "session_id": "session_abc" -} +**Knowledge Base Management:** Support for dragging and dropping various documents, automatically identifying document structures and extracting core knowledge to establish indexes. The system clearly displays processing progress and document status, achieving efficient knowledge base management. + +### Document Knowledge Graph + + + + + + +
Knowledge Graph View 1Knowledge Graph View 2
+ +WeKnora supports transforming documents into knowledge graphs, displaying the relationships between different sections of the documents. Once the knowledge graph feature is enabled, the system analyzes and constructs an internal semantic association network that not only helps users understand document content but also provides structured support for indexing and retrieval, enhancing the relevance and breadth of search results. + +### MCP Server Integration Effects +MCP Server Integration Demo + +## 📘 API Reference + +Troubleshooting FAQ: [Troubleshooting FAQ](./docs/QA.md) + +Detailed API documentation is available at: [API Docs](./docs/API.md) + +## 🧭 Developer Guide + +### 📁 Directory Structure + +``` +WeKnora/ +├── cmd/ # Main entry point +├── internal/ # Core business logic +├── config/ # Configuration files +├── migrations/ # DB migration scripts +├── scripts/ # Shell scripts +├── services/ # Microservice logic +├── frontend/ # Frontend app +└── docs/ # Project documentation ``` -### Query Chat History +### 🔧 Common Commands -```http -GET /api/v1/chat/history?session_id=session_abc&page=1&page_size=20 -Authorization: Bearer YOUR_API_KEY +```bash +# Wipe all data from DB (use with caution) +make clean-db ``` -## Security +## 🤝 Contributing + +We welcome community contributions! For suggestions, bugs, or feature requests, please submit an [Issue](https://github.com/Tencent/WeKnora/issues) or directly create a Pull Request. -- Data encryption (TLS 1.3, AES-256) -- Multi-tenant data isolation -- Role-based access control -- ISO 27001, SOC 2 Type II certified -- GDPR compliant +### 🎯 How to Contribute -## Contributing +- 🐛 **Bug Fixes**: Discover and fix system defects +- ✨ **New Features**: Propose and implement new capabilities +- 📚 **Documentation**: Improve project documentation +- 🧪 **Test Cases**: Write unit and integration tests +- 🎨 **UI/UX Enhancements**: Improve user interface and experience -We welcome contributions! Please see our [Contributing Guide](./CONTRIBUTING.md) for details. +### 📋 Contribution Process -## License +1. **Fork the project** to your GitHub account +2. **Create a feature branch** `git checkout -b feature/amazing-feature` +3. **Commit changes** `git commit -m 'Add amazing feature'` +4. **Push branch** `git push origin feature/amazing-feature` +5. **Create a Pull Request** with detailed description of changes -Copyright © 2024 C-Cube. All rights reserved. +### 🎨 Code Standards -## Contact +- Follow [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) +- Format code using `gofmt` +- Add necessary unit tests +- Update relevant documentation -- Website: https://www.c-cube.ai -- Email: support@c-cube.ai -- Sales: sales@c-cube.ai +### 📝 Commit Guidelines + +Use [Conventional Commits](https://www.conventionalcommits.org/) standard: + +``` +feat: Add document batch upload functionality +fix: Resolve vector retrieval precision issue +docs: Update API documentation +test: Add retrieval engine test cases +refactor: Restructure document parsing module +``` ---- +## 📄 License -For detailed Chinese documentation, please refer to [产品文档.md](./产品文档.md) +This project is licensed under the [MIT License](./LICENSE). +You are free to use, modify, and distribute the code with proper attribution. diff --git a/README_CN.md b/README_CN.md new file mode 100644 index 0000000..5903a72 --- /dev/null +++ b/README_CN.md @@ -0,0 +1,326 @@ +

+ + WeKnora Logo + +

+ +

+ + 官方网站 + + + 微信对话开放平台 + + + License + + + 版本 + +

+ +

+| English | 简体中文 | 日本語 | +

+ +

+

+ + [项目介绍](#-项目介绍) • [架构设计](#-架构设计) • [核心特性](#-核心特性) • [快速开始](#-快速开始) • [文档](#-文档) • [开发指南](#-开发指南) + +

+

+ +# 💡 WeKnora - 基于大模型的文档理解检索框架 + +## 📌 项目介绍 + +[**WeKnora（维娜拉）**](https://weknora.weixin.qq.com) 是一款基于大语言模型（LLM）的文档理解与语义检索框架，专为结构复杂、内容异构的文档场景而打造。 + +框架采用模块化架构，融合多模态预处理、语义向量索引、智能召回与大模型生成推理，构建起高效、可控的文档问答流程。核心检索流程基于 **RAG（Retrieval-Augmented Generation）** 机制，将上下文相关片段与语言模型结合，实现更高质量的语义回答。 + +**官网：** https://weknora.weixin.qq.com + +## 🔒 安全声明 + +**重要提示：** 从 v0.1.3 版本开始，WeKnora 提供了登录鉴权功能，以增强系统安全性。在生产环境部署时，我们强烈建议： + +- 将 WeKnora 服务部署在内网/私有网络环境中，而非公网环境 +- 避免将服务直接暴露在公网上，以防止重要信息泄露风险 +- 为部署环境配置适当的防火墙规则和访问控制 +- 定期更新到最新版本以获取安全补丁和改进 + +## 🏗️ 架构设计 + +![weknora-pipelone.png](./docs/images/pipeline.jpg) + +WeKnora 采用现代化模块化设计，构建了一条完整的文档理解与检索流水线。系统主要包括文档解析、向量化处理、检索引擎和大模型推理等核心模块，每个组件均可灵活配置与扩展。 + +## 🎯 核心特性 + +- **🔍 精准理解**：支持 PDF、Word、图片等文档的结构化内容提取，统一构建语义视图 +- **🧠 智能推理**：借助大语言模型理解文档上下文与用户意图，支持精准问答与多轮对话 +- **🔧 灵活扩展**：从解析、嵌入、召回到生成全流程解耦，便于灵活集成与定制扩展 +- **⚡ 高效检索**：混合多种检索策略：关键词、向量、知识图谱 +- **🎯 简单易用**：直观的Web界面与标准API，零技术门槛快速上手 +- **🔒 安全可控**：支持本地化与私有云部署，数据完全自主可控 + +## 📊 适用场景 + +| 应用场景 | 具体应用 | 核心价值 | +|---------|----------|----------| +| **企业知识管理** | 内部文档检索、规章制度问答、操作手册查询 | 提升知识查找效率，降低培训成本 | +| **科研文献分析** | 论文检索、研究报告分析、学术资料整理 | 加速文献调研，辅助研究决策 | +| **产品技术支持** | 产品手册问答、技术文档检索、故障排查 | 提升客户服务质量，减少技术支持负担 | +| **法律合规审查** | 合同条款检索、法规政策查询、案例分析 | 提高合规效率，降低法律风险 | +| **医疗知识辅助** | 医学文献检索、诊疗指南查询、病例分析 | 辅助临床决策，提升诊疗质量 | + +## 🧩 功能模块能力 + +| 功能模块 | 支持情况 | 说明 | +|---------|---------|------| +| 文档格式支持 | ✅ PDF / Word / Txt / Markdown / 图片（含 OCR / Caption） | 支持多种结构化与非结构化文档内容解析，支持图文混排与图像文字提取 | +| 嵌入模型支持 | ✅ 本地模型、BGE / GTE API 等 | 支持自定义 embedding 模型，兼容本地部署与云端向量生成接口 | +| 向量数据库接入 | ✅ PostgreSQL（pgvector）、Elasticsearch | 支持主流向量索引后端，可灵活切换与扩展，适配不同检索场景 | +| 检索机制 | ✅ BM25 / Dense Retrieve / GraphRAG | 支持稠密/稀疏召回、知识图谱增强检索等多种策略，可自由组合召回-重排-生成流程 | +| 大模型集成 | ✅ 支持 Qwen、DeepSeek 等，思考/非思考模式切换 | 可接入本地大模型（如 Ollama 启动）或调用外部 API 服务，支持推理模式灵活配置 | +| 问答能力 | ✅ 上下文感知、多轮对话、提示词模板 | 支持复杂语义建模、指令控制与链式问答，可配置提示词与上下文窗口 | +| 端到端测试支持 | ✅ 检索+生成过程可视化与指标评估 | 提供一体化链路测试工具，支持评估召回命中率、回答覆盖度、BLEU / ROUGE 等主流指标 | +| 部署模式 | ✅ 支持本地部署 / Docker 镜像 | 满足私有化、离线部署与灵活运维的需求 | +| 用户界面 | ✅ Web UI + RESTful API | 提供交互式界面与标准 API 接口，适配开发者与业务用户使用习惯 | + +## 🚀 快速开始 + +### 🛠 环境要求 + +确保本地已安装以下工具： + +* [Docker](https://www.docker.com/) +* [Docker Compose](https://docs.docker.com/compose/) +* [Git](https://git-scm.com/) + +### 📦 安装步骤 + +#### ① 克隆代码仓库 + +```bash +# 克隆主仓库 +git clone https://github.com/Tencent/WeKnora.git +cd WeKnora +``` + +#### ② 配置环境变量 + +```bash +# 复制示例配置文件 +cp .env.example .env + +# 编辑 .env，填入对应配置信息 +# 所有变量说明详见 .env.example 注释 +``` + +#### ③ 启动服务 + +```bash +# 启动全部服务（含 Ollama 与后端容器） +./scripts/start_all.sh +# 或 +make start-all +``` + +#### ③ 启动服务备选 + +```bash +# 启动 ollama 服务 (可选) +ollama serve > /dev/null 2>&1 & + +# 启动服务 +docker compose up -d +``` + +#### ④ 停止服务 + +```bash +./scripts/start_all.sh --stop +# 或 +make stop-all +``` + +### 🌐 服务访问地址 + +启动成功后，可访问以下地址： + +* Web UI：`http://localhost` +* 后端 API：`http://localhost:8080` +* 链路追踪（Jaeger）：`http://localhost:16686` + +### 🔌 使用微信对话开放平台 + +WeKnora 作为[微信对话开放平台](https://chatbot.weixin.qq.com)的核心技术框架，提供更简便的使用方式： + +- **零代码部署**：只需上传知识，即可在微信生态中快速部署智能问答服务，实现"即问即答"的体验 +- **高效问题管理**：支持高频问题的独立分类管理，提供丰富的数据工具，确保回答精准可靠且易于维护 +- **微信生态覆盖**：通过微信对话开放平台，WeKnora 的智能问答能力可无缝集成到公众号、小程序等微信场景中，提升用户交互体验 +### 🔗MCP服务器访问已经部署好的WEKnora +#### 1️⃣克隆储存库 +``` +git clone https://github.com/Tencent/WeKnora +``` +#### 2️⃣配置MCP服务器 +mcp客户端配置服务器 +```json +{ + "mcpServers": { + "weknora": { + "args": [ + "path/to/WeKnora/mcp-server/run_server.py" + ], + "command": "python", + "env":{ + "WEKNORA_API_KEY":"进入你的weknora实例，打开开发者工具，查看请求头x-api-key，以sk开头", + "WEKNORA_BASE_URL":"http(s)://你的weknora地址/api/v1" + } + } + } +} +``` +使用stdio命令直接运行 +``` +pip install weknora-mcp-server +python -m weknora-mcp-server +``` + +## 🔧 初始化配置引导 + +为了方便用户快速配置各类模型，降低试错成本，我们改进了原来的配置文件初始化方式，增加了Web UI界面进行各种模型的配置。在使用之前，请确保代码更新到最新版本。具体使用步骤如下： +如果是第一次使用本项目，可跳过①②步骤，直接进入③④步骤。 + +### ① 关闭服务 + +```bash +./scripts/start_all.sh --stop +``` + +### ② 清空原有数据表（建议在没有重要数据的情况下使用） + +```bash +make clean-db +``` + +### ③ 编译并启动服务 + +```bash +./scripts/start_all.sh +``` + +### ④ 访问Web UI + +http://localhost + +首次访问会自动跳转到初始化配置页面，配置完成后会自动跳转到知识库页面。请按照页面提示信息完成模型的配置。 + +![配置页面](./docs/images/config.png) + + +## 📱 功能展示 + +### Web UI 界面 + + + + + + + + + +
知识上传
知识上传界面		知识问答入口
知识问答入口
图文结果回答
图文结果回答
+ +**知识库管理：** 支持拖拽上传各类文档，自动识别文档结构并提取核心知识，建立索引。系统清晰展示处理进度和文档状态，实现高效的知识库管理。 + +### 文档知识图谱 + + + + + + +
知识图谱展示1知识图谱展示2
+ +WeKnora 支持将文档转化为知识图谱，展示文档中不同段落之间的关联关系。开启知识图谱功能后，系统会分析并构建文档内部的语义关联网络，不仅帮助用户理解文档内容，还为索引和检索提供结构化支撑，提升检索结果的相关性和广度。 +### 配套MCP服务器调用效果 +118d078426f42f3d4983c13386085d7f + + +## 📘 文档 + +常见问题排查：[常见问题排查](./docs/QA.md) + +详细接口说明请参考：[API 文档](./docs/API.md) + +## 🧭 开发指南 + +### 📁 项目目录结构 + +``` +WeKnora/ +├── cmd/ # 应用入口 +├── internal/ # 核心业务逻辑 +├── config/ # 配置文件 +├── migrations/ # 数据库迁移脚本 +├── scripts/ # 启动与工具脚本 +├── services/ # 各子服务实现 +├── frontend/ # 前端项目 +└── docs/ # 项目文档 +``` + +### 🔧 常用命令 + +```bash +# 清空数据库（慎用！） +make clean-db +``` + +## 🤝 贡献指南 + +我们欢迎社区用户参与贡献！如有建议、Bug 或新功能需求，请通过 [Issue](https://github.com/Tencent/WeKnora/issues) 提出，或直接提交 Pull Request。 + +### 🎯 贡献方式 + +- 🐛 **Bug修复**: 发现并修复系统缺陷 +- ✨ **新功能**: 提出并实现新特性 +- 📚 **文档改进**: 完善项目文档 +- 🧪 **测试用例**: 编写单元测试和集成测试 +- 🎨 **UI/UX优化**: 改进用户界面和体验 + +### 📋 贡献流程 + +1. **Fork项目** 到你的GitHub账户 +2. **创建特性分支** `git checkout -b feature/amazing-feature` +3. **提交更改** `git commit -m 'Add amazing feature'` +4. **推送分支** `git push origin feature/amazing-feature` +5. **创建Pull Request** 并详细描述变更内容 + +### 🎨 代码规范 + +- 遵循 [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments) +- 使用 `gofmt` 格式化代码 +- 添加必要的单元测试 +- 更新相关文档 + +### 📝 提交规范 + +使用 [Conventional Commits](https://www.conventionalcommits.org/) 规范： + +``` +feat: 添加文档批量上传功能 +fix: 修复向量检索精度问题 +docs: 更新API文档 +test: 添加检索引擎测试用例 +refactor: 重构文档解析模块 +``` + +## 📄 许可证 + +本项目基于 [MIT](./LICENSE) 协议发布。 +你可以自由使用、修改和分发本项目代码，但需保留原始版权声明。 diff --git a/README_JA.md b/README_JA.md new file mode 100644 index 0000000..b7cdfc4 --- /dev/null +++ b/README_JA.md @@ -0,0 +1,328 @@ +

+ + WeKnora Logo + +

+ +

+ + 公式サイト + + + WeChat対話オープンプラットフォーム + + + License + + + バージョン + +

+ +

+| English | 简体中文 | 日本語 | +

+ +

+

+ + [プロジェクト紹介](#-プロジェクト紹介) • [アーキテクチャ設計](#️-アーキテクチャ設計) • [コア機能](#-コア機能) • [クイックスタート](#-クイックスタート) • [ドキュメント](#-ドキュメント) • [開発ガイド](#-開発ガイド) + +

+

+ +# 💡 WeKnora - 大規模言語モデルベースの文書理解検索フレームワーク + +## 📌 プロジェクト紹介 + +[**WeKnora（ウィーノラ）**](https://weknora.weixin.qq.com) は、大規模言語モデル（LLM）をベースとした文書理解と意味検索フレームワークで、構造が複雑で内容が異質な文書シナリオ向けに特別に設計されています。 + +フレームワークはモジュラーアーキテクチャを採用し、マルチモーダル前処理、意味ベクトルインデックス、インテリジェント検索、大規模モデル生成推論を統合して、効率的で制御可能な文書Q&Aワークフローを構築します。コア検索プロセスは **RAG（Retrieval-Augmented Generation）** メカニズムに基づいており、文脈関連フラグメントと言語モデルを組み合わせて、より高品質な意味的回答を実現します。 + +**公式サイト：** https://weknora.weixin.qq.com + +## 🔒 セキュリティ通知 + +**重要：** v0.1.3バージョンより、WeKnoraにはシステムセキュリティを強化するためのログイン認証機能が含まれています。本番環境でのデプロイメントにおいて、以下を強く推奨します： + +- WeKnoraサービスはパブリックインターネットではなく、内部/プライベートネットワーク環境にデプロイしてください +- 重要な情報漏洩を防ぐため、サービスを直接パブリックネットワークに公開することは避けてください +- デプロイメント環境に適切なファイアウォールルールとアクセス制御を設定してください +- セキュリティパッチと改善のため、定期的に最新バージョンに更新してください + +## 🏗️ アーキテクチャ設計 + +![weknora-pipelone.png](./docs/images/pipeline.jpg) + +WeKnoraは現代的なモジュラー設計を採用し、完全な文書理解と検索パイプラインを構築しています。システムには主に文書解析、ベクトル化処理、検索エンジン、大規模モデル推論などのコアモジュールが含まれ、各コンポーネントは柔軟に設定および拡張できます。 + +## 🎯 コア機能 + +- **🔍 正確な理解**：PDF、Word、画像などの文書の構造化コンテンツ抽出をサポートし、統一された意味ビューを構築 +- **🧠 インテリジェント推論**：大規模言語モデルを活用して文書コンテキストとユーザーの意図を理解し、正確なQ&Aとマルチターン対話をサポート +- **🔧 柔軟な拡張**：解析、埋め込み、検索から生成までの全プロセスを分離し、柔軟な統合とカスタマイズ拡張を容易に +- **⚡ 効率的な検索**：複数の検索戦略のハイブリッド：キーワード、ベクトル、ナレッジグラフ +- **🎯 使いやすさ**：直感的なWebインターフェースと標準API、技術的な障壁なしで素早く開始可能 +- **🔒 セキュアで制御可能**：ローカルおよびプライベートクラウドデプロイメントをサポート、データは完全に自己管理可能 + +## 📊 適用シナリオ + +| 応用シナリオ | 具体的な応用 | コア価値 | +|---------|----------|----------| +| **企業ナレッジ管理** | 内部文書検索、規則Q&A、操作マニュアル照会 | ナレッジ検索効率の向上、トレーニングコストの削減 | +| **科学研究文献分析** | 論文検索、研究レポート分析、学術資料整理 | 文献調査の加速、研究意思決定の支援 | +| **製品技術サポート** | 製品マニュアルQ&A、技術文書検索、トラブルシューティング | カスタマーサービス品質の向上、技術サポート負担の軽減 | +| **法的コンプライアンス審査** | 契約条項検索、法規政策照会、ケース分析 | コンプライアンス効率の向上、法的リスクの削減 | +| **医療知識支援** | 医学文献検索、診療ガイドライン照会、症例分析 | 臨床意思決定の支援、診療品質の向上 | + +## 🧩 機能モジュール能力 + +| 機能モジュール | サポート状況 | 説明 | +|---------|---------|------| +| 文書フォーマットサポート | ✅ PDF / Word / Txt / Markdown / 画像（OCR / Caption含む） | 様々な構造化・非構造化文書コンテンツの解析をサポート、図文混在と画像文字抽出をサポート | +| 埋め込みモデルサポート | ✅ ローカルモデル、BGE / GTE API等 | カスタムembeddingモデルをサポート、ローカルデプロイとクラウドベクトル生成インターフェースに対応 | +| ベクトルデータベース接続 | ✅ PostgreSQL（pgvector）、Elasticsearch | 主流のベクトルインデックスバックエンドをサポート、柔軟な切り替えと拡張が可能、異なる検索シナリオに適応 | +| 検索メカニズム | ✅ BM25 / Dense Retrieve / GraphRAG | 密・疎検索、ナレッジグラフ強化検索など複数の戦略をサポート、検索-再ランキング-生成プロセスを自由に組み合わせ可能 | +| 大規模モデル統合 | ✅ Qwen、DeepSeek等をサポート、思考/非思考モード切り替え | ローカル大規模モデル（Ollama起動など）に接続可能、または外部APIサービスを呼び出し、推論モードの柔軟な設定をサポート | +| Q&A能力 | ✅ コンテキスト認識、マルチターン対話、プロンプトテンプレート | 複雑な意味モデリング、指示制御、チェーンQ&Aをサポート、プロンプトとコンテキストウィンドウを設定可能 | +| エンドツーエンドテストサポート | ✅ 検索+生成プロセスの可視化と指標評価 | 一体化されたリンクテストツールを提供、リコール的中率、回答カバレッジ、BLEU / ROUGE等の主流指標の評価をサポート | +| デプロイメントモード | ✅ ローカルデプロイメント / Dockerイメージをサポート | プライベート化、オフラインデプロイメント、柔軟な運用保守のニーズに対応 | +| ユーザーインターフェース | ✅ Web UI + RESTful API | インタラクティブインターフェースと標準APIインターフェースを提供、開発者とビジネスユーザーの使用習慣に適応 | + +## 🚀 クイックスタート + +### 🛠 環境要件 + +以下のツールがローカルにインストールされていることを確認してください： + +* [Docker](https://www.docker.com/) +* [Docker Compose](https://docs.docker.com/compose/) +* [Git](https://git-scm.com/) + +### 📦 インストール手順 + +#### ① コードリポジトリのクローン + +```bash +# メインリポジトリをクローン +git clone https://github.com/Tencent/WeKnora.git +cd WeKnora +``` + +#### ② 環境変数の設定 + +```bash +# サンプル設定ファイルをコピー +cp .env.example .env + +# .envを編集し、対応する設定情報を入力 +# すべての変数の説明は.env.exampleのコメントを参照 +``` + +#### ③ サービスの起動 + +```bash +# すべてのサービスを起動（Ollamaとバックエンドコンテナを含む） +./scripts/start_all.sh +# または +make start-all +``` + +#### ③ サービス起動の代替方法 + +```bash +# ollamaサービスを起動（オプション） +ollama serve > /dev/null 2>&1 & + +# サービスを起動 +docker compose up -d +``` + +#### ④ サービスの停止 + +```bash +./scripts/start_all.sh --stop +# または +make stop-all +``` + +### 🌐 サービスアクセスアドレス + +起動成功後、以下のアドレスにアクセスできます： + +* Web UI：`http://localhost` +* バックエンドAPI：`http://localhost:8080` +* リンクトレース（Jaeger）：`http://localhost:16686` + +### 🔌 WeChat対話オープンプラットフォームの使用 + +WeKnoraは[WeChat対話オープンプラットフォーム](https://chatbot.weixin.qq.com)のコア技術フレームワークとして、より簡単な使用方法を提供します： + +- **ノーコードデプロイメント**：知識をアップロードするだけで、WeChatエコシステムで迅速にインテリジェントQ&Aサービスをデプロイし、「即座に質問して即座に回答」の体験を実現 +- **効率的な問題管理**：高頻度の問題の独立した分類管理をサポートし、豊富なデータツールを提供して、正確で信頼性が高く、メンテナンスが容易な回答を保証 +- **WeChatエコシステムカバレッジ**：WeChat対話オープンプラットフォームを通じて、WeKnoraのインテリジェントQ&A能力を公式アカウント、ミニプログラムなどのWeChatシナリオにシームレスに統合し、ユーザーインタラクション体験を向上 + +### 🔗MCPサーバーを使用してデプロイ済みのWeKnoraにアクセス +#### 1️⃣リポジトリのクローン +``` +git clone https://github.com/Tencent/WeKnora +``` +#### 2️⃣MCPサーバーの設定 +MCPクライアントでサーバーを設定 +```json +{ + "mcpServers": { + "weknora": { + "args": [ + "path/to/WeKnora/mcp-server/run_server.py" + ], + "command": "python", + "env":{ + "WEKNORA_API_KEY":"WeKnoraインスタンスに入り、開発者ツールを開いて、リクエストヘッダーx-api-keyを確認、skで始まる", + "WEKNORA_BASE_URL":"http(s)://あなたのWeKnoraアドレス/api/v1" + } + } + } +} +``` +stdioコマンドで直接実行 +``` +pip install weknora-mcp-server +python -m weknora-mcp-server +``` + +## 🔧 初期設定ガイド + +ユーザーが各種モデルを素早く設定し、試行錯誤のコストを削減するために、元の設定ファイル初期化方法を改善し、Web UIインターフェースを追加して各種モデルの設定を行えるようにしました。使用前に、コードが最新バージョンに更新されていることを確認してください。具体的な使用手順は以下の通りです： +本プロジェクトを初めて使用する場合は、①②の手順をスキップして、直接③④の手順に進んでください。 + +### ① サービスの停止 + +```bash +./scripts/start_all.sh --stop +``` + +### ② 既存のデータテーブルをクリア（重要なデータがない場合の推奨） + +```bash +make clean-db +``` + +### ③ コンパイルしてサービスを起動 + +```bash +./scripts/start_all.sh +``` + +### ④ Web UIにアクセス + +http://localhost + +初回アクセス時は自動的に初期設定ページにリダイレクトされ、設定完了後は自動的にナレッジベースページにリダイレクトされます。ページの指示に従ってモデルの設定を完了してください。 + +![設定ページ](./docs/images/config.png) + + +## 📱 機能デモ + +### Web UIインターフェース + + + + + + + + + +
ナレッジアップロード
ナレッジアップロードインターフェース		ナレッジQ&Aエントリー
ナレッジQ&Aエントリー
図文結果回答
図文結果回答
+ +**ナレッジベース管理：** 各種文書のドラッグ＆ドロップアップロードをサポートし、文書構造を自動認識してコア知識を抽出し、インデックスを構築します。システムは処理の進行状況と文書のステータスを明確に表示し、効率的なナレッジベース管理を実現します。 + +### 文書ナレッジグラフ + + + + + + +
ナレッジグラフ表示1ナレッジグラフ表示2
+ +WeKnoraは文書をナレッジグラフに変換し、文書内の異なる段落間の関連関係を表示することをサポートします。ナレッジグラフ機能を有効にすると、システムは文書内部の意味関連ネットワークを分析・構築し、ユーザーが文書内容を理解するのを助けるだけでなく、インデックスと検索に構造化サポートを提供し、検索結果の関連性と幅を向上させます。 + +### 対応MCPサーバー呼び出し効果 +118d078426f42f3d4983c13386085d7f + + +## 📘 ドキュメント + +よくある問題の解決：[よくある問題](./docs/QA.md) + +詳細なAPIドキュメントは：[APIドキュメント](./docs/API.md)を参照してください + +## 🧭 開発ガイド + +### 📁 プロジェクトディレクトリ構造 + +``` +WeKnora/ +├── cmd/ # アプリケーションエントリー +├── internal/ # コアビジネスロジック +├── config/ # 設定ファイル +├── migrations/ # データベースマイグレーションスクリプト +├── scripts/ # 起動とツールスクリプト +├── services/ # 各サブサービスの実装 +├── frontend/ # フロントエンドプロジェクト +└── docs/ # プロジェクトドキュメント +``` + +### 🔧 よく使うコマンド + +```bash +# データベースをクリア（注意して使用！） +make clean-db +``` + +## 🤝 貢献ガイド + +コミュニティユーザーの貢献を歓迎します！提案、バグ、新機能のリクエストがある場合は、[Issue](https://github.com/Tencent/WeKnora/issues)を通じて提出するか、直接Pull Requestを提出してください。 + +### 🎯 貢献方法 + +- 🐛 **バグ修正**: システムの欠陥を発見して修正 +- ✨ **新機能**: 新しい機能を提案して実装 +- 📚 **ドキュメント改善**: プロジェクトドキュメントを改善 +- 🧪 **テストケース**: ユニットテストと統合テストを作成 +- 🎨 **UI/UX最適化**: ユーザーインターフェースと体験を改善 + +### 📋 貢献フロー + +1. **プロジェクトをFork** してあなたのGitHubアカウントへ +2. **機能ブランチを作成** `git checkout -b feature/amazing-feature` +3. **変更をコミット** `git commit -m 'Add amazing feature'` +4. **ブランチをプッシュ** `git push origin feature/amazing-feature` +5. **Pull Requestを作成** して変更内容を詳しく説明 + +### 🎨 コード規約 + +- [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments)に従う +- `gofmt`を使用してコードをフォーマット +- 必要なユニットテストを追加 +- 関連ドキュメントを更新 + +### 📝 コミット規約 + +[Conventional Commits](https://www.conventionalcommits.org/)規約を使用： + +``` +feat: 文書バッチアップロード機能を追加 +fix: ベクトル検索精度の問題を修正 +docs: APIドキュメントを更新 +test: 検索エンジンテストケースを追加 +refactor: 文書解析モジュールをリファクタリング +``` + +## 📄 ライセンス + +このプロジェクトは[MIT](./LICENSE)ライセンスの下で公開されています。 +このプロジェクトのコードを自由に使用、変更、配布できますが、元の著作権表示を保持する必要があります。 diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..446ba66 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.4 \ No newline at end of file diff --git a/client/README.md b/client/README.md new file mode 100644 index 0000000..b236055 --- /dev/null +++ b/client/README.md @@ -0,0 +1,183 @@ +# WeKnora HTTP 客户端 + +这个包提供了与WeKnora服务进行交互的客户端库，支持所有基于HTTP的接口调用，使其他模块更方便地集成WeKnora服务，无需直接编写HTTP请求代码。 + +## 主要功能 + +该客户端包含以下主要功能模块： + +1. **会话管理**：创建、获取、更新和删除会话 +2. **知识库管理**：创建、获取、更新和删除知识库 +3. **知识管理**：添加、获取和删除知识内容 +4. **租户管理**：租户的CRUD操作 +5. **知识问答**：支持普通问答和流式问答 +6. **分块管理**：查询、更新和删除知识分块 +7. **消息管理**：获取和删除会话消息 +8. **模型管理**：创建、获取、更新和删除模型 + +## 使用方法 + +### 创建客户端实例 + +```go +import ( + "context" + "github.com/Tencent/WeKnora/internal/client" + "time" +) + +// 创建客户端实例 +apiClient := client.NewClient( + "http://api.example.com", + client.WithToken("your-auth-token"), + client.WithTimeout(30*time.Second), +) +``` + +### 示例：创建知识库并上传文件 + +```go +// 创建知识库 +kb := &client.KnowledgeBase{ + Name: "测试知识库", + Description: "这是一个测试知识库", + ChunkingConfig: client.ChunkingConfig{ + ChunkSize: 500, + ChunkOverlap: 50, + Separators: []string{"\n\n", "\n", ". ", "? ", "! "}, + }, + ImageProcessingConfig: client.ImageProcessingConfig{ + ModelID: "image_model_id", + }, + EmbeddingModelID: "embedding_model_id", + SummaryModelID: "summary_model_id", +} + +kb, err := apiClient.CreateKnowledgeBase(context.Background(), kb) +if err != nil { + // 处理错误 +} + +// 上传知识文件并添加元数据 +metadata := map[string]string{ + "source": "local", + "type": "document", +} +knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), kb.ID, "path/to/file.pdf", metadata) +if err != nil { + // 处理错误 +} +``` + +### 示例：创建会话并进行问答 + +```go +// 创建会话 +sessionRequest := &client.CreateSessionRequest{ + KnowledgeBaseID: knowledgeBaseID, + SessionStrategy: &client.SessionStrategy{ + MaxRounds: 10, + EnableRewrite: true, + FallbackStrategy: "fixed_answer", + FallbackResponse: "抱歉，我无法回答这个问题", + EmbeddingTopK: 5, + KeywordThreshold: 0.5, + VectorThreshold: 0.7, + RerankModelID: "rerank_model_id", + RerankTopK: 3, + RerankThreshold: 0.8, + SummaryModelID: "summary_model_id", + }, +} + +session, err := apiClient.CreateSession(context.Background(), sessionRequest) +if err != nil { + // 处理错误 +} + +// 普通问答 +answer, err := apiClient.KnowledgeQA(context.Background(), session.ID, &client.KnowledgeQARequest{ + Query: "什么是人工智能?", +}) +if err != nil { + // 处理错误 +} + +// 流式问答 +err = apiClient.KnowledgeQAStream(context.Background(), session.ID, "什么是机器学习?", func(response *client.StreamResponse) error { + // 处理每个响应片段 + fmt.Print(response.Content) + return nil +}) +if err != nil { + // 处理错误 +} +``` + +### 示例：管理模型 + +```go +// 创建模型 +modelRequest := &client.CreateModelRequest{ + Name: "测试模型", + Type: client.ModelTypeChat, + Source: client.ModelSourceInternal, + Description: "这是一个测试模型", + Parameters: client.ModelParameters{ + "temperature": 0.7, + "top_p": 0.9, + }, + IsDefault: true, +} +model, err := apiClient.CreateModel(context.Background(), modelRequest) +if err != nil { + // 处理错误 +} + +// 列出所有模型 +models, err := apiClient.ListModels(context.Background()) +if err != nil { + // 处理错误 +} +``` + +### 示例：管理知识分块 + +```go +// 列出知识分块 +chunks, total, err := apiClient.ListKnowledgeChunks(context.Background(), knowledgeID, 1, 10) +if err != nil { + // 处理错误 +} + +// 更新分块 +updateRequest := &client.UpdateChunkRequest{ + Content: "更新后的分块内容", + IsEnabled: true, +} +updatedChunk, err := apiClient.UpdateChunk(context.Background(), knowledgeID, chunkID, updateRequest) +if err != nil { + // 处理错误 +} +``` + +### 示例：获取会话消息 + +```go +// 获取最近消息 +messages, err := apiClient.GetRecentMessages(context.Background(), sessionID, 10) +if err != nil { + // 处理错误 +} + +// 获取指定时间之前的消息 +beforeTime := time.Now().Add(-24 * time.Hour) +olderMessages, err := apiClient.GetMessagesBefore(context.Background(), sessionID, beforeTime, 10) +if err != nil { + // 处理错误 +} +``` + +## 完整示例 + +请参考 `example.go` 文件中的 `ExampleUsage` 函数，其中展示了客户端的完整使用流程。 \ No newline at end of file diff --git a/client/README_EN.md b/client/README_EN.md new file mode 100644 index 0000000..99ff39d --- /dev/null +++ b/client/README_EN.md @@ -0,0 +1,184 @@ +# WeKnora HTTP Client + +This package provides a client library for interacting with WeKnora services, supporting all HTTP-based interface calls, making it easier for other modules to integrate with WeKnora services without having to write HTTP request code directly. + +## Main Features + +The client includes the following main functional modules: + +1. **Session Management**: Create, retrieve, update, and delete sessions +2. **Knowledge Base Management**: Create, retrieve, update, and delete knowledge bases +3. **Knowledge Management**: Add, retrieve, and delete knowledge content +4. **Tenant Management**: CRUD operations for tenants +5. **Knowledge Q&A**: Supports regular Q&A and streaming Q&A +6. **Chunk Management**: Query, update, and delete knowledge chunks +7. **Message Management**: Retrieve and delete session messages +8. **Model Management**: Create, retrieve, update, and delete models +9. **Evaluation Function**: Start evaluation tasks and get evaluation results + +## Usage + +### Creating Client Instance + +```go +import ( + "context" + "github.com/Tencent/WeKnora/internal/client" + "time" +) + +// Create client instance +apiClient := client.NewClient( + "http://api.example.com", + client.WithToken("your-auth-token"), + client.WithTimeout(30*time.Second), +) +``` + +### Example: Create Knowledge Base and Upload File + +```go +// Create knowledge base +kb := &client.KnowledgeBase{ + Name: "Test Knowledge Base", + Description: "This is a test knowledge base", + ChunkingConfig: client.ChunkingConfig{ + ChunkSize: 500, + ChunkOverlap: 50, + Separators: []string{"\n\n", "\n", ". ", "? ", "! "}, + }, + ImageProcessingConfig: client.ImageProcessingConfig{ + ModelID: "image_model_id", + }, + EmbeddingModelID: "embedding_model_id", + SummaryModelID: "summary_model_id", +} + +kb, err := apiClient.CreateKnowledgeBase(context.Background(), kb) +if err != nil { + // Handle error +} + +// Upload knowledge file with metadata +metadata := map[string]string{ + "source": "local", + "type": "document", +} +knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), kb.ID, "path/to/file.pdf", metadata) +if err != nil { + // Handle error +} +``` + +### Example: Create Session and Chat + +```go +// Create session +sessionRequest := &client.CreateSessionRequest{ + KnowledgeBaseID: knowledgeBaseID, + SessionStrategy: &client.SessionStrategy{ + MaxRounds: 10, + EnableRewrite: true, + FallbackStrategy: "fixed_answer", + FallbackResponse: "Sorry, I cannot answer this question", + EmbeddingTopK: 5, + KeywordThreshold: 0.5, + VectorThreshold: 0.7, + RerankModelID: "rerank_model_id", + RerankTopK: 3, + RerankThreshold: 0.8, + SummaryModelID: "summary_model_id", + }, +} + +session, err := apiClient.CreateSession(context.Background(), sessionRequest) +if err != nil { + // Handle error +} + +// Regular Q&A +answer, err := apiClient.KnowledgeQA(context.Background(), session.ID, &client.KnowledgeQARequest{ + Query: "What is artificial intelligence?", +}) +if err != nil { + // Handle error +} + +// Streaming Q&A +err = apiClient.KnowledgeQAStream(context.Background(), session.ID, "What is machine learning?", func(response *client.StreamResponse) error { + // Handle each response chunk + fmt.Print(response.Content) + return nil +}) +if err != nil { + // Handle error +} +``` + +### Example: Managing Models + +```go +// Create model +modelRequest := &client.CreateModelRequest{ + Name: "Test Model", + Type: client.ModelTypeChat, + Source: client.ModelSourceInternal, + Description: "This is a test model", + Parameters: client.ModelParameters{ + "temperature": 0.7, + "top_p": 0.9, + }, + IsDefault: true, +} +model, err := apiClient.CreateModel(context.Background(), modelRequest) +if err != nil { + // Handle error +} + +// List all models +models, err := apiClient.ListModels(context.Background()) +if err != nil { + // Handle error +} +``` + +### Example: Managing Knowledge Chunks + +```go +// List knowledge chunks +chunks, total, err := apiClient.ListKnowledgeChunks(context.Background(), knowledgeID, 1, 10) +if err != nil { + // Handle error +} + +// Update chunk +updateRequest := &client.UpdateChunkRequest{ + Content: "Updated chunk content", + IsEnabled: true, +} +updatedChunk, err := apiClient.UpdateChunk(context.Background(), knowledgeID, chunkID, updateRequest) +if err != nil { + // Handle error +} +``` + +### Example: Getting Session Messages + +```go +// Get recent messages +messages, err := apiClient.GetRecentMessages(context.Background(), sessionID, 10) +if err != nil { + // Handle error +} + +// Get messages before a specific time +beforeTime := time.Now().Add(-24 * time.Hour) +olderMessages, err := apiClient.GetMessagesBefore(context.Background(), sessionID, beforeTime, 10) +if err != nil { + // Handle error +} +``` + +## Complete Example + +Please refer to the `ExampleUsage` function in the `example.go` file, which demonstrates the complete usage flow of the client. diff --git a/client/chunk.go b/client/chunk.go new file mode 100644 index 0000000..6ed5148 --- /dev/null +++ b/client/chunk.go @@ -0,0 +1,171 @@ +// Package client provides the implementation for interacting with the WeKnora API +// This package encapsulates CRUD operations for server resources and provides a friendly interface for callers +// The Chunk related interfaces are used to manage document chunks in the knowledge base +package client + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" +) + +// Chunk represents the information about a document chunk +// Chunks are the basic units of storage and indexing in the knowledge base +type Chunk struct { + ID string `json:"id"` // Unique identifier of the chunk + KnowledgeID string `json:"knowledge_id"` // Identifier of the parent knowledge + TenantID uint `json:"tenant_id"` // Tenant ID + Content string `json:"content"` // Text content of the chunk + Embedding []float32 `json:"embedding"` // Vector embedding representation + ChunkIndex int `json:"chunk_index"` // Index position of chunk in the document + TotalChunks int `json:"total_chunks"` // Total number of chunks in the document + IsEnabled bool `json:"is_enabled"` // Whether this chunk is enabled + StartAt int `json:"start_at"` // Starting position in original text + EndAt int `json:"end_at"` // Ending position in original text + VectorStoreID string `json:"vector_store_id"` // Vector storage ID + KeywordStoreID string `json:"keyword_store_id"` // Keyword storage ID + EmbeddingStatus int `json:"embedding_status"` // Embedding status: 0-unprocessed, 1-processing, 2-completed + ChunkType string `json:"chunk_type"` + ImageInfo string `json:"image_info"` + CreatedAt string `json:"created_at"` // Creation time + UpdatedAt string `json:"updated_at"` // Last update time +} + +// ChunkResponse represents the response for a single chunk +// API response structure containing a single chunk information +type ChunkResponse struct { + Success bool `json:"success"` // Whether operation was successful + Data Chunk `json:"data"` // Chunk data +} + +// ChunkListResponse represents the response for a list of chunks +// API response structure for returning a list of chunks +type ChunkListResponse struct { + Success bool `json:"success"` // Whether operation was successful + Data []Chunk `json:"data"` // List of chunks + Total int64 `json:"total"` // Total count + Page int `json:"page"` // Current page + PageSize int `json:"page_size"` // Items per page +} + +// UpdateChunkRequest represents the request structure for updating a chunk +// Used for requesting chunk information updates +type UpdateChunkRequest struct { + Content string `json:"content"` // Chunk content + Embedding []float32 `json:"embedding"` // Vector embedding + ChunkIndex int `json:"chunk_index"` // Chunk index + IsEnabled bool `json:"is_enabled"` // Whether enabled + StartAt int `json:"start_at"` // Start position + EndAt int `json:"end_at"` // End position +} + +// ListKnowledgeChunks lists all chunks under a knowledge document +// Queries all chunks by knowledge ID with pagination support +// Parameters: +// - ctx: Context +// - knowledgeID: Knowledge ID +// - page: Page number, starts from 1 +// - pageSize: Number of items per page +// +// Returns: +// - []Chunk: List of chunks +// - int64: Total count +// - error: Error information +func (c *Client) ListKnowledgeChunks(ctx context.Context, + knowledgeID string, page int, pageSize int, +) ([]Chunk, int64, error) { + path := fmt.Sprintf("/api/v1/chunks/%s", knowledgeID) + + queryParams := url.Values{} + queryParams.Add("page", strconv.Itoa(page)) + queryParams.Add("page_size", strconv.Itoa(pageSize)) + + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams) + if err != nil { + return nil, 0, err + } + + var response ChunkListResponse + if err := parseResponse(resp, &response); err != nil { + return nil, 0, err + } + + return response.Data, response.Total, nil +} + +// UpdateChunk updates a chunk's information +// Updates information for a specific chunk under a knowledge document +// Parameters: +// - ctx: Context +// - knowledgeID: Knowledge ID +// - chunkID: Chunk ID +// - request: Update request +// +// Returns: +// - *Chunk: Updated chunk +// - error: Error information +func (c *Client) UpdateChunk(ctx context.Context, + knowledgeID string, chunkID string, request *UpdateChunkRequest, +) (*Chunk, error) { + path := fmt.Sprintf("/api/v1/chunks/%s/%s", knowledgeID, chunkID) + resp, err := c.doRequest(ctx, http.MethodPut, path, request, nil) + if err != nil { + return nil, err + } + + var response ChunkResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// DeleteChunk deletes a specific chunk +// Deletes a specific chunk under a knowledge document +// Parameters: +// - ctx: Context +// - knowledgeID: Knowledge ID +// - chunkID: Chunk ID +// +// Returns: +// - error: Error information +func (c *Client) DeleteChunk(ctx context.Context, knowledgeID string, chunkID string) error { + path := fmt.Sprintf("/api/v1/chunks/%s/%s", knowledgeID, chunkID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} + +// DeleteChunksByKnowledgeID deletes all chunks under a knowledge document +// Batch deletes all chunks under the specified knowledge document +// Parameters: +// - ctx: Context +// - knowledgeID: Knowledge ID +// +// Returns: +// - error: Error information +func (c *Client) DeleteChunksByKnowledgeID(ctx context.Context, knowledgeID string) error { + path := fmt.Sprintf("/api/v1/chunks/%s", knowledgeID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..c92a276 --- /dev/null +++ b/client/client.go @@ -0,0 +1,104 @@ +// Package client provides the implementation for interacting with the WeKnora API +// This package encapsulates CRUD operations for server resources and provides a friendly interface for callers +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// Client is the client for interacting with the WeKnora service +type Client struct { + baseURL string + httpClient *http.Client + token string +} + +// ClientOption defines client configuration options +type ClientOption func(*Client) + +// WithTimeout sets the HTTP client timeout +func WithTimeout(timeout time.Duration) ClientOption { + return func(c *Client) { + c.httpClient.Timeout = timeout + } +} + +// WithToken sets the authentication token +func WithToken(token string) ClientOption { + return func(c *Client) { + c.token = token + } +} + +// NewClient creates a new client instance +func NewClient(baseURL string, options ...ClientOption) *Client { + client := &Client{ + baseURL: baseURL, + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } + + for _, option := range options { + option(client) + } + + return client +} + +// doRequest executes an HTTP request +func (c *Client) doRequest(ctx context.Context, + method, path string, body interface{}, query url.Values, +) (*http.Response, error) { + var reqBody io.Reader + if body != nil { + jsonData, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to serialize request body: %w", err) + } + reqBody = bytes.NewBuffer(jsonData) + } + + url := fmt.Sprintf("%s%s", c.baseURL, path) + if len(query) > 0 { + url = fmt.Sprintf("%s?%s", url, query.Encode()) + } + + req, err := http.NewRequestWithContext(ctx, method, url, reqBody) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + if c.token != "" { + req.Header.Set("X-API-Key", c.token) + } + if requestID := ctx.Value("RequestID"); requestID != nil { + req.Header.Set("X-Request-ID", requestID.(string)) + } + + return c.httpClient.Do(req) +} + +// parseResponse parses an HTTP response +func parseResponse(resp *http.Response, target interface{}) error { + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(body)) + } + + if target == nil { + return nil + } + + return json.NewDecoder(resp.Body).Decode(target) +} diff --git a/client/evaluation.go b/client/evaluation.go new file mode 100644 index 0000000..bb312f1 --- /dev/null +++ b/client/evaluation.go @@ -0,0 +1,113 @@ +// Package client provides the implementation for interacting with the WeKnora API +// The Evaluation related interfaces are used for starting and retrieving model evaluation task results +// Evaluation tasks can be used to measure model performance and +// compare different embedding models, chat models, and reranking models +package client + +import ( + "context" + "net/http" + "net/url" +) + +// EvaluationTask represents an evaluation task +// Contains basic information about a model evaluation task +type EvaluationTask struct { + ID string `json:"id"` // Task unique identifier + Status string `json:"status"` // Task status: pending, running, completed, failed + Progress int `json:"progress"` // Task progress, integer value 0-100 + DatasetID string `json:"dataset_id"` // Evaluation dataset ID + EmbeddingID string `json:"embedding_id"` // Embedding model ID + ChatID string `json:"chat_id"` // Chat model ID + RerankID string `json:"rerank_id"` // Reranking model ID + CreatedAt string `json:"created_at"` // Task creation time + CompleteAt string `json:"complete_at"` // Task completion time + ErrorMsg string `json:"error_msg"` // Error message, has value when task fails +} + +// EvaluationResult represents the evaluation results +// Contains detailed evaluation result information +type EvaluationResult struct { + TaskID string `json:"task_id"` // Associated task ID + Status string `json:"status"` // Task status + Progress int `json:"progress"` // Task progress + TotalQueries int `json:"total_queries"` // Total number of queries + TotalSamples int `json:"total_samples"` // Total number of samples + Metrics map[string]float64 `json:"metrics"` // Evaluation metrics collection + QueriesStat []map[string]interface{} `json:"queries_stat"` // Statistics for each query + CreatedAt string `json:"created_at"` // Creation time + CompleteAt string `json:"complete_at"` // Completion time + ErrorMsg string `json:"error_msg"` // Error message +} + +// EvaluationRequest represents an evaluation request +// Parameters used to start a new evaluation task +type EvaluationRequest struct { + DatasetID string `json:"dataset_id"` // Dataset ID to evaluate + EmbeddingModelID string `json:"embedding_id"` // Embedding model ID + ChatModelID string `json:"chat_id"` // Chat model ID + RerankModelID string `json:"rerank_id"` // Reranking model ID +} + +// EvaluationTaskResponse represents an evaluation task response +// API response structure for evaluation tasks +type EvaluationTaskResponse struct { + Success bool `json:"success"` // Whether operation was successful + Data EvaluationTask `json:"data"` // Evaluation task data +} + +// EvaluationResultResponse represents an evaluation result response +// API response structure for evaluation results +type EvaluationResultResponse struct { + Success bool `json:"success"` // Whether operation was successful + Data EvaluationResult `json:"data"` // Evaluation result data +} + +// StartEvaluation starts an evaluation task +// Creates and starts a new evaluation task based on provided parameters +// Parameters: +// - ctx: Context, used for passing request context information such as deadline, cancellation signals, etc. +// - request: Evaluation request parameters, including dataset ID and model IDs +// +// Returns: +// - *EvaluationTask: Created evaluation task information +// - error: Error information if the request fails +func (c *Client) StartEvaluation(ctx context.Context, request *EvaluationRequest) (*EvaluationTask, error) { + resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/evaluation", request, nil) + if err != nil { + return nil, err + } + + var response EvaluationTaskResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetEvaluationResult retrieves evaluation results +// Retrieves detailed results for an evaluation task by task ID +// Parameters: +// - ctx: Context, used for passing request context information +// - taskID: Evaluation task ID, used to identify the specific evaluation task to query +// +// Returns: +// - *EvaluationResult: Detailed evaluation task results +// - error: Error information if the request fails +func (c *Client) GetEvaluationResult(ctx context.Context, taskID string) (*EvaluationResult, error) { + queryParams := url.Values{} + queryParams.Add("task_id", taskID) + + resp, err := c.doRequest(ctx, http.MethodGet, "/api/v1/evaluation", nil, queryParams) + if err != nil { + return nil, err + } + + var response EvaluationResultResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} diff --git a/client/example.go b/client/example.go new file mode 100644 index 0000000..62833ae --- /dev/null +++ b/client/example.go @@ -0,0 +1,271 @@ +package client + +import ( + "context" + "fmt" + "os" + "strings" + "time" +) + +// ExampleUsage demonstrates the complete usage flow of the WeKnora client, including: +// - Creating a client instance +// - Creating a knowledge base +// - Uploading knowledge files +// - Creating a session +// - Performing question-answering +// - Using streaming question-answering +// - Managing models +// - Managing knowledge chunks +// - Getting session messages +// - Cleaning up resources +func ExampleUsage() { + // Create a client instance + apiClient := NewClient( + "http://localhost:8080", + WithToken("your-auth-token"), + WithTimeout(30*time.Second), + ) + + // 1. Create a knowledge base + fmt.Println("1. Creating knowledge base...") + kb := &KnowledgeBase{ + Name: "Test Knowledge Base", + Description: "This is a test knowledge base", + ChunkingConfig: ChunkingConfig{ + ChunkSize: 500, + ChunkOverlap: 50, + Separators: []string{"\n\n", "\n", ". ", "? ", "! "}, + }, + ImageProcessingConfig: ImageProcessingConfig{ + ModelID: "image_model_id", + }, + EmbeddingModelID: "embedding_model_id", + SummaryModelID: "summary_model_id", + } + + createdKB, err := apiClient.CreateKnowledgeBase(context.Background(), kb) + if err != nil { + fmt.Printf("Failed to create knowledge base: %v\n", err) + return + } + fmt.Printf("Knowledge base created successfully: ID=%s, Name=%s\n", createdKB.ID, createdKB.Name) + + // 2. Upload knowledge file + fmt.Println("\n2. Uploading knowledge file...") + filePath := "path/to/sample.pdf" // Sample file path + + // Check if file exists before uploading + if _, err := os.Stat(filePath); os.IsNotExist(err) { + fmt.Printf("File does not exist: %s, skipping upload step\n", filePath) + } else { + // Add metadata + metadata := map[string]string{ + "source": "local", + "type": "document", + } + knowledge, err := apiClient.CreateKnowledgeFromFile(context.Background(), createdKB.ID, filePath, metadata, nil) + if err != nil { + fmt.Printf("Failed to upload knowledge file: %v\n", err) + } else { + fmt.Printf("File uploaded successfully: Knowledge ID=%s, Title=%s\n", knowledge.ID, knowledge.Title) + } + } + + // Create text knowledge (alternative to file upload) + // Note: This is just an example, the client package may not support creating text knowledge directly + // In actual use, refer to the methods provided in client.knowledge.go + fmt.Println("\nCreating text knowledge (example)") + fmt.Println("Title: Test Text Knowledge") + fmt.Println("Description: Test knowledge created from text") + + // 3. Create a model + fmt.Println("\n3. Creating model...") + modelRequest := &CreateModelRequest{ + Name: "Test Model", + Type: ModelTypeChat, + Source: ModelSourceInternal, + Description: "This is a test model", + Parameters: ModelParameters{ + "temperature": 0.7, + "top_p": 0.9, + }, + IsDefault: true, + } + + model, err := apiClient.CreateModel(context.Background(), modelRequest) + if err != nil { + fmt.Printf("Failed to create model: %v\n", err) + } else { + fmt.Printf("Model created successfully: ID=%s, Name=%s\n", model.ID, model.Name) + } + + // List all models + models, err := apiClient.ListModels(context.Background()) + if err != nil { + fmt.Printf("Failed to get model list: %v\n", err) + } else { + fmt.Printf("System has %d models\n", len(models)) + } + + // 4. Create a session + fmt.Println("\n4. Creating session...") + sessionRequest := &CreateSessionRequest{ + KnowledgeBaseID: createdKB.ID, + SessionStrategy: &SessionStrategy{ + MaxRounds: 10, + EnableRewrite: true, + FallbackStrategy: "fixed_answer", + FallbackResponse: "Sorry, I cannot answer this question", + EmbeddingTopK: 5, + KeywordThreshold: 0.5, + VectorThreshold: 0.7, + RerankModelID: "rerank_model_id", + RerankTopK: 3, + RerankThreshold: 0.8, + SummaryModelID: "summary_model_id", + SummaryParameters: &SummaryConfig{ + Temperature: 0.7, + TopP: 0.9, + MaxTokens: 100, + }, + }, + } + + session, err := apiClient.CreateSession(context.Background(), sessionRequest) + if err != nil { + fmt.Printf("Failed to create session: %v\n", err) + return + } + fmt.Printf("Session created successfully: ID=%s\n", session.ID) + + // 5. Perform knowledge Q&A (using streaming API) + fmt.Println("\n5. Performing knowledge Q&A...") + question := "What is artificial intelligence?" + fmt.Printf("Question: %s\nAnswer: ", question) + + // Use streaming API for Q&A (Note: Client may only provide streaming Q&A API) + var answer strings.Builder + var references []*SearchResult + + err = apiClient.KnowledgeQAStream(context.Background(), + session.ID, + question, + func(response *StreamResponse) error { + if response.ResponseType == ResponseTypeAnswer { + answer.WriteString(response.Content) + } + + if response.Done && len(response.KnowledgeReferences) > 0 { + references = response.KnowledgeReferences + } + return nil + }) + + if err != nil { + fmt.Printf("Q&A failed: %v\n", err) + } else { + fmt.Printf("%s\n", answer.String()) + if len(references) > 0 { + fmt.Println("References:") + for i, ref := range references { + fmt.Printf("%d. %s\n", i+1, ref.Content[:min(50, len(ref.Content))]+"...") + } + } + } + + // 6. Perform another streaming Q&A + fmt.Println("\n6. Performing streaming Q&A...") + streamQuestion := "What is machine learning?" + fmt.Printf("Question: %s\nAnswer: ", streamQuestion) + + err = apiClient.KnowledgeQAStream(context.Background(), + session.ID, + streamQuestion, + func(response *StreamResponse) error { + fmt.Print(response.Content) + return nil + }, + ) + if err != nil { + fmt.Printf("\nStreaming Q&A failed: %v\n", err) + } + fmt.Println() // Line break + + // 7. Get session messages + fmt.Println("\n7. Getting session messages...") + messages, err := apiClient.GetRecentMessages(context.Background(), session.ID, 10) + if err != nil { + fmt.Printf("Failed to get session messages: %v\n", err) + } else { + fmt.Printf("Retrieved %d recent messages:\n", len(messages)) + for i, msg := range messages { + fmt.Printf("%d. Role: %s, Content: %s\n", i+1, msg.Role, msg.Content[:min(30, len(msg.Content))]+"...") + } + } + + // 8. Manage knowledge chunks + // Assume we have uploaded knowledge and have a knowledge ID + knowledgeID := "knowledge_id_example" // In actual use, use a real knowledge ID + + fmt.Println("\n8. Managing knowledge chunks...") + chunks, total, err := apiClient.ListKnowledgeChunks(context.Background(), knowledgeID, 1, 10) + if err != nil { + fmt.Printf("Failed to get knowledge chunks: %v\n", err) + } else { + fmt.Printf("Knowledge has %d chunks, retrieved %d chunks\n", total, len(chunks)) + + if len(chunks) > 0 { + // Update the first chunk + chunkID := chunks[0].ID + updateRequest := &UpdateChunkRequest{ + Content: "Updated chunk content - " + chunks[0].Content, + IsEnabled: true, + } + + updatedChunk, err := apiClient.UpdateChunk(context.Background(), knowledgeID, chunkID, updateRequest) + if err != nil { + fmt.Printf("Failed to update chunk: %v\n", err) + } else { + fmt.Printf("Chunk updated successfully: ID=%s\n", updatedChunk.ID) + } + } + } + + // 10. Clean up resources (optional, in actual use, keep or delete as needed) + fmt.Println("\n10. Cleaning up resources...") + if session != nil { + if err := apiClient.DeleteSession(context.Background(), session.ID); err != nil { + fmt.Printf("Failed to delete session: %v\n", err) + } else { + fmt.Println("Session deleted") + } + } + + // Delete knowledge (assuming we have a valid knowledge ID) + if knowledgeID != "" { + if err := apiClient.DeleteKnowledge(context.Background(), knowledgeID); err != nil { + fmt.Printf("Failed to delete knowledge: %v\n", err) + } else { + fmt.Println("Knowledge deleted") + } + } + + if createdKB != nil { + if err := apiClient.DeleteKnowledgeBase(context.Background(), createdKB.ID); err != nil { + fmt.Printf("Failed to delete knowledge base: %v\n", err) + } else { + fmt.Println("Knowledge base deleted") + } + } + + fmt.Println("\nExample completed") +} + +// min returns the smaller of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/client/go.mod b/client/go.mod new file mode 100644 index 0000000..1a20c4b --- /dev/null +++ b/client/go.mod @@ -0,0 +1,3 @@ +module github.com/Tencent/WeKnora/client + +go 1.24.2 diff --git a/client/go.sum b/client/go.sum new file mode 100644 index 0000000..e69de29 diff --git a/client/knowledge.go b/client/knowledge.go new file mode 100644 index 0000000..3a3ea9d --- /dev/null +++ b/client/knowledge.go @@ -0,0 +1,356 @@ +// Package client provides the implementation for interacting with the WeKnora API +// The Knowledge related interfaces are used to manage knowledge entries in the knowledge base +// Knowledge entries can be created from local files, web URLs, or directly from text content +// They can also be retrieved, deleted, and downloaded as files +package client + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "os" + "strconv" + "time" +) + +// Knowledge represents knowledge information +type Knowledge struct { + ID string `json:"id"` + TenantID uint `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + Type string `json:"type"` + Title string `json:"title"` + Description string `json:"description"` + Source string `json:"source"` + ParseStatus string `json:"parse_status"` + EnableStatus string `json:"enable_status"` + EmbeddingModelID string `json:"embedding_model_id"` + FileName string `json:"file_name"` + FileType string `json:"file_type"` + FileSize int64 `json:"file_size"` + FilePath string `json:"file_path"` + Metadata map[string]string `json:"metadata"` // Extensible metadata for storing machine information, paths, etc. + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ProcessedAt *time.Time `json:"processed_at"` + ErrorMessage string `json:"error_message"` +} + +// KnowledgeResponse represents the API response containing a single knowledge entry +type KnowledgeResponse struct { + Success bool `json:"success"` + Data Knowledge `json:"data"` + Code string `json:"code"` + Message string `json:"message"` +} + +// KnowledgeListResponse represents the API response containing a list of knowledge entries with pagination +type KnowledgeListResponse struct { + Success bool `json:"success"` + Data []Knowledge `json:"data"` + Total int64 `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` +} + +// KnowledgeBatchResponse represents the API response for batch knowledge retrieval +type KnowledgeBatchResponse struct { + Success bool `json:"success"` + Data []Knowledge `json:"data"` +} + +// UpdateImageInfoRequest represents the request structure for updating a chunk +// Used for requesting chunk information updates +type UpdateImageInfoRequest struct { + ImageInfo string `json:"image_info"` // Image information in JSON format +} + +// ErrDuplicateFile is returned when attempting to create a knowledge entry with a file that already exists +var ErrDuplicateFile = errors.New("file already exists") + +// ErrDuplicateURL is returned when attempting to create a knowledge entry with a URL that already exists +var ErrDuplicateURL = errors.New("URL already exists") + +// CreateKnowledgeFromFile creates a knowledge entry from a local file path +func (c *Client) CreateKnowledgeFromFile(ctx context.Context, + knowledgeBaseID string, filePath string, metadata map[string]string, enableMultimodel *bool, +) (*Knowledge, error) { + // Open the local file + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + + // Get file information + fileInfo, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("failed to get file information: %w", err) + } + + // Create the HTTP request + path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/file", knowledgeBaseID) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + // Create a multipart form writer + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", fileInfo.Name()) + if err != nil { + return nil, fmt.Errorf("failed to create form file: %w", err) + } + + // Copy file contents + _, err = io.Copy(part, file) + if err != nil { + return nil, fmt.Errorf("failed to copy file content: %w", err) + } + + // Add enable_multimodel field + if enableMultimodel != nil { + if err := writer.WriteField("enable_multimodel", strconv.FormatBool(*enableMultimodel)); err != nil { + return nil, fmt.Errorf("failed to write enable_multimodel field: %w", err) + } + } + + // Add metadata to the request if provided + if metadata != nil { + metadataBytes, err := json.Marshal(metadata) + if err != nil { + return nil, fmt.Errorf("failed to serialize metadata: %w", err) + } + if err := writer.WriteField("metadata", string(metadataBytes)); err != nil { + return nil, fmt.Errorf("failed to write metadata field: %w", err) + } + } + + // Close the multipart writer + err = writer.Close() + if err != nil { + return nil, fmt.Errorf("failed to close writer: %w", err) + } + + // Set request headers + req.Header.Set("Content-Type", writer.FormDataContentType()) + if c.token != "" { + req.Header.Set("X-API-Key", c.token) + } + if requestID := ctx.Value("RequestID"); requestID != nil { + req.Header.Set("X-Request-ID", requestID.(string)) + } + + // Set the request body + req.Body = io.NopCloser(body) + + // Send the request + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + // Parse the response + var response KnowledgeResponse + if resp.StatusCode == http.StatusConflict { + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + return &response.Data, ErrDuplicateFile + } else if err := parseResponse(resp, &response); err != nil { + return nil, err + } + return &response.Data, nil +} + +// CreateKnowledgeFromURL creates a knowledge entry from a web URL +func (c *Client) CreateKnowledgeFromURL(ctx context.Context, knowledgeBaseID string, url string, enableMultimodel *bool) (*Knowledge, error) { + path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/url", knowledgeBaseID) + + reqBody := struct { + URL string `json:"url"` + EnableMultimodel *bool `json:"enable_multimodel"` + }{ + URL: url, + EnableMultimodel: enableMultimodel, + } + + resp, err := c.doRequest(ctx, http.MethodPost, path, reqBody, nil) + if err != nil { + return nil, err + } + + var response KnowledgeResponse + if resp.StatusCode == http.StatusConflict { + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + return &response.Data, ErrDuplicateURL + } else if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetKnowledge retrieves a knowledge entry by its ID +func (c *Client) GetKnowledge(ctx context.Context, knowledgeID string) (*Knowledge, error) { + path := fmt.Sprintf("/api/v1/knowledge/%s", knowledgeID) + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return nil, err + } + + var response KnowledgeResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetKnowledgeBatch retrieves multiple knowledge entries by their IDs +func (c *Client) GetKnowledgeBatch(ctx context.Context, knowledgeIDs []string) ([]Knowledge, error) { + path := "/api/v1/knowledge/batch" + + queryParams := url.Values{} + for _, id := range knowledgeIDs { + queryParams.Add("ids", id) + } + + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams) + if err != nil { + return nil, err + } + + var response KnowledgeBatchResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return response.Data, nil +} + +// ListKnowledge lists knowledge entries in a knowledge base with pagination +func (c *Client) ListKnowledge(ctx context.Context, + knowledgeBaseID string, + page int, + pageSize int, +) ([]Knowledge, int64, error) { + path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge", knowledgeBaseID) + + queryParams := url.Values{} + queryParams.Add("page", strconv.Itoa(page)) + queryParams.Add("page_size", strconv.Itoa(pageSize)) + + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams) + if err != nil { + return nil, 0, err + } + + var response KnowledgeListResponse + if err := parseResponse(resp, &response); err != nil { + return nil, 0, err + } + + return response.Data, response.Total, nil +} + +// DeleteKnowledge deletes a knowledge entry by its ID +func (c *Client) DeleteKnowledge(ctx context.Context, knowledgeID string) error { + path := fmt.Sprintf("/api/v1/knowledge/%s", knowledgeID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} + +// DownloadKnowledgeFile downloads a knowledge file to the specified local path +func (c *Client) DownloadKnowledgeFile(ctx context.Context, knowledgeID string, destPath string) error { + path := fmt.Sprintf("/api/v1/knowledge/%s/download", knowledgeID) + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return err + } + defer resp.Body.Close() + + // Check for HTTP errors + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(body)) + } + + // Create destination file + out, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer out.Close() + + // Copy response body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + return nil +} + +func (c *Client) UpdateKnowledge(ctx context.Context, knowledge *Knowledge) error { + path := fmt.Sprintf("/api/v1/knowledge/%s", knowledge.ID) + + resp, err := c.doRequest(ctx, http.MethodPut, path, knowledge, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} + +// UpdateChunk updates a chunk's information +// Updates information for a specific chunk under a knowledge document +// Parameters: +// - ctx: Context +// - knowledgeID: Knowledge ID +// - chunkID: Chunk ID +// - request: Update request +// +// Returns: +// - *Chunk: Updated chunk +// - error: Error information +func (c *Client) UpdateImageInfo(ctx context.Context, + knowledgeID string, chunkID string, request *UpdateImageInfoRequest, +) error { + path := fmt.Sprintf("/api/v1/knowledge/image/%s/%s", knowledgeID, chunkID) + resp, err := c.doRequest(ctx, http.MethodPut, path, request, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} diff --git a/client/knowledgebase.go b/client/knowledgebase.go new file mode 100644 index 0000000..63aa921 --- /dev/null +++ b/client/knowledgebase.go @@ -0,0 +1,211 @@ +// Package client provides the implementation for interacting with the WeKnora API +// The KnowledgeBase related interfaces are used to manage knowledge bases +// Knowledge bases are collections of knowledge entries that can be used for question-answering +// They can also be searched and queried using hybrid search +package client + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" +) + +// KnowledgeBase represents a knowledge base +type KnowledgeBase struct { + ID string `json:"id"` + Name string `json:"name"` // Name must be unique within the same tenant + Description string `json:"description"` + TenantID uint `json:"tenant_id"` // Changed to uint type + ChunkingConfig ChunkingConfig `json:"chunking_config"` + ImageProcessingConfig ImageProcessingConfig `json:"image_processing_config"` + EmbeddingModelID string `json:"embedding_model_id"` + SummaryModelID string `json:"summary_model_id"` // Summary model ID + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// KnowledgeBaseConfig represents knowledge base configuration +type KnowledgeBaseConfig struct { + ChunkingConfig ChunkingConfig `json:"chunking_config"` + ImageProcessingConfig ImageProcessingConfig `json:"image_processing_config"` +} + +// ChunkingConfig represents document chunking configuration +type ChunkingConfig struct { + ChunkSize int `json:"chunk_size"` // Chunk size + ChunkOverlap int `json:"chunk_overlap"` // Overlap size + Separators []string `json:"separators"` // Separators + EnableMultimodal bool `json:"enable_multimodal"` // Whether to enable multimodal processing +} + +// ImageProcessingConfig represents image processing configuration +type ImageProcessingConfig struct { + ModelID string `json:"model_id"` // Multimodal model ID +} + +// KnowledgeBaseResponse knowledge base response +type KnowledgeBaseResponse struct { + Success bool `json:"success"` + Data KnowledgeBase `json:"data"` +} + +// KnowledgeBaseListResponse knowledge base list response +type KnowledgeBaseListResponse struct { + Success bool `json:"success"` + Data []KnowledgeBase `json:"data"` +} + +// SearchResult represents search result +type SearchResult struct { + ID string `json:"id"` + Content string `json:"content"` + KnowledgeID string `json:"knowledge_id"` + ChunkIndex int `json:"chunk_index"` + KnowledgeTitle string `json:"knowledge_title"` + StartAt int `json:"start_at"` + EndAt int `json:"end_at"` + Seq int `json:"seq"` + Score float64 `json:"score"` + ChunkType string `json:"chunk_type"` + ImageInfo string `json:"image_info"` + Metadata map[string]string `json:"metadata"` + KnowledgeFilename string `json:"knowledge_filename"` + KnowledgeSource string `json:"knowledge_source"` +} + +// HybridSearchResponse hybrid search response +type HybridSearchResponse struct { + Success bool `json:"success"` + Data []*SearchResult `json:"data"` +} + +type CopyKnowledgeBaseRequest struct { + SourceID string `json:"source_id"` + TargetID string `json:"target_id"` +} + +// CreateKnowledgeBase creates a knowledge base +func (c *Client) CreateKnowledgeBase(ctx context.Context, knowledgeBase *KnowledgeBase) (*KnowledgeBase, error) { + resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-bases", knowledgeBase, nil) + if err != nil { + return nil, err + } + + var response KnowledgeBaseResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetKnowledgeBase gets a knowledge base +func (c *Client) GetKnowledgeBase(ctx context.Context, knowledgeBaseID string) (*KnowledgeBase, error) { + path := fmt.Sprintf("/api/v1/knowledge-bases/%s", knowledgeBaseID) + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return nil, err + } + + var response KnowledgeBaseResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// ListKnowledgeBases lists knowledge bases +func (c *Client) ListKnowledgeBases(ctx context.Context) ([]KnowledgeBase, error) { + resp, err := c.doRequest(ctx, http.MethodGet, "/api/v1/knowledge-bases", nil, nil) + if err != nil { + return nil, err + } + + var response KnowledgeBaseListResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return response.Data, nil +} + +// UpdateKnowledgeBaseRequest update knowledge base request +type UpdateKnowledgeBaseRequest struct { + Name string `json:"name"` + Description string `json:"description"` + Config *KnowledgeBaseConfig `json:"config"` +} + +// UpdateKnowledgeBase updates a knowledge base +func (c *Client) UpdateKnowledgeBase(ctx context.Context, + knowledgeBaseID string, + request *UpdateKnowledgeBaseRequest, +) (*KnowledgeBase, error) { + path := fmt.Sprintf("/api/v1/knowledge-bases/%s", knowledgeBaseID) + resp, err := c.doRequest(ctx, http.MethodPut, path, request, nil) + if err != nil { + return nil, err + } + + var response KnowledgeBaseResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// DeleteKnowledgeBase deletes a knowledge base +func (c *Client) DeleteKnowledgeBase(ctx context.Context, knowledgeBaseID string) error { + path := fmt.Sprintf("/api/v1/knowledge-bases/%s", knowledgeBaseID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} + +// HybridSearch performs hybrid search +func (c *Client) HybridSearch(ctx context.Context, knowledgeBaseID string, query string) ([]*SearchResult, error) { + path := fmt.Sprintf("/api/v1/knowledge-bases/%s/hybrid-search", knowledgeBaseID) + + queryParams := url.Values{} + queryParams.Add("query", query) + + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams) + if err != nil { + return nil, err + } + + var response HybridSearchResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return response.Data, nil +} + +func (c *Client) CopyKnowledgeBase(ctx context.Context, request *CopyKnowledgeBaseRequest) error { + path := "/api/v1/knowledge-bases/copy" + + resp, err := c.doRequest(ctx, http.MethodPost, path, request, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} diff --git a/client/message.go b/client/message.go new file mode 100644 index 0000000..28ae55a --- /dev/null +++ b/client/message.go @@ -0,0 +1,82 @@ +// Package client provides the implementation for interacting with the WeKnora API +// The Message related interfaces are used to manage messages in a session +// Messages can be created, retrieved, deleted, and queried +package client + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "time" +) + +// Message message information +type Message struct { + ID string `json:"id"` + SessionID string `json:"session_id"` + RequestID string `json:"request_id"` + Content string `json:"content"` + Role string `json:"role"` + KnowledgeReferences []*SearchResult `json:"knowledge_references" ` + IsCompleted bool `json:"is_completed"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// MessageListResponse message list response +type MessageListResponse struct { + Success bool `json:"success"` + Data []Message `json:"data"` +} + +// LoadMessages loads session messages, supports pagination and time filtering +func (c *Client) LoadMessages(ctx context.Context, sessionID string, limit int, beforeTime *time.Time) ([]Message, error) { + path := fmt.Sprintf("/api/v1/messages/%s/load", sessionID) + + queryParams := url.Values{} + queryParams.Add("limit", strconv.Itoa(limit)) + + if beforeTime != nil { + queryParams.Add("before_time", beforeTime.Format(time.RFC3339Nano)) + } + + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams) + if err != nil { + return nil, err + } + + var response MessageListResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return response.Data, nil +} + +// GetRecentMessages gets recent messages from a session +func (c *Client) GetRecentMessages(ctx context.Context, sessionID string, limit int) ([]Message, error) { + return c.LoadMessages(ctx, sessionID, limit, nil) +} + +// GetMessagesBefore gets messages before a specified time +func (c *Client) GetMessagesBefore(ctx context.Context, sessionID string, beforeTime time.Time, limit int) ([]Message, error) { + return c.LoadMessages(ctx, sessionID, limit, &beforeTime) +} + +// DeleteMessage deletes a message +func (c *Client) DeleteMessage(ctx context.Context, sessionID string, messageID string) error { + path := fmt.Sprintf("/api/v1/messages/%s/%s", sessionID, messageID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} diff --git a/client/model.go b/client/model.go new file mode 100644 index 0000000..ecde9f6 --- /dev/null +++ b/client/model.go @@ -0,0 +1,155 @@ +// Package client provides the implementation for interacting with the WeKnora API +// The Model related interfaces are used to manage models for different tasks +// Models can be created, retrieved, updated, deleted, and queried +package client + +import ( + "context" + "fmt" + "net/http" +) + +// ModelType model type +type ModelType string + +// ModelSource model source +type ModelSource string + +// ModelParameters model parameters +type ModelParameters map[string]interface{} + +// Model model information +type Model struct { + ID string `json:"id"` + TenantID uint `json:"tenant_id"` + Name string `json:"name"` + Type ModelType `json:"type"` + Source ModelSource `json:"source"` + Description string `json:"description"` + Parameters ModelParameters `json:"parameters"` + IsDefault bool `json:"is_default"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// CreateModelRequest model creation request +type CreateModelRequest struct { + Name string `json:"name"` + Type ModelType `json:"type"` + Source ModelSource `json:"source"` + Description string `json:"description"` + Parameters ModelParameters `json:"parameters"` + IsDefault bool `json:"is_default"` +} + +// UpdateModelRequest model update request +type UpdateModelRequest struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters ModelParameters `json:"parameters"` + IsDefault bool `json:"is_default"` +} + +// ModelResponse model response +type ModelResponse struct { + Success bool `json:"success"` + Data Model `json:"data"` +} + +// ModelListResponse model list response +type ModelListResponse struct { + Success bool `json:"success"` + Data []Model `json:"data"` +} + +// Model type constants +const ( + ModelTypeEmbedding ModelType = "embedding" + ModelTypeChat ModelType = "chat" + ModelTypeRerank ModelType = "rerank" + ModelTypeSummary ModelType = "summary" +) + +// Model source constants +const ( + ModelSourceInternal ModelSource = "internal" + ModelSourceExternal ModelSource = "external" +) + +// CreateModel creates a model +func (c *Client) CreateModel(ctx context.Context, request *CreateModelRequest) (*Model, error) { + resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/models", request, nil) + if err != nil { + return nil, err + } + + var response ModelResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetModel gets a model +func (c *Client) GetModel(ctx context.Context, modelID string) (*Model, error) { + path := fmt.Sprintf("/api/v1/models/%s", modelID) + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return nil, err + } + + var response ModelResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// ListModels lists all models +func (c *Client) ListModels(ctx context.Context) ([]Model, error) { + resp, err := c.doRequest(ctx, http.MethodGet, "/api/v1/models", nil, nil) + if err != nil { + return nil, err + } + + var response ModelListResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return response.Data, nil +} + +// UpdateModel updates a model +func (c *Client) UpdateModel(ctx context.Context, modelID string, request *UpdateModelRequest) (*Model, error) { + path := fmt.Sprintf("/api/v1/models/%s", modelID) + resp, err := c.doRequest(ctx, http.MethodPut, path, request, nil) + if err != nil { + return nil, err + } + + var response ModelResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// DeleteModel deletes a model +func (c *Client) DeleteModel(ctx context.Context, modelID string) error { + path := fmt.Sprintf("/api/v1/models/%s", modelID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} diff --git a/client/session.go b/client/session.go new file mode 100644 index 0000000..86b380d --- /dev/null +++ b/client/session.go @@ -0,0 +1,403 @@ +// Package client provides the implementation for interacting with the WeKnora API +// The Session related interfaces are used to manage sessions for question-answering +// Sessions can be created, retrieved, updated, deleted, and queried +// They can also be used to generate titles for sessions +package client + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +// SessionStrategy defines session strategy +type SessionStrategy struct { + MaxRounds int `json:"max_rounds"` // Maximum number of rounds to maintain + EnableRewrite bool `json:"enable_rewrite"` // Enable query rewrite + FallbackStrategy string `json:"fallback_strategy"` // Fallback strategy + FallbackResponse string `json:"fallback_response"` // Fixed fallback response content + EmbeddingTopK int `json:"embedding_top_k"` // Top K for vector retrieval + KeywordThreshold float64 `json:"keyword_threshold"` // Keyword retrieval threshold + VectorThreshold float64 `json:"vector_threshold"` // Vector retrieval threshold + RerankModelID string `json:"rerank_model_id"` // Rerank model ID + RerankTopK int `json:"rerank_top_k"` // Top K for reranking + RerankThreshold float64 `json:"reranking_threshold"` // Reranking threshold + SummaryModelID string `json:"summary_model_id"` // Summary model ID + SummaryParameters *SummaryConfig `json:"summary_parameters"` // Summary model parameters + NoMatchPrefix string `json:"no_match_prefix"` // Fallback response prefix +} + +// SummaryConfig defines summary configuration +type SummaryConfig struct { + MaxTokens int `json:"max_tokens"` + TopP float64 `json:"top_p"` + TopK int `json:"top_k"` + FrequencyPenalty float64 `json:"frequency_penalty"` + PresencePenalty float64 `json:"presence_penalty"` + RepeatPenalty float64 `json:"repeat_penalty"` + Prompt string `json:"prompt"` + ContextTemplate string `json:"context_template"` + NoMatchPrefix string `json:"no_match_prefix"` + Temperature float64 `json:"temperature"` + Seed int `json:"seed"` + MaxCompletionTokens int `json:"max_completion_tokens"` +} + +// CreateSessionRequest session creation request +type CreateSessionRequest struct { + KnowledgeBaseID string `json:"knowledge_base_id"` // Associated knowledge base ID + SessionStrategy *SessionStrategy `json:"session_strategy"` // Session strategy +} + +// Session session information +type Session struct { + ID string `json:"id"` + TenantID uint `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + Title string `json:"title"` + MaxRounds int `json:"max_rounds"` + EnableRewrite bool `json:"enable_rewrite"` + FallbackStrategy string `json:"fallback_strategy"` + FallbackResponse string `json:"fallback_response"` + EmbeddingTopK int `json:"embedding_top_k"` + KeywordThreshold float64 `json:"keyword_threshold"` + VectorThreshold float64 `json:"vector_threshold"` + RerankModelID string `json:"rerank_model_id"` + RerankTopK int `json:"rerank_top_k"` + RerankThreshold float64 `json:"reranking_threshold"` // Reranking threshold + SummaryModelID string `json:"summary_model_id"` + SummaryParameters *SummaryConfig `json:"summary_parameters"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// SessionResponse session response +type SessionResponse struct { + Success bool `json:"success"` + Data Session `json:"data"` +} + +// SessionListResponse session list response +type SessionListResponse struct { + Success bool `json:"success"` + Data []Session `json:"data"` + Total int `json:"total"` + Page int `json:"page"` + PageSize int `json:"page_size"` +} + +// CreateSession creates a session +func (c *Client) CreateSession(ctx context.Context, request *CreateSessionRequest) (*Session, error) { + resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/sessions", request, nil) + if err != nil { + return nil, err + } + + var response SessionResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetSession gets a session +func (c *Client) GetSession(ctx context.Context, sessionID string) (*Session, error) { + path := fmt.Sprintf("/api/v1/sessions/%s", sessionID) + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return nil, err + } + + var response SessionResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetSessionsByTenant gets all sessions for a tenant +func (c *Client) GetSessionsByTenant(ctx context.Context, page int, pageSize int) ([]Session, int, error) { + queryParams := url.Values{} + queryParams.Add("page", strconv.Itoa(page)) + queryParams.Add("page_size", strconv.Itoa(pageSize)) + resp, err := c.doRequest(ctx, http.MethodGet, "/api/v1/sessions", nil, queryParams) + if err != nil { + return nil, 0, err + } + + var response SessionListResponse + if err := parseResponse(resp, &response); err != nil { + return nil, 0, err + } + + return response.Data, response.Total, nil +} + +// UpdateSession updates a session +func (c *Client) UpdateSession(ctx context.Context, sessionID string, request *CreateSessionRequest) (*Session, error) { + path := fmt.Sprintf("/api/v1/sessions/%s", sessionID) + resp, err := c.doRequest(ctx, http.MethodPut, path, request, nil) + if err != nil { + return nil, err + } + + var response SessionResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// DeleteSession deletes a session +func (c *Client) DeleteSession(ctx context.Context, sessionID string) error { + path := fmt.Sprintf("/api/v1/sessions/%s", sessionID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} + +// GenerateTitleRequest title generation request +type GenerateTitleRequest struct { + Messages []Message `json:"messages"` +} + +// GenerateTitleResponse title generation response +type GenerateTitleResponse struct { + Success bool `json:"success"` + Data string `json:"data"` +} + +// GenerateTitle generates a session title +func (c *Client) GenerateTitle(ctx context.Context, sessionID string, request *GenerateTitleRequest) (string, error) { + path := fmt.Sprintf("/api/v1/sessions/%s/generate_title", sessionID) + resp, err := c.doRequest(ctx, http.MethodPost, path, request, nil) + if err != nil { + return "", err + } + + var response GenerateTitleResponse + if err := parseResponse(resp, &response); err != nil { + return "", err + } + + return response.Data, nil +} + +// KnowledgeQARequest knowledge Q&A request +type KnowledgeQARequest struct { + Query string `json:"query"` +} + +type ResponseType string + +const ( + ResponseTypeAnswer ResponseType = "answer" + ResponseTypeReferences ResponseType = "references" +) + +// StreamResponse streaming response +type StreamResponse struct { + ID string `json:"id"` // Unique identifier + ResponseType ResponseType `json:"response_type"` // Response type + Content string `json:"content"` // Current content fragment + Done bool `json:"done"` // Whether completed + KnowledgeReferences []*SearchResult `json:"knowledge_references"` // Knowledge references +} + +// KnowledgeQAStream knowledge Q&A streaming API +func (c *Client) KnowledgeQAStream(ctx context.Context, sessionID string, query string, callback func(*StreamResponse) error) error { + path := fmt.Sprintf("/api/v1/knowledge-chat/%s", sessionID) + fmt.Printf("Starting KnowledgeQAStream request, session ID: %s, query: %s\n", sessionID, query) + + request := &KnowledgeQARequest{ + Query: query, + } + + resp, err := c.doRequest(ctx, http.MethodPost, path, request, nil) + if err != nil { + fmt.Printf("Request failed: %v\n", err) + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + err := fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(body)) + fmt.Printf("Request returned error status: %v\n", err) + return err + } + + fmt.Println("Successfully established SSE connection, processing data stream") + + // Use bufio to read SSE data line by line + scanner := bufio.NewScanner(resp.Body) + var dataBuffer string + var eventType string + messageCount := 0 + + for scanner.Scan() { + line := scanner.Text() + fmt.Printf("Received SSE line: %s\n", line) + + // Empty line indicates the end of an event + if line == "" { + if dataBuffer != "" { + fmt.Printf("Processing data: %s, event type: %s\n", dataBuffer, eventType) + var streamResponse StreamResponse + if err := json.Unmarshal([]byte(dataBuffer), &streamResponse); err != nil { + fmt.Printf("Failed to parse SSE data: %v\n", err) + return fmt.Errorf("failed to parse SSE data: %w", err) + } + + messageCount++ + fmt.Printf("Parsed message #%d, done status: %v\n", messageCount, streamResponse.Done) + + if err := callback(&streamResponse); err != nil { + fmt.Printf("Callback processing failed: %v\n", err) + return err + } + dataBuffer = "" + eventType = "" + } + continue + } + + // Process lines with event: prefix + if strings.HasPrefix(line, "event:") { + eventType = line[6:] // Remove "event:" prefix + fmt.Printf("Set event type: %s\n", eventType) + } + + // Process lines with data: prefix + if strings.HasPrefix(line, "data:") { + dataBuffer = line[5:] // Remove "data:" prefix + } + } + + if err := scanner.Err(); err != nil { + fmt.Printf("Failed to read SSE stream: %v\n", err) + return fmt.Errorf("failed to read SSE stream: %w", err) + } + + fmt.Printf("KnowledgeQAStream completed, processed %d messages\n", messageCount) + return nil +} + +// ContinueStream continues to receive an active stream for a session +func (c *Client) ContinueStream(ctx context.Context, sessionID string, messageID string, callback func(*StreamResponse) error) error { + path := fmt.Sprintf("/api/v1/sessions/continue-stream/%s", sessionID) + + queryParams := url.Values{} + queryParams.Add("message_id", messageID) + + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, queryParams) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(body)) + } + + // Use bufio to read SSE data line by line + scanner := bufio.NewScanner(resp.Body) + var dataBuffer string + var eventType string + + for scanner.Scan() { + line := scanner.Text() + + // Empty line indicates the end of an event + if line == "" { + if dataBuffer != "" && eventType == "message" { + var streamResponse StreamResponse + if err := json.Unmarshal([]byte(dataBuffer), &streamResponse); err != nil { + return fmt.Errorf("failed to parse SSE data: %w", err) + } + + if err := callback(&streamResponse); err != nil { + return err + } + dataBuffer = "" + eventType = "" + } + continue + } + + // Process lines with event: prefix + if strings.HasPrefix(line, "event:") { + eventType = line[6:] // Remove "event:" prefix + } + + // Process lines with data: prefix + if strings.HasPrefix(line, "data:") { + dataBuffer = line[5:] // Remove "data:" prefix + } + } + + if err := scanner.Err(); err != nil { + return fmt.Errorf("failed to read SSE stream: %w", err) + } + + return nil +} + +// SearchKnowledgeRequest knowledge search request +type SearchKnowledgeRequest struct { + Query string `json:"query"` // Query content + KnowledgeBaseID string `json:"knowledge_base_id"` // Knowledge base ID +} + +// SearchKnowledgeResponse search results response +type SearchKnowledgeResponse struct { + Success bool `json:"success"` + Data []*SearchResult `json:"data"` +} + +// SearchKnowledge performs knowledge base search without LLM summarization +func (c *Client) SearchKnowledge(ctx context.Context, request *SearchKnowledgeRequest) ([]*SearchResult, error) { + fmt.Printf("Starting SearchKnowledge request, knowledge base ID: %s, query: %s\n", + request.KnowledgeBaseID, request.Query) + + resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/knowledge-search", request, nil) + if err != nil { + fmt.Printf("Request failed: %v\n", err) + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + err := fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(body)) + fmt.Printf("Request returned error status: %v\n", err) + return nil, err + } + + var response SearchKnowledgeResponse + if err := parseResponse(resp, &response); err != nil { + fmt.Printf("Failed to parse response: %v\n", err) + return nil, err + } + + fmt.Printf("SearchKnowledge completed, found %d results\n", len(response.Data)) + return response.Data, nil +} diff --git a/client/tenant.go b/client/tenant.go new file mode 100644 index 0000000..e2f8069 --- /dev/null +++ b/client/tenant.go @@ -0,0 +1,136 @@ +// Package client provides the implementation for interacting with the WeKnora API +// The Tenant related interfaces are used to manage tenants in the system +// Tenants can be created, retrieved, updated, deleted, and queried +// They can also be used to manage retriever engines for different tasks +package client + +import ( + "context" + "fmt" + "net/http" + "time" +) + +// RetrieverEngines defines a collection of retriever engine parameters +type RetrieverEngines struct { + Engines []RetrieverEngineParams `json:"engines"` +} + +// RetrieverEngineParams contains configuration for retriever engines +type RetrieverEngineParams struct { + RetrieverType string `json:"retriever_type"` // Type of retriever (e.g., keywords, vector) + RetrieverEngineType string `json:"retriever_engine_type"` // Type of engine implementing the retriever +} + +// Tenant represents tenant information in the system +type Tenant struct { + ID uint `yaml:"id" json:"id" gorm:"primaryKey"` + // Tenant name + Name string `yaml:"name" json:"name"` + // Tenant description + Description string `yaml:"description" json:"description"` + // API key for authentication + APIKey string `yaml:"api_key" json:"api_key"` + // Tenant status (active, inactive) + Status string `yaml:"status" json:"status" gorm:"default:'active'"` + // Configured retrieval engines + RetrieverEngines RetrieverEngines `yaml:"retriever_engines" json:"retriever_engines" gorm:"type:json"` + // Business/department information + Business string `yaml:"business" json:"business"` + // Creation timestamp + CreatedAt time.Time `yaml:"created_at" json:"created_at"` + // Last update timestamp + UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"` +} + +// TenantResponse represents the API response structure for tenant operations +type TenantResponse struct { + Success bool `json:"success"` // Whether the operation was successful + Data Tenant `json:"data"` // Tenant data +} + +// TenantListResponse represents the API response structure for listing tenants +type TenantListResponse struct { + Success bool `json:"success"` // Whether the operation was successful + Data struct { + Items []Tenant `json:"items"` // List of tenant items + } `json:"data"` +} + +// CreateTenant creates a new tenant +func (c *Client) CreateTenant(ctx context.Context, tenant *Tenant) (*Tenant, error) { + resp, err := c.doRequest(ctx, http.MethodPost, "/api/v1/tenants", tenant, nil) + if err != nil { + return nil, err + } + + var response TenantResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// GetTenant retrieves a tenant by ID +func (c *Client) GetTenant(ctx context.Context, tenantID uint) (*Tenant, error) { + path := fmt.Sprintf("/api/v1/tenants/%d", tenantID) + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return nil, err + } + + var response TenantResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// UpdateTenant updates an existing tenant +func (c *Client) UpdateTenant(ctx context.Context, tenant *Tenant) (*Tenant, error) { + path := fmt.Sprintf("/api/v1/tenants/%d", tenant.ID) + resp, err := c.doRequest(ctx, http.MethodPut, path, tenant, nil) + if err != nil { + return nil, err + } + + var response TenantResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return &response.Data, nil +} + +// DeleteTenant removes a tenant by ID +func (c *Client) DeleteTenant(ctx context.Context, tenantID uint) error { + path := fmt.Sprintf("/api/v1/tenants/%d", tenantID) + resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + + var response struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + } + + return parseResponse(resp, &response) +} + +// ListTenants retrieves all tenants +func (c *Client) ListTenants(ctx context.Context) ([]Tenant, error) { + resp, err := c.doRequest(ctx, http.MethodGet, "/api/v1/tenants", nil, nil) + if err != nil { + return nil, err + } + + var response TenantListResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + + return response.Data.Items, nil +} diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 0000000..b253f35 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,104 @@ +// Package main is the main package for the WeKnora server +// It contains the main function and the entry point for the server +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gin-gonic/gin" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/container" + "github.com/Tencent/WeKnora/internal/runtime" + "github.com/Tencent/WeKnora/internal/tracing" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +func main() { + // Set log format with request ID + log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile) + log.SetOutput(os.Stdout) + + // Set Gin mode + if os.Getenv("GIN_MODE") == "release" { + gin.SetMode(gin.ReleaseMode) + } else { + gin.SetMode(gin.DebugMode) + } + + // Build dependency injection container + c := container.BuildContainer(runtime.GetContainer()) + + // Run application + err := c.Invoke(func( + cfg *config.Config, + router *gin.Engine, + tracer *tracing.Tracer, + resourceCleaner interfaces.ResourceCleaner, + ) error { + // Create context for resource cleanup + shutdownTimeout := cfg.Server.ShutdownTimeout + if shutdownTimeout == 0 { + shutdownTimeout = 30 * time.Second + } + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cleanupCancel() + + // Register tracer cleanup function to resource cleaner + resourceCleaner.RegisterWithName("Tracer", func() error { + return tracer.Cleanup(cleanupCtx) + }) + + // Create HTTP server + server := &http.Server{ + Addr: fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port), + Handler: router, + } + + ctx, done := context.WithCancel(context.Background()) + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + go func() { + sig := <-signals + log.Printf("Received signal: %v, starting server shutdown...", sig) + + // Create a context with timeout for server shutdown + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer shutdownCancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + log.Fatalf("Server forced to shutdown: %v", err) + } + + // Clean up all registered resources + log.Println("Cleaning up resources...") + errs := resourceCleaner.Cleanup(cleanupCtx) + if len(errs) > 0 { + log.Printf("Errors occurred during resource cleanup: %v", errs) + } + + log.Println("Server has exited") + done() + }() + + // Start server + log.Printf("Server is running at %s:%d", cfg.Server.Host, cfg.Server.Port) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + return fmt.Errorf("failed to start server: %v", err) + } + + // Wait for shutdown signal + <-ctx.Done() + return nil + }) + if err != nil { + log.Fatalf("Failed to run application: %v", err) + } +} diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..cc8e6e6 --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,602 @@ +# 服务器配置 +server: + port: 8080 + host: "0.0.0.0" + +# 对话服务配置 +conversation: + max_rounds: 5 + keyword_threshold: 0.3 + embedding_top_k: 10 + vector_threshold: 0.5 + rerank_threshold: 0.5 + rerank_top_k: 5 + fallback_strategy: "fixed" + fallback_response: "抱歉，我无法回答这个问题。" + fallback_prompt: | + 你是一个专业、友好的AI助手。现在用户提出的问题超出了你的知识库范围，你需要生成一个礼貌且有帮助的回复。 + + ## 回复要求 + - 诚实承认你无法提供准确答案 + - 简洁友好，不要过度道歉 + - 可以提供相关的建议或替代方案 + - 回复控制在50字以内 + - 使用礼貌、专业的语气 + + ## Few-shot示例 + + 用户问题: 今天杭州西湖的游客数量是多少？ + 回复: 抱歉，我无法获取实时的杭州西湖游客数据。您可以通过杭州旅游官网或相关APP查询这一信息。 + + 用户问题: 张教授的新论文发表了吗？ + 回复: 我没有张教授的最新论文信息。建议您查询学术数据库或直接联系张教授获取最新动态。 + + 用户问题: 我的银行卡号是多少？ + 回复: 作为AI助手，我无法获取您的个人银行信息。请登录您的银行APP或联系银行客服获取相关信息。 + + ## 用户当前的问题是: + {{.Query}} + enable_rewrite: true + enable_rerank: true + rewrite_prompt_system: | + 你是一个专注于指代消解和省略补全的智能助手，你的任务是根据历史对话上下文，清晰识别用户问题中的代词并替换为明确的主语，同时补全省略的关键信息。 + + ## 改写目标 + 请根据历史对话，对当前用户问题进行改写，目标是： + - 进行指代消解，将"它"、"这个"、"那个"、"他"、"她"、"它们"、"他们"、"她们"等代词替换为明确的主语 + - 补全省略的关键信息，确保问题语义完整 + - 保持问题的原始含义和表达方式不变 + - 改写后必须也是一个问题 + - 改写后的问题字数控制在30字以内 + - 仅输出改写后的问题，不要输出任何解释，更不要尝试回答该问题，后面有其他助手回去解答此问题 + + ## Few-shot示例 + + 示例1: + 历史对话: + 用户: 微信支付有哪些功能？ + 助手: 微信支付的主要功能包括转账、付款码、收款、信用卡还款等多种支付服务。 + + 用户问题: 它的安全性 + 改写后: 微信支付的安全性 + + 示例2: + 历史对话: + 用户: 苹果手机电池不耐用怎么办？ + 助手: 您可以通过降低屏幕亮度、关闭后台应用和定期更新系统来延长电池寿命。 + + 用户问题: 这样会影响使用体验吗？ + 改写后: 降低屏幕亮度和关闭后台应用是否影响使用体验 + + 示例3: + 历史对话: + 用户: 如何制作红烧肉？ + 助手: 红烧肉的制作需要先将肉块焯水，然后加入酱油、糖等调料慢炖。 + + 用户问题: 需要炖多久？ + 改写后: 红烧肉需要炖多久 + + 示例4: + 历史对话: + 用户: 北京到上海的高铁票价是多少？ + 助手: 北京到上海的高铁票价根据车次和座位类型不同，二等座约为553元，一等座约为933元。 + + 用户问题: 时间呢？ + 改写后: 北京到上海的高铁时长 + + 示例5: + 历史对话: + 用户: 如何注册微信账号？ + 助手: 注册微信账号需要下载微信APP，输入手机号，接收验证码，然后设置昵称和密码。 + + 用户问题: 国外手机号可以吗？ + 改写后: 国外手机号是否可以注册微信账号 + rewrite_prompt_user: | + ## 历史对话背景 + {{range .Conversation}} + ------BEGIN------ + 用户的问题是：{{.Query}} + 助手的回答是：{{.Answer}} + ------END------ + {{end}} + + ## 需要改写的用户问题 + {{.Query}} + + ## 改写后的问题 + keywords_extraction_prompt: | + # 角色 + 你是一个专业的关键词提取助手，你的任务是根据用户的问题，提取出最重要的关键词/短语。 + + # 要求 + - 总结用户的问题，并给出最重要的关键词/短语，关键词/短语的数量不超过5个 + - 使用逗号作为分隔符来分隔关键词/短语 + - 关键词/短语必须来自于用户的问题，不得虚构 + - 不要输出任何解释，直接输出关键词/短语，不要有任何前缀、解释或标点符号，不要尝试回答该问题，后面有其他助手会去搜索此问题 + + # 输出格式 + keyword1, keyword2, keyword3, keyword4, keyword5 + + # Examples + + ## Example 1 + USER: 如何提高英语口语水平？ + ############### + Output: 英语口语, 口语水平, 提高英语口语, 英语口语提升, 英语口语练习 + + ## Example 2 + USER: 最近上海有什么好玩的展览活动？ + ############### + Output: 上海展览, 展览活动, 上海展览推荐, 展览活动推荐, 上海展览活动 + + ## Example 3 + USER: 苹果手机电池不耐用怎么解决？ + ############### + Output: 苹果手机, 电池不耐用, 电池优化, 电池寿命, 电池保养 + + ## Example 4 + USER: Python的Logo长啥样？ + ############### + Output: Python Logo + + ## Example 5 + USER: 如何使用iPhone连接WiFi？ + ############### + Output: iPhone, 连接WiFi, 使用iPhone连接WiFi + + # Real Data + USER: {{.Query}} + + keywords_extraction_prompt_user: | + Output: + + generate_summary_prompt: | + 你是一个精准的文章总结专家。你的任务是提取并总结用户提供的文章或片段的核心内容。 + + ## 核心要求 + - 总结结果长度为50-100个字，根据内容复杂度灵活调整 + - 完全基于提供的文章内容生成总结，不添加任何未在文章中出现的信息 + - 确保总结包含文章的关键信息点和主要结论 + - 即使文章内容较复杂或专业，也必须尝试提取核心要点进行总结 + - 直接输出总结结果，不包含任何引言、前缀或解释 + + ## 格式与风格 + - 使用客观、中立的第三人称陈述语气 + - 使用清晰简洁的中文表达 + - 保持逻辑连贯性，确保句与句之间有合理过渡 + - 避免重复使用相同的表达方式或句式结构 + + ## 注意事项 + - 绝对不输出"无法生成"、"无法总结"、"内容不足"等拒绝回应的词语 + - 不要照抄或参考示例中的任何内容，确保总结完全基于用户提供的新文章 + - 对于任何文本都尽最大努力提取重点并总结，无论长度或复杂度 + + ## 以下是用户给出的文章相关信息： + + generate_session_title_prompt: | + 你是一个专业的会话标题生成助手，你的任务是为用户提问创建简洁、精准且具描述性的标题。 + + ## 格式要求 + - 标题长度必须在10个字以内 + - 标题应准确反映用户问题的核心主题 + - 使用名词短语结构，避免使用问句 + - 保持简洁明了，删除非必要词语 + - 不要使用"关于"、"如何"等冗余词语开头 + - 直接输出标题文本，不要有任何前缀、解释或标点符号 + + ## Few-shot示例 + + 用户问题: 如何提高英语口语水平？ + 标题: 英语口语提升 + + 用户问题: 最近上海有什么好玩的展览活动？ + 标题: 上海展览推荐 + + 用户问题: 苹果手机电池不耐用怎么解决？ + 标题: 苹果电池优化 + + ## 用户的问题是： + summary: + repeat_penalty: 1.0 + temperature: 0.3 + max_completion_tokens: 2048 + no_match_prefix: |- + + + NO_MATCH + prompt: | + 这是用户和助手之间的对话。当用户提出问题时，助手会基于特定的信息进行解答。助手首先在心中思考推理过程，然后向用户提供答案。 + 推理过程用 标签包围，答案直接输出在think标签后面，即： + + 这里是推理过程 + + 这里是答案 + context_template: | + 你是一个专业的智能信息检索助手，名为小微，犹如专业的高级秘书，依据检索到的信息回答用户问题。 + 当用户提出问题时，助手只能基于给定的信息进行解答，不能利用任何先验知识。 + + ## 回答问题规则 + - 仅根据检索到的信息中的事实进行回复，不得运用任何先验知识，保持回应的客观性和准确性。 + - 复杂问题和答案的按Markdown分结构展示，总述部分不需要拆分 + - 如果是比较简单的答案，不需要把最终答案拆分的过于细碎 + - 结果中使用的图片地址必须来自于检索到的信息，不得虚构 + - 检查结果中的文字和图片是否来自于检索到的信息，如果扩展了不在检索到的信息中的内容，必须进行修改，直到得到最终答案 + - 如果用户问题无法回答，只输出NO_MATCH即可，即： + + + NO_MATCH + + ## 输出限制 + - 以Markdown图文格式输出你的最终结果 + - 输出内容要保证简短且全面，条理清晰，信息明确，不重复。 + + ## 当前时间是： + {{.CurrentTime}} {{.CurrentWeek}} + + ## 检索到的信息如下： + ------BEGIN------ + {{range .Contexts}} + {{.}} + {{end}} + ------END------ + + ## 用户当前的问题是： + {{.Query}} + extract_entities_prompt: | + ## 任务 + 用户提供的文本中，提取所有符合以下实体类型的实体： + EntityTypes: [Person, Organization, Location, Product, Event, Date, Work, Concept, Resource, Category, Operation] + + ## 要求 + 1. 提取结果必须以JSON数组格式输出 + 2. 每个实体必须包含 title 和 type 字段，description 字段可选但强烈建议提供 + 3. 确保 type 字段的值必须严格从 EntityTypes 列表中选择，不得创建新类型 + 4. 如果无法确定实体类型，不要强行归类，宁可不提取该实体 + 5. 不要输出任何解释或额外内容，只输出JSON数组 + 6. 所有字段值不能包含HTML标签或其他代码 + 7. 如果实体有歧义，需在description中说明具体指代 + 8. 若没有找到任何实体，返回空数组 [] + + ## 实体提取规则 + - Person: 真实或虚构的人物，包括历史人物、现代人物、文学角色等 + - Organization: 公司、政府机构、团队、学校等组织实体 + - Location: 地理位置、地标、国家、城市等 + - Product: 商品、服务、品牌等商业产品 + - Event: 事件、会议、节日、历史事件等 + - Date: 日期、时间段、年代等时间相关信息 + - Work: 书籍、电影、音乐、艺术作品等创作内容 + - Concept: 抽象概念、思想、理论等 + - Resource: 自然资源、信息资源、工具等 + - Category: 分类、类别、领域等 + - Operation: 操作、动作、方法、过程等 + + ## 提取步骤 + 1. 仔细阅读文本，识别可能的实体 + 2. 对每个识别到的实体，确定其最适合的实体类型（必须从EntityTypes中选择） + 3. 为每个实体创建包含以下字段的JSON对象： + - title: 实体的标准名称，不包含修饰词，如引号等 + - type: 从EntityTypes中选择的实体类型 + - description: 对该实体的简明中文描述，应基于文本内容 + 4. 验证每个实体的所有字段是否正确且格式化恰当 + 5. 将所有实体对象合并为一个JSON数组 + 6. 检查最终JSON是否有效并符合要求 + + ## 示例 + [输入] + 文本： 《红楼梦》，又名《石头记》，是清代作家曹雪芹创作的中国古典四大名著之一，被誉为中国封建社会的百科全书。该书前80回由曹雪芹所著，后40回一般认为是高鹗所续。小说以贾、史、王、薛四大家族的兴衰为背景，以贾宝玉、林黛玉和薛宝钗的爱情悲剧为主线，刻画了以贾宝玉和金陵十二钗为中心的正邪两赋、贤愚并出的高度复杂的人物群像。成书于乾隆年间（1743年前后），是中国文学史上现实主义的高峰，对后世影响深远。 + + [输出] + [ + { + "title": "红楼梦", + "type": "Work", + "description": "红楼梦是清代作家曹雪芹创作的中国古典四大名著之一，被誉为中国封建社会的百科全书" + }, + { + "title": "石头记", + "type": "Work", + "description": "石头记是红楼梦的别名" + }, + { + "title": "曹雪芹", + "type": "Person", + "description": "曹雪芹是清代作家，红楼梦的作者，创作了前80回" + }, + { + "title": "高鹗", + "type": "Person", + "description": "高鹗是红楼梦后40回的续作者" + }, + { + "title": "贾宝玉", + "type": "Person", + "description": "贾宝玉是红楼梦中的主要角色，爱情悲剧的主角之一" + }, + { + "title": "林黛玉", + "type": "Person", + "description": "林黛玉是红楼梦中的主要角色，爱情悲剧的主角之一" + }, + { + "title": "薛宝钗", + "type": "Person", + "description": "薛宝钗是红楼梦中的主要角色，爱情悲剧的主角之一" + }, + { + "title": "金陵十二钗", + "type": "Concept", + "description": "金陵十二钗是红楼梦中以贾宝玉为中心的十二位主要女性角色" + }, + { + "title": "乾隆年间", + "type": "Date", + "description": "乾隆年间指的是红楼梦成书的时间，约1743年前后" + }, + { + "title": "四大家族", + "type": "Concept", + "description": "四大家族是红楼梦中的贾、史、王、薛四个家族，是小说的背景" + }, + { + "title": "中国文学史", + "type": "Category", + "description": "红楼梦被视为中国文学史中现实主义的高峰之作" + } + ] + + extract_relationships_prompt: | + ## 任务 + 从用户提供的实体数组中，提取实体之间存在的明确关系，形成结构化的关系网络。 + + ## 要求 + 1. 关系提取必须基于提供的文本内容，不得臆测不存在的关系 + 2. 结果必须以JSON数组格式输出，每个关系为数组中的一个对象 + 3. 每个关系对象必须包含 source, target, description 和 strength 字段 + 4. 不要输出任何解释或额外内容，只输出JSON数组 + 5. 若没有找到任何关系，返回空数组 [] + + ## 关系提取规则 + - 只有在文本中明确体现的关系才应被提取 + - 源实体(source)和目标实体(target)必须是实体数组中已有的实体 + - 关系描述(description)应简明扼要地说明两个实体间的具体关系 + - 关系强度(strength)应根据以下标准确定： + * 10分：直接创造/从属关系（如作者与作品、发明者与发明、母公司与子公司） + * 9分：同一实体的不同表现形式（如别名、曾用名） + * 8分：紧密相关且互相影响的关系（如密切合作伙伴、家庭成员） + * 7分：明确但非直接的关系（如作品中的角色、组织中的成员） + * 6分：间接关联且有明确联系（如同事关系、相似产品） + * 5分：存在关联但较为松散（如同一领域的不同概念） + + ## 提取步骤 + 1. 仔细分析文本内容，确定哪些实体之间存在明确关系 + 2. 只考虑文本中明确提及的关系，不要臆测 + 3. 对每个找到的关系，确定： + - source: 关系的源实体标题（必须是实体列表中已有的实体） + - target: 关系的目标实体标题（必须是实体列表中已有的实体） + - description: 简明准确的关系描述（用中文表述） + - strength: 基于上述标准的关系强度（5-10之间的整数） + 4. 检查每个关系是否双向： + - 如果关系是双向的（如"A是B的朋友"意味着"B也是A的朋友"），考虑是否需要创建反向关系 + - 如果关系是单向的（如"A创作了B"），则只保留单向关系 + 5. 验证所有关系的一致性和合理性： + - 确保没有矛盾的关系（如A同时是B的父亲和兄弟） + - 确保关系描述与关系强度匹配 + 6. 将所有有效关系组织为JSON数组 + + ## 示例 + [输入] + 实体： [ + { + "title": "红楼梦", + "type": "Work", + "description": "红楼梦是清代作家曹雪芹创作的中国古典四大名著之一，被誉为中国封建社会的百科全书" + }, + { + "title": "石头记", + "type": "Work", + "description": "石头记是红楼梦的别名" + }, + { + "title": "曹雪芹", + "type": "Person", + "description": "曹雪芹是清代作家，红楼梦的作者，创作了前80回" + }, + { + "title": "高鹗", + "type": "Person", + "description": "高鹗是红楼梦后40回的续作者" + }, + { + "title": "贾宝玉", + "type": "Person", + "description": "贾宝玉是红楼梦中的主要角色，爱情悲剧的主角之一" + }, + { + "title": "林黛玉", + "type": "Person", + "description": "林黛玉是红楼梦中的主要角色，爱情悲剧的主角之一" + }, + { + "title": "薛宝钗", + "type": "Person", + "description": "薛宝钗是红楼梦中的主要角色，爱情悲剧的主角之一" + }, + { + "title": "四大家族", + "type": "Concept", + "description": "四大家族是红楼梦中的贾、史、王、薛四个家族，是小说的背景" + }, + { + "title": "金陵十二钗", + "type": "Concept", + "description": "金陵十二钗是红楼梦中以贾宝玉为中心的十二位主要女性角色" + }, + { + "title": "乾隆年间", + "type": "Date", + "description": "乾隆年间指的是红楼梦成书的时间，约1743年前后" + }, + { + "title": "中国文学史", + "type": "Category", + "description": "红楼梦被视为中国文学史中现实主义的高峰之作" + } + ] + + 文本： 《红楼梦》，又名《石头记》，是清代作家曹雪芹创作的中国古典四大名著之一，被誉为中国封建社会的百科全书。该书前80回由曹雪芹所著，后40回一般认为是高鹗所续。小说以贾、史、王、薛四大家族的兴衰为背景，以贾宝玉、林黛玉和薛宝钗的爱情悲剧为主线，刻画了以贾宝玉和金陵十二钗为中心的正邪两赋、贤愚并出的高度复杂的人物群像。成书于乾隆年间（1743年前后），是中国文学史上现实主义的高峰，对后世影响深远。 + + [输出] + [ + { + "source": "曹雪芹", + "target": "红楼梦", + "description": "曹雪芹是红楼梦的主要作者，创作了前80回", + "strength": 10 + }, + { + "source": "高鹗", + "target": "红楼梦", + "description": "高鹗是红楼梦后40回的续作者", + "strength": 10 + }, + { + "source": "红楼梦", + "target": "石头记", + "description": "石头记是红楼梦的别名", + "strength": 9 + }, + { + "source": "红楼梦", + "target": "中国文学史", + "description": "红楼梦被视为中国文学史中现实主义的高峰之作", + "strength": 7 + }, + { + "source": "贾宝玉", + "target": "林黛玉", + "description": "贾宝玉与林黛玉有深厚的爱情关系，是小说主线之一", + "strength": 8 + }, + { + "source": "贾宝玉", + "target": "薛宝钗", + "description": "贾宝玉与薛宝钗的关系是小说爱情悲剧主线的一部分", + "strength": 8 + }, + { + "source": "贾宝玉", + "target": "金陵十二钗", + "description": "贾宝玉是金陵十二钗故事的中心人物", + "strength": 8 + }, + { + "source": "红楼梦", + "target": "贾宝玉", + "description": "贾宝玉是红楼梦中的主要角色", + "strength": 7 + }, + { + "source": "红楼梦", + "target": "林黛玉", + "description": "林黛玉是红楼梦中的主要角色", + "strength": 7 + }, + { + "source": "红楼梦", + "target": "薛宝钗", + "description": "薛宝钗是红楼梦中的主要角色", + "strength": 7 + }, + { + "source": "红楼梦", + "target": "四大家族", + "description": "四大家族是红楼梦的背景设定", + "strength": 7 + }, + { + "source": "红楼梦", + "target": "金陵十二钗", + "description": "金陵十二钗是红楼梦中的重要概念", + "strength": 7 + }, + { + "source": "红楼梦", + "target": "乾隆年间", + "description": "红楼梦成书于乾隆年间，约1743年前后", + "strength": 6 + } + ] + +# 知识库配置 +knowledge_base: + chunk_size: 512 + chunk_overlap: 50 + split_markers: ["\n\n", "\n", "。"] + image_processing: + enable_multimodal: true + +extract: + extract_graph: + description: | + 请基于给定文本，按以下步骤完成信息提取任务，确保逻辑清晰、信息完整准确： + + ## 一、实体提取与属性补充 + 1. **提取核心实体**：通读文本，按逻辑顺序（如文本叙述顺序、实体关联紧密程度）提取所有与任务相关的核心实体。 + 2. **补充实体详细属性**：针对每个提取的实体，全面补充其在文本中明确提及的详细属性，确保无关键属性遗漏。 + + ## 二、关系提取与验证 + 1. **明确关系类型**：仅从指定关系列表中选择对应类型，限定关系类型为: %s。 + 2. **提取有效关系**：基于已提取的实体及属性，识别文本中真实存在的关系，确保关系符合文本事实、无虚假关联。 + 3. **明确关系主体**：对每一组提取的关系，清晰标注两个关联主体，避免主体混淆。 + 4. **补充关联属性**：若文本中存在与该关系直接相关的补充信息，需将该信息作为关系的关联属性补充，进一步完善关系信息。 + tags: + - "作者" + - "别名" + examples: + - text: | + 《红楼梦》，又名《石头记》，是清代作家曹雪芹创作的中国古典四大名著之一，被誉为中国封建社会的百科全书。该书前80回由曹雪芹所著，后40回一般认为是高鹗所续。 + 小说以贾、史、王、薛四大家族的兴衰为背景，以贾宝玉、林黛玉和薛宝钗的爱情悲剧为主线，刻画了以贾宝玉和金陵十二钗为中心的正邪两赋、贤愚并出的高度复杂的人物群像。 + 成书于乾隆年间（1743年前后），是中国文学史上现实主义的高峰，对后世影响深远。 + node: + - name: "红楼梦" + attributes: + - "中国古典四大名著之一" + - "又名《石头记》" + - "被誉为中国封建社会的百科全书" + - name: "石头记" + attributes: + - "《红楼梦》的别名" + - name: "曹雪芹" + attributes: + - "清代作家" + - "《红楼梦》前 80 回的作者" + - name: "高鹗" + attributes: + - "一般认为是《红楼梦》后 40 回的续写者" + relation: + - node1: "红楼梦" + node2: "曹雪芹" + type: "作者" + - node1: "红楼梦" + node2: "高鹗" + type: "作者" + - node1: "红楼梦" + node2: "石头记" + type: "别名" + extract_entity: + description: | + 请基于用户给的问题，按以下步骤处理关键信息提取任务： + 1. 梳理逻辑关联：首先完整分析文本内容，明确其核心逻辑关系，并简要标注该核心逻辑类型； + 2. 提取关键实体：围绕梳理出的逻辑关系，精准提取文本中的关键信息并归类为明确实体，确保不遗漏核心信息、不添加冗余内容； + 3. 排序实体优先级：按实体与文本核心主题的关联紧密程度排序，优先呈现对理解文本主旨最重要的实体； + examples: + - text: "《红楼梦》，又名《石头记》，是清代作家曹雪芹创作的中国古典四大名著之一，被誉为中国封建社会的百科全书。" + node: + - name: "红楼梦" + - name: "曹雪芹" + - name: "中国古典四大名著" + fabri_text: + with_tag: | + 请随机生成一段文本，要求内容与 %s 等相关，字数在 [50-200] 之间，并且尽量包含一些与这些标签相关的专业术语或典型元素，使文本更具针对性和相关性。 + with_no_tag: | + 请随机生成一段文本，内容请自由发挥，字数在 [50-200] 之间。 \ No newline at end of file diff --git a/dataset/README b/dataset/README new file mode 100644 index 0000000..6c78bca --- /dev/null +++ b/dataset/README @@ -0,0 +1,273 @@ +# QA Dataset Sampling Tool + +A comprehensive tool for sampling QA datasets and generating answers using OpenAI's GPT models. This tool helps you create high-quality question-answering datasets from large-scale collections like MS MARCO. + +## Features + +- **Smart Sampling**: Intelligently sample queries, documents, and relevance judgments from large datasets +- **Answer Generation**: Automatically generate high-quality answers using OpenAI's GPT models +- **Resume Support**: Continue interrupted answer generation from where it left off +- **Progress Tracking**: Real-time progress updates and statistics +- **Result Visualization**: Easy-to-read display of generated QA pairs with context + +## Installation + +### Prerequisites + +- Python 3.7+ +- OpenAI API key + +### Install Dependencies + +```bash +pip install pandas pyarrow openai +``` + +### Set Environment Variables + +```bash +export OPENAI_API_KEY="your-openai-api-key" +# Optional: Use custom OpenAI endpoint +export OPENAI_BASE_URL="https://api.openai.com/v1" +``` + +### Parpare dataset + +We provide pre-processed samples from popular QA datasets: + +MarkrAI/msmarco_sample_autorag + +## Quick Start + +### 1. Sample Data from Large Dataset + +First, sample a subset of queries, documents, and relevance judgments from your full dataset: + +```bash +python dataset/qa_dataset.py sample \ + --queries ~/dataset/mmarco-queries.parquet \ + --corpus ~/dataset/mmarco-corpus.parquet \ + --qrels ~/dataset/mmarco-qrels.parquet \ + --nq 100 \ + --output_dir ./dataset/samples +``` + +### 2. Generate Answers + +Use OpenAI's GPT model to generate answers for the sampled questions: + +```bash +python dataset/qa_dataset.py generate \ + --input_dir ./dataset/samples \ + --output_dir ./dataset/samples +``` + +### 3. View Results + +Display the generated QA pairs with their context: + +```bash +python dataset/qa_dataset.py show \ + --input_dir ./dataset/samples \ + -n 5 +``` + +## Detailed Usage + +### Sample Command + +Create a representative sample from your full dataset. + +```bash +python dataset/qa_dataset.py sample [OPTIONS] +``` + +**Required Parameters:** +- `--queries`: Path to queries parquet file (columns: `id`, `text`) +- `--corpus`: Path to corpus parquet file (columns: `id`, `text`) +- `--qrels`: Path to qrels parquet file (columns: `qid`, `pid`) + +**Optional Parameters:** +- `--nq`: Number of queries to sample (default: 1000) +- `--output_dir`: Output directory for sampled data (default: ./save) + +**Example:** +```bash +python dataset/qa_dataset.py sample \ + --queries data/queries.parquet \ + --corpus data/corpus.parquet \ + --qrels data/qrels.parquet \ + --nq 500 \ + --output_dir ./my_sample +``` + +### Generate Command + +Generate answers for sampled questions using OpenAI API. + +```bash +python dataset/qa_dataset.py generate [OPTIONS] +``` + +**Required Parameters:** +- `--input_dir`: Directory containing sampled data (queries.parquet, corpus.parquet, qrels.parquet) + +**Optional Parameters:** +- `--output_dir`: Output directory for generated answers (default: ./save) + +**Features:** +- **Resume Support**: Automatically continues from where it left off if interrupted +- **Error Handling**: Retries failed API calls up to 3 times +- **Progress Saving**: Saves progress after each successful answer generation + +**Example:** +```bash +python dataset/qa_dataset.py generate \ + --input_dir ./my_sample \ + --output_dir ./my_sample +``` + +### Show Command + +Display generated QA pairs with full context. + +```bash +python dataset/qa_dataset.py show [OPTIONS] +``` + +**Required Parameters:** +- `--input_dir`: Directory containing QA data (queries.parquet, corpus.parquet, qrels.parquet, qas.parquet, answers.parquet) + +**Optional Parameters:** +- `-n`: Number of results to display (default: 5) + +**Example:** +```bash +python dataset/qa_dataset.py show \ + --input_dir ./my_sample \ + -n 3 +``` + +## Input Data Format + +### Queries File (queries.parquet) +| Column | Type | Description | +|--------|------|-------------| +| id | string | Unique query identifier | +| text | string | The actual question text | + +### Corpus File (corpus.parquet) +| Column | Type | Description | +|--------|------|-------------| +| id | string | Unique passage/document identifier | +| text | string | The passage/document content | + +### Qrels File (qrels.parquet) +| Column | Type | Description | +|--------|------|-------------| +| qid | string | Query ID (matches queries.id) | +| pid | string | Passage ID (matches corpus.id) | + +## Output Files + +After running all commands, your output directory will contain: + +### Sampled Data +- `queries.parquet`: Sampled queries subset +- `corpus.parquet`: Sampled documents subset +- `qrels.parquet`: Sampled relevance judgments + +### Generated Answers +- `answers.parquet`: Generated answers with unique IDs +- `qas.parquet`: Question-answer mapping (qid → aid) + +## Advanced Usage + +### Custom OpenAI Configuration + +You can use different OpenAI models or endpoints: + +```bash +# Use GPT-4 Turbo +export OPENAI_API_KEY="your-key" +python dataset/qa_dataset.py generate --input_dir ./samples + +# Use Azure OpenAI +export OPENAI_API_KEY="azure-key" +export OPENAI_BASE_URL="https://your-resource.openai.azure.com/openai/deployments/gpt-4" +python dataset/qa_dataset.py generate --input_dir ./samples +``` + +### Large Dataset Sampling + +For very large datasets, consider sampling in batches: + +```bash +# First batch +python dataset/qa_dataset.py sample --nq 1000 --output_dir ./batch1 +python dataset/qa_dataset.py generate --input_dir ./batch1 + +# Second batch +python dataset/qa_dataset.py sample --nq 1000 --output_dir ./batch2 +python dataset/qa_dataset.py generate --input_dir ./batch2 +``` + +## Troubleshooting + +### Common Issues + +**1. OpenAI API Errors** +- Ensure your API key is set correctly: `echo $OPENAI_API_KEY` +- Check your API quota and billing status +- Verify network connectivity to OpenAI + +**2. Memory Issues with Large Datasets** +- Reduce `--nq` parameter for smaller samples +- Ensure sufficient RAM for pandas operations +- Consider using smaller parquet files + +**3. File Not Found Errors** +- Verify all input file paths are correct +- Ensure parquet files have correct column names +- Check file permissions + +### Debug Mode + +Enable verbose output by adding print statements or using Python debugger: + +```bash +python -m pdb dataset/qa_dataset.py sample --queries ... +``` + +## Example Workflow + +```bash +# 1. Setup environment +export OPENAI_API_KEY="sk-..." + +# 2. Sample 200 queries from MS MARCO +python dataset/qa_dataset.py sample \ + --queries ~/mmarco/queries.parquet \ + --corpus ~/mmarco/corpus.parquet \ + --qrels ~/mmarco/qrels.parquet \ + --nq 200 \ + --output_dir ./marco_sample + +# 3. Generate answers (may take time depending on API rate limits) +python dataset/qa_dataset.py generate \ + --input_dir ./marco_sample \ + --output_dir ./marco_sample + +# 4. Review results +python dataset/qa_dataset.py show \ + --input_dir ./marco_sample \ + -n 10 +``` + +## Contributing + +Feel free to submit issues and enhancement requests! + +## License + +MIT License - feel free to use this tool for your research and projects. \ No newline at end of file diff --git a/dataset/README_zh.md b/dataset/README_zh.md new file mode 100644 index 0000000..2c4e5b0 --- /dev/null +++ b/dataset/README_zh.md @@ -0,0 +1,284 @@ +# QA数据集采样工具 + +一个全面的QA数据集采样工具，使用OpenAI的GPT模型生成答案。该工具帮助您从大规模数据集（如MS MARCO）创建高质量的问答数据集。 + +## 功能特性 + +- **智能采样**：智能地从大型数据集中采样查询、文档和相关性判断 +- **答案生成**：使用OpenAI的GPT模型自动生成高质量答案 +- **断点续传**：支持中断后继续生成，从上次位置开始 +- **进度跟踪**：实时进度更新和统计信息 +- **结果可视化**：易于阅读的问答对展示，包含完整上下文 + +## 安装指南 + +### 系统要求 + +- Python 3.7+ +- OpenAI API密钥 + +### 安装依赖 + +```bash +pip install pandas pyarrow openai +``` + +### 设置环境变量 + +```bash +export OPENAI_API_KEY="你的openai-api-key" +# 可选：使用自定义OpenAI端点 +export OPENAI_BASE_URL="https://api.openai.com/v1" +``` + +### 准备数据集 + +您可以使用任何符合格式要求的QA数据集，或下载预处理好的样本： + +**使用HuggingFace/ModelScope样本** +我们提供了来自流行QA数据集的预处理样本： +- MarkrAI/eli5_sample_autorag +- MarkrAI/msmarco_sample_autorag +- MarkrAI/triviaqa_sample_autorag +- gnekt/hotpotqa_small_sample_autorag + +**使用您自己的数据集** +确保您的数据集包含以下文件： +- `queries.parquet`（列：id, text） +- `corpus.parquet`（列：id, text） +- `qrels.parquet`（列：qid, pid） + +## 快速开始 + +### 1. 从大型数据集采样 + +首先，从完整数据集中采样查询、文档和相关性判断的子集： + +```bash +python dataset/qa_dataset.py sample \ + --queries ~/dataset/mmarco-queries.parquet \ + --corpus ~/dataset/mmarco-corpus.parquet \ + --qrels ~/dataset/mmarco-qrels.parquet \ + --nq 100 \ + --output_dir ./dataset/samples +``` + +### 2. 生成答案 + +使用OpenAI的GPT模型为采样的问答生成答案： + +```bash +python dataset/qa_dataset.py generate \ + --input_dir ./dataset/samples \ + --output_dir ./dataset/samples +``` + +### 3. 查看结果 + +展示生成的问答对及其上下文： + +```bash +python dataset/qa_dataset.py show \ + --input_dir ./dataset/samples \ + -n 5 +``` + +## 详细使用说明 + +### 采样命令 + +从完整数据集中创建代表性样本。 + +```bash +python dataset/qa_dataset.py sample [选项] +``` + +**必需参数：** +- `--queries`：查询parquet文件路径（列：`id`, `text`） +- `--corpus`：语料库parquet文件路径（列：`id`, `text`） +- `--qrels`：相关性判断parquet文件路径（列：`qid`, `pid`） + +**可选参数：** +- `--nq`：要采样的查询数量（默认：1000） +- `--output_dir`：采样数据输出目录（默认：./save） + +**示例：** +```bash +python dataset/qa_dataset.py sample \ + --queries data/queries.parquet \ + --corpus data/corpus.parquet \ + --qrels data/qrels.parquet \ + --nq 500 \ + --output_dir ./my_sample +``` + +### 生成命令 + +使用OpenAI API为采样问题生成答案。 + +```bash +python dataset/qa_dataset.py generate [选项] +``` + +**必需参数：** +- `--input_dir`：包含采样数据的目录（queries.parquet, corpus.parquet, qrels.parquet） + +**可选参数：** +- `--output_dir`：生成答案的输出目录（默认：./save） + +**特性：** +- **断点续传**：中断后自动从上次位置继续 +- **错误处理**：API调用失败自动重试3次 +- **进度保存**：每成功生成一个答案就保存进度 + +**示例：** +```bash +python dataset/qa_dataset.py generate \ + --input_dir ./my_sample \ + --output_dir ./my_sample +``` + +### 展示命令 + +展示生成的问答对及完整上下文。 + +```bash +python dataset/qa_dataset.py show [选项] +``` + +**必需参数：** +- `--input_dir`：包含QA数据的目录（queries.parquet, corpus.parquet, qrels.parquet, qas.parquet, answers.parquet） + +**可选参数：** +- `-n`：要展示的结果数量（默认：5） + +**示例：** +```bash +python dataset/qa_dataset.py show \ + --input_dir ./my_sample \ + -n 3 +``` + +## 输入数据格式 + +### 查询文件 (queries.parquet) +| 列名 | 类型 | 描述 | +|------|------|------| +| id | string | 唯一查询标识符 | +| text | string | 实际的问题文本 | + +### 语料库文件 (corpus.parquet) +| 列名 | 类型 | 描述 | +|------|------|------| +| id | string | 唯一段落/文档标识符 | +| text | string | 段落/文档内容 | + +### 相关性判断文件 (qrels.parquet) +| 列名 | 类型 | 描述 | +|------|------|------| +| qid | string | 查询ID（匹配queries.id） | +| pid | string | 段落ID（匹配corpus.id） | + +## 输出文件 + +运行所有命令后，输出目录将包含： + +### 采样数据 +- `queries.parquet`：采样的查询子集 +- `corpus.parquet`：采样的文档子集 +- `qrels.parquet`：采样的相关性判断 + +### 生成的答案 +- `answers.parquet`：生成的答案（含唯一ID） +- `qas.parquet`：问答映射（qid → aid） + +## 高级用法 + +### 自定义OpenAI配置 + +您可以使用不同的OpenAI模型或端点： + +```bash +# 使用GPT-4 Turbo +export OPENAI_API_KEY="你的密钥" +python dataset/qa_dataset.py generate --input_dir ./samples + +# 使用Azure OpenAI +export OPENAI_API_KEY="azure密钥" +export OPENAI_BASE_URL="https://你的资源.openai.azure.com/openai/deployments/gpt-4" +python dataset/qa_dataset.py generate --input_dir ./samples +``` + +### 大型数据集采样 + +对于非常大的数据集，建议分批采样： + +```bash +# 第一批 +python dataset/qa_dataset.py sample --nq 1000 --output_dir ./batch1 +python dataset/qa_dataset.py generate --input_dir ./batch1 + +# 第二批 +python dataset/qa_dataset.py sample --nq 1000 --output_dir ./batch2 +python dataset/qa_dataset.py generate --input_dir ./batch2 +``` + +## 故障排除 + +### 常见问题 + +**1. OpenAI API错误** +- 确保API密钥设置正确：`echo $OPENAI_API_KEY` +- 检查API配额和账单状态 +- 验证与OpenAI的网络连接 + +**2. 大数据集内存问题** +- 减小`--nq`参数以获得更小的样本 +- 确保pandas操作有足够的RAM +- 考虑使用更小的parquet文件 + +**3. 文件未找到错误** +- 验证所有输入文件路径是否正确 +- 确保parquet文件有正确的列名 +- 检查文件权限 + +### 调试模式 + +通过添加打印语句或使用Python调试器启用详细输出： + +```bash +python -m pdb dataset/qa_dataset.py sample --queries ... +``` + +## 示例工作流 + +```bash +# 1. 设置环境 +export OPENAI_API_KEY="sk-..." + +# 2. 从MS MARCO采样200个查询 +python dataset/qa_dataset.py sample \ + --queries ~/mmarco/queries.parquet \ + --corpus ~/mmarco/corpus.parquet \ + --qrels ~/mmarco/qrels.parquet \ + --nq 200 \ + --output_dir ./marco_sample + +# 3. 生成答案（根据API速率限制可能需要一些时间） +python dataset/qa_dataset.py generate \ + --input_dir ./marco_sample \ + --output_dir ./marco_sample + +# 4. 查看结果 +python dataset/qa_dataset.py show \ + --input_dir ./marco_sample \ + -n 10 +``` + +## 贡献 + +欢迎提交问题和功能增强请求！ + +## 许可证 + +MIT许可证 - 可自由用于研究和项目。 \ No newline at end of file diff --git a/dataset/qa_dataset.py b/dataset/qa_dataset.py new file mode 100644 index 0000000..45f8265 --- /dev/null +++ b/dataset/qa_dataset.py @@ -0,0 +1,381 @@ +""" +QA Dataset Sampling Tool + +``` +pip install pandas pyarrow +pip install openai +``` + +# 采样数据 +python dataset/qa_dataset.py sample \ + --queries ~/dataset/mmarco-queries.parquet \ + --corpus ~/dataset/mmarco-corpus.parquet \ + --qrels ~/dataset/mmarco-qrels.parquet \ + --nq 100 \ + --output_dir ./dataset/samples + +# 生成答案(基于采样结果) +python dataset/qa_dataset.py generate \ + --input_dir ./dataset/samples \ + --output_dir ./dataset/samples + +# 展示结果 +python dataset/qa_dataset.py show \ + --input_dir ./dataset/samples \ + -n 1 +""" + +import os +from pathlib import Path +import argparse + +import pandas as pd +import openai + + +def read_parquet(path): + return pd.read_parquet(path) + + +def save_to_parquet(df: pd.DataFrame, path: str): + """Save DataFrame to parquet file""" + Path(path).parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(path) + print(f"Saved to {path}") + + +def print_stats(df: pd.DataFrame, name: str): + """Print statistics of a DataFrame""" + print(f"\n{name} Statistics:") + print(f"- Total records: {len(df)}") + if "id" in df.columns: + print(f"- Unique ids: {df['id'].nunique()}") + if "qid" in df.columns: + print(f"- Unique qids: {df['qid'].nunique()}") + if "pid" in df.columns: + print(f"- Unique pids: {df['pid'].nunique()}") + + +def sample_data( + queries: pd.DataFrame, corpus: pd.DataFrame, qrels: pd.DataFrame, nq=1000 +): + """ + Sample data from the dataset with validation checks. + + Args: + queries: DataFrame with qid and text columns (one-to-one) + corpus: DataFrame with pid and text columns (one-to-one) + qrels: DataFrame with qid and pid columns (many-to-many) + nq: Number of queries to sample (default: 1000) + + Returns: + Tuple of (sampled_queries, sampled_corpus, sampled_qrels) + """ + # 1. Filter qrels to only include qids that exist in queries + valid_qids = set(queries["id"]) + qrels = qrels[qrels["qid"].isin(valid_qids)] + + # 2. Filter qrels to only include pids that exist in corpus + valid_pids = set(corpus["id"]) + qrels = qrels[qrels["pid"].isin(valid_pids)] + + # 3. Sample queries (ensure we have enough qrels samples for each) + # Get qids with most associated pids to ensure diversity + qid_counts = qrels["qid"].value_counts() + sampled_qids = qid_counts.nlargest(min(nq, len(qid_counts))).index + + # 4. Get all pids associated with sampled qids + sampled_qrels = qrels[qrels["qid"].isin(sampled_qids)] + sampled_pids = set(sampled_qrels["pid"]) + + # 5. Add extra pids from corpus for redundancy (20% of sampled pids) + extra_pids = set(corpus["id"].sample(int(0.2 * len(sampled_pids)))) + all_pids = sampled_pids.union(extra_pids) + + # 6. Create final sampled datasets + sampled_queries = queries[queries["id"].isin(sampled_qids)] + sampled_corpus = corpus[corpus["id"].isin(all_pids)] + + return sampled_queries, sampled_corpus, sampled_qrels + + +class QAAnsweringSystem: + def __init__( + self, queries: pd.DataFrame, corpus: pd.DataFrame, qrels: pd.DataFrame + ): + """ + Initialize QA system with data + + Args: + queries: DataFrame with qid and text columns + corpus: DataFrame with pid and text columns + qrels: DataFrame with qid and pid mapping + """ + self.queries = queries + self.corpus = corpus + self.qrels = qrels + self.client = openai.Client( + api_key=os.getenv("OPENAI_API_KEY"), + base_url=os.getenv("OPENAI_BASE_URL"), + ) + + # Create lookup dictionaries + self.qid_to_text = dict(zip(queries["id"], queries["text"])) + self.pid_to_text = dict(zip(corpus["id"], corpus["text"])) + self.qid_to_pids = qrels.groupby("qid")["pid"].apply(list).to_dict() + + def get_context_for_qid(self, qid: str) -> str: + """ + Get all relevant text for a query ID + + Args: + qid: Query ID to search for + + Returns: + Combined context text from all related passages + """ + if qid not in self.qid_to_pids: + raise ValueError("Question ID not found") + + context_parts = [] + print(f"Context for Question ID {qid}: {self.qid_to_pids[qid]}") + for pid in self.qid_to_pids[qid]: + if pid in self.pid_to_text: + context_parts.append(self.pid_to_text[pid]) + + return "\n\n".join(context_parts) + + def answer_question(self, qid: str, model: str = "gpt-4o-2024-05-13") -> str: + """ + Use OpenAI API to answer question based on qid context + + Args: + qid: Query ID to answer + model: OpenAI model to use + + Returns: + Generated answer from LLM + """ + if qid not in self.qid_to_text: + raise ValueError("Question ID not found") + + question = self.qid_to_text[qid] + context = self.get_context_for_qid(qid) + + if not context: + raise ValueError("No context found for this question") + + prompt = f"""Answer the question based on the context below. Keep the answer concise. + +Question: {question} + +Context: {context} + +Answer:""" + response = self.client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=0.3, + ) + return response.choices[0].message.content + + +def sample_command(args): + """Handle sample command""" + # Load data + print("Loading data...") + queries = read_parquet(args.queries) + corpus = read_parquet(args.corpus) + qrels = read_parquet(args.qrels) + + # Print original stats + print("\nOriginal Dataset Statistics:") + print_stats(queries, "Queries") + print_stats(corpus, "Corpus") + print_stats(qrels, "Qrels") + + # Sample data + print(f"\nSampling {args.nq} queries...") + sampled_queries, sampled_corpus, sampled_qrels = sample_data( + queries, corpus, qrels, args.nq + ) + + # Print sampled stats + print("\nSampled Dataset Statistics:") + print_stats(sampled_queries, "Sampled Queries") + print_stats(sampled_corpus, "Sampled Corpus") + print_stats(sampled_qrels, "Sampled Qrels") + + # Save sampled data + print("\nSaving sampled data...") + save_to_parquet(sampled_queries, f"{args.output_dir}/queries.parquet") + save_to_parquet(sampled_corpus, f"{args.output_dir}/corpus.parquet") + save_to_parquet(sampled_qrels, f"{args.output_dir}/qrels.parquet") + print("\nSampling completed successfully!") + + +def generate_answers(input_dir: str, output_dir: str, max_retries: int = 3): + """ + Generate answers for sampled queries with resume support + + Args: + input_dir: Directory containing sampled queries/corpus/qrels + output_dir: Directory to save answer files + max_retries: Maximum retry attempts for failed queries + """ + print("\nLoading sampled data...") + queries = read_parquet(f"{input_dir}/queries.parquet") + corpus = read_parquet(f"{input_dir}/corpus.parquet") + qrels = read_parquet(f"{input_dir}/qrels.parquet") + + # Try to load existing answers if any + answers_path = f"{output_dir}/answers.parquet" + qa_pairs_path = f"{output_dir}/qas.parquet" + + try: + existing_answers = read_parquet(answers_path) + existing_qas = read_parquet(qa_pairs_path) + processed_qids = set(existing_qas["qid"]) + print(f"\nFound {len(processed_qids)} previously processed queries") + except (FileNotFoundError, KeyError): + print("No existing answers found, use empty state") + existing_answers = pd.DataFrame(columns=["id", "text"]) + existing_qas = pd.DataFrame(columns=["qid", "aid"]) + processed_qids = set() + + qa_system = QAAnsweringSystem(queries, corpus, qrels) + + answers = existing_answers.to_dict("records") + qa_pairs = existing_qas.to_dict("records") + answer_id_counter = len(answers) + 1 + + for qid in queries["id"]: + if qid in processed_qids: + continue + + retry_count = 0 + while retry_count <= max_retries: + try: + answer_text = qa_system.answer_question(qid) + aid = answer_id_counter + answers.append({"id": aid, "text": answer_text}) + qa_pairs.append({"qid": qid, "aid": aid}) + answer_id_counter += 1 + + # Save progress after each successful answer + save_to_parquet(pd.DataFrame(answers), answers_path) + save_to_parquet(pd.DataFrame(qa_pairs), qa_pairs_path) + print(f"Processed qid: {qid}") + break + except (openai.APIError, openai.APIConnectionError) as e: + retry_count += 1 + if retry_count > max_retries: + print( + f"\nFailed to process qid {qid} after {max_retries} attempts: {str(e)}" + ) + # Save failed state + save_to_parquet(pd.DataFrame(answers), answers_path) + save_to_parquet(pd.DataFrame(qa_pairs), qa_pairs_path) + else: + print(f"\nRetry {retry_count} for qid {qid}...") + + print("\nAnswer generation completed!") + print(f"Total queries: {len(queries)}") + print(f"Successfully processed: {len(qa_pairs)}") + print(f"Failed queries: {len(queries) - len(qa_pairs)}") + + +def show_results(input_dir: str, n: int = 5): + """ + Show n random results with question, context and answer + + Args: + input_dir: Directory containing the QA data + n: Number of results to show (default: 5) + """ + print(f"\nShowing {n} random results:") + + # Load data + queries = read_parquet(f"{input_dir}/queries.parquet") + corpus = read_parquet(f"{input_dir}/corpus.parquet") + qrels = read_parquet(f"{input_dir}/qrels.parquet") + qa_pairs = read_parquet(f"{input_dir}/qas.parquet") + answers = read_parquet(f"{input_dir}/answers.parquet") + + # Create QA system for context lookup + qa_system = QAAnsweringSystem(queries, corpus, qrels) + + # Get first n QA pairs + for _, row in qa_pairs.sample(n).iterrows(): + qid = row["qid"] + aid = row["aid"] + + # Get question + question = qa_system.qid_to_text[qid] + + # Get context + context = qa_system.get_context_for_qid(qid) + + # Get answer + answer = answers[answers["id"] == aid]["text"].values[0] + + print("\n" + "=" * 50) + print(f"Question (qid={qid}):\n{question}") + print("\nContext:") + print(context) + print(f"\nAnswer (aid={aid}):\n{answer}") + print("=" * 50 + "\n") + + +def main(): + # Set up command line arguments + parser = argparse.ArgumentParser(description="QA Dataset Tool") + subparsers = parser.add_subparsers(dest="command", required=True) + + # Sample command + sample_parser = subparsers.add_parser("sample", help="Sample dataset") + sample_parser.add_argument( + "--queries", type=str, required=True, help="Path to queries parquet file" + ) + sample_parser.add_argument( + "--corpus", type=str, required=True, help="Path to corpus parquet file" + ) + sample_parser.add_argument( + "--qrels", type=str, required=True, help="Path to qrels parquet file" + ) + sample_parser.add_argument( + "--nq", type=int, default=1000, help="Number of queries to sample" + ) + sample_parser.add_argument( + "--output_dir", type=str, default="./save", help="Output directory" + ) + sample_parser.set_defaults(func=sample_command) + + # Generate command + generate_parser = subparsers.add_parser("generate", help="Generate answers") + generate_parser.add_argument( + "--input_dir", type=str, required=True, help="Directory with sampled data" + ) + generate_parser.add_argument( + "--output_dir", type=str, default="./save", help="Output directory" + ) + generate_parser.set_defaults( + func=lambda args: generate_answers(args.input_dir, args.output_dir) + ) + + # Show command + show_parser = subparsers.add_parser("show", help="Show QA results") + show_parser.add_argument( + "--input_dir", type=str, required=True, help="Directory with QA data" + ) + show_parser.add_argument( + "-n", type=int, default=5, help="Number of results to show (default: 5)" + ) + show_parser.set_defaults(func=lambda args: show_results(args.input_dir, args.n)) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/dataset/samples/answers.parquet b/dataset/samples/answers.parquet new file mode 100644 index 0000000..0dc3ce5 Binary files /dev/null and b/dataset/samples/answers.parquet differ diff --git a/dataset/samples/corpus.parquet b/dataset/samples/corpus.parquet new file mode 100644 index 0000000..5984746 Binary files /dev/null and b/dataset/samples/corpus.parquet differ diff --git a/dataset/samples/qas.parquet b/dataset/samples/qas.parquet new file mode 100644 index 0000000..faa461e Binary files /dev/null and b/dataset/samples/qas.parquet differ diff --git a/dataset/samples/qrels.parquet b/dataset/samples/qrels.parquet new file mode 100644 index 0000000..1fde976 Binary files /dev/null and b/dataset/samples/qrels.parquet differ diff --git a/dataset/samples/queries.parquet b/dataset/samples/queries.parquet new file mode 100644 index 0000000..dc54a20 Binary files /dev/null and b/dataset/samples/queries.parquet differ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9f29764 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,246 @@ +services: + app: + image: wechatopenai/weknora-app:latest + container_name: WeKnora-app + ports: + - "${APP_PORT:-8080}:8080" + volumes: + - data-files:/data/files + - ./config/config.yaml:/app/config/config.yaml + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8080/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + environment: + - COS_SECRET_ID=${COS_SECRET_ID:-} + - COS_SECRET_KEY=${COS_SECRET_KEY:-} + - COS_REGION=${COS_REGION:-} + - COS_BUCKET_NAME=${COS_BUCKET_NAME:-} + - COS_APP_ID=${COS_APP_ID:-} + - COS_PATH_PREFIX=${COS_PATH_PREFIX:-} + - COS_ENABLE_OLD_DOMAIN=${COS_ENABLE_OLD_DOMAIN:-} + - GIN_MODE=${GIN_MODE:-} + - DB_DRIVER=postgres + - DB_HOST=postgres + - DB_PORT=5432 + - DB_USER=${DB_USER:-} + - DB_PASSWORD=${DB_PASSWORD:-} + - DB_NAME=${DB_NAME:-} + - TZ=Asia/Shanghai + - OTEL_EXPORTER_OTLP_ENDPOINT=jaeger:4317 + - OTEL_SERVICE_NAME=WeKnora + - OTEL_TRACES_EXPORTER=otlp + - OTEL_METRICS_EXPORTER=none + - OTEL_LOGS_EXPORTER=none + - OTEL_PROPAGATORS=tracecontext,baggage + - RETRIEVE_DRIVER=${RETRIEVE_DRIVER:-} + - ELASTICSEARCH_ADDR=${ELASTICSEARCH_ADDR:-} + - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME:-} + - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-} + - ELASTICSEARCH_INDEX=${ELASTICSEARCH_INDEX:-} + - DOCREADER_ADDR=docreader:50051 + - STORAGE_TYPE=${STORAGE_TYPE:-} + - LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-} + - MINIO_ENDPOINT=minio:9000 + - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin} + - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} + - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME:-} + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434} + - STREAM_MANAGER_TYPE=${STREAM_MANAGER_TYPE:-} + - REDIS_ADDR=redis:6379 + - REDIS_PASSWORD=${REDIS_PASSWORD:-} + - REDIS_DB=${REDIS_DB:-} + - REDIS_PREFIX=${REDIS_PREFIX:-} + - ENABLE_GRAPH_RAG=${ENABLE_GRAPH_RAG:-} + - NEO4J_ENABLE=${NEO4J_ENABLE:-} + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-password} + - TENANT_AES_KEY=${TENANT_AES_KEY:-} + - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} + - INIT_LLM_MODEL_NAME=${INIT_LLM_MODEL_NAME:-} + - INIT_LLM_MODEL_BASE_URL=${INIT_LLM_MODEL_BASE_URL:-} + - INIT_LLM_MODEL_API_KEY=${INIT_LLM_MODEL_API_KEY:-} + - INIT_EMBEDDING_MODEL_NAME=${INIT_EMBEDDING_MODEL_NAME:-} + - INIT_EMBEDDING_MODEL_BASE_URL=${INIT_EMBEDDING_MODEL_BASE_URL:-} + - INIT_EMBEDDING_MODEL_API_KEY=${INIT_EMBEDDING_MODEL_API_KEY:-} + - INIT_EMBEDDING_MODEL_DIMENSION=${INIT_EMBEDDING_MODEL_DIMENSION:-} + - INIT_EMBEDDING_MODEL_ID=${INIT_EMBEDDING_MODEL_ID:-} + - INIT_RERANK_MODEL_NAME=${INIT_RERANK_MODEL_NAME:-} + - INIT_RERANK_MODEL_BASE_URL=${INIT_RERANK_MODEL_BASE_URL:-} + - INIT_RERANK_MODEL_API_KEY=${INIT_RERANK_MODEL_API_KEY:-} + depends_on: + redis: + condition: service_started + postgres: + condition: service_healthy + minio: + condition: service_started + docreader: + condition: service_healthy + neo4j: + condition: service_started + networks: + - WeKnora-network + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + + minio: + image: minio/minio:latest + container_name: WeKnora-minio + ports: + - "${MINIO_PORT:-9000}:9000" + - "${MINIO_CONSOLE_PORT:-9001}:9001" + environment: + - MINIO_ROOT_USER=${MINIO_ACCESS_KEY_ID:-minioadmin} + - MINIO_ROOT_PASSWORD=${MINIO_SECRET_ACCESS_KEY:-minioadmin} + command: server --console-address ":9001" /data + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - WeKnora-network + + frontend: + image: wechatopenai/weknora-ui:latest + container_name: WeKnora-frontend + ports: + - "${FRONTEND_PORT:-80}:80" + depends_on: + app: + condition: service_healthy + networks: + - WeKnora-network + restart: unless-stopped + + docreader: + image: wechatopenai/weknora-docreader:latest + container_name: WeKnora-docreader + ports: + - "${DOCREADER_PORT:-50051}:50051" + environment: + - COS_SECRET_ID=${COS_SECRET_ID:-} + - COS_SECRET_KEY=${COS_SECRET_KEY:-} + - COS_REGION=${COS_REGION:-} + - COS_BUCKET_NAME=${COS_BUCKET_NAME:-} + - COS_APP_ID=${COS_APP_ID:-} + - COS_PATH_PREFIX=${COS_PATH_PREFIX:-} + - COS_ENABLE_OLD_DOMAIN=${COS_ENABLE_OLD_DOMAIN:-} + - VLM_MODEL_BASE_URL=${VLM_MODEL_BASE_URL:-} + - VLM_MODEL_NAME=${VLM_MODEL_NAME:-} + - VLM_MODEL_API_KEY=${VLM_MODEL_API_KEY:-} + - STORAGE_TYPE=${STORAGE_TYPE:-} + - MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000} + - MINIO_ENDPOINT=minio:9000 + - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin} + - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} + - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME:-} + - MINIO_USE_SSL=${MINIO_USE_SSL:-} + - WEB_PROXY=${WEB_PROXY:-} + healthcheck: + test: ["CMD", "grpc_health_probe", "-addr=:50051"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + networks: + - WeKnora-network + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - "6831:6831/udp" # Jaeger Thrift接收器 + - "6832:6832/udp" # Jaeger Thrift接收器(Compact) + - "5778:5778" # 配置端口 + - "16686:16686" # Web UI + - "4317:4317" # OTLP gRPC接收器 + - "4318:4318" # OTLP HTTP接收器 + - "14250:14250" # 接收模型端口 + - "14268:14268" # Jaeger HTTP接收器 + - "9411:9411" # Zipkin兼容性端口 + environment: + - COLLECTOR_OTLP_ENABLED=true + - COLLECTOR_ZIPKIN_HOST_PORT=:9411 + volumes: + - jaeger_data:/var/lib/jaeger # 持久化 Jaeger 数据 + networks: + - WeKnora-network + restart: unless-stopped + # 修改的PostgreSQL配置 + postgres: + image: paradedb/paradedb:v0.18.9-pg17 + container_name: WeKnora-postgres + ports: + - "${DB_PORT}:5432" + environment: + - POSTGRES_USER=${DB_USER} + # NOCC:hardcode-password(工具误报) + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=${DB_NAME} + volumes: + - postgres-data:/var/lib/postgresql/data + - ./migrations/paradedb/00-init-db.sql:/docker-entrypoint-initdb.d/00-init-db.sql + - ./migrations/paradedb/01-migrate-to-paradedb.sql:/docker-entrypoint-initdb.d/01-migrate-to-paradedb.sql + networks: + - WeKnora-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER}"] + interval: 10s # 增加时间间隔 + timeout: 10s # 增加超时时间 + retries: 3 # 减少重试次数，让失败更快反馈 + start_period: 30s # 给予初始启动更多时间 + restart: unless-stopped + # 添加停机时的优雅退出时间 + stop_grace_period: 1m + + redis: + image: redis:7.0-alpine + container_name: WeKnora-redis + ports: + - "${REDIS_PORT}:6379" + volumes: + - redis_data:/data + command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD} + restart: always + networks: + - WeKnora-network + + neo4j: + image: neo4j:latest + container_name: WeKnora-neo4j + volumes: + - neo4j-data:/data + environment: + - NEO4J_AUTH=${NEO4J_USERNAME:-neo4j}/${NEO4J_PASSWORD:-password} + - NEO4J_apoc_export_file_enabled=true + - NEO4J_apoc_import_file_enabled=true + - NEO4J_apoc_import_file_use__neo4j__config=true + - NEO4JLABS_PLUGINS=["apoc"] + ports: + - "7474:7474" + - "7687:7687" + restart: always + networks: + - WeKnora-network + +networks: + WeKnora-network: + driver: bridge + +volumes: + postgres-data: + data-files: + jaeger_data: + redis_data: + minio_data: + neo4j-data: diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app new file mode 100644 index 0000000..777ee62 --- /dev/null +++ b/docker/Dockerfile.app @@ -0,0 +1,79 @@ +# Build stage +FROM golang:1.24-alpine AS builder + +WORKDIR /app + +# 通过构建参数接收敏感信息 +ARG GOPRIVATE_ARG +ARG GOPROXY_ARG +ARG GOSUMDB_ARG=off + +# 设置Go环境变量 +ENV GOPRIVATE=${GOPRIVATE_ARG} +ENV GOPROXY=${GOPROXY_ARG} +ENV GOSUMDB=${GOSUMDB_ARG} + +# Install dependencies +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \ + apk add --no-cache git build-base + +# Install migrate tool +RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest + +# Copy go mod and sum files +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . + +# Get version and commit info for build injection +ARG VERSION_ARG +ARG COMMIT_ID_ARG +ARG BUILD_TIME_ARG +ARG GO_VERSION_ARG + +# Set build-time variables +ENV VERSION=${VERSION_ARG} +ENV COMMIT_ID=${COMMIT_ID_ARG} +ENV BUILD_TIME=${BUILD_TIME_ARG} +ENV GO_VERSION=${GO_VERSION_ARG} + +# Build the application with version info +RUN --mount=type=cache,target=/go/pkg/mod make build-prod +RUN --mount=type=cache,target=/go/pkg/mod cp -r /go/pkg/mod/github.com/yanyiwu/ /app/yanyiwu/ + +# Final stage +FROM alpine:3.17 + +WORKDIR /app + +# Install runtime dependencies +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories && \ + apk update && apk upgrade && \ + apk add --no-cache build-base postgresql-client mysql-client ca-certificates tzdata sed curl bash vim wget + +# Create a non-root user and switch to it +RUN mkdir -p /data/files && \ + adduser -D -g '' appuser && \ + chown -R appuser:appuser /app /data/files + +# Copy migrate tool from builder stage +COPY --from=builder /go/bin/migrate /usr/local/bin/ +COPY --from=builder /app/yanyiwu/ /go/pkg/mod/github.com/yanyiwu/ + +# Copy the binary from the builder stage +COPY --from=builder /app/config ./config +COPY --from=builder /app/scripts ./scripts +COPY --from=builder /app/migrations ./migrations +COPY --from=builder /app/dataset/samples ./dataset/samples +COPY --from=builder /app/WeKnora . + +# Make scripts executable +RUN chmod +x ./scripts/*.sh + +# Expose ports +EXPOSE 8080 + +# Switch to non-root user and run the application directly +USER appuser + +CMD ["./WeKnora"] \ No newline at end of file diff --git a/docker/Dockerfile.docreader b/docker/Dockerfile.docreader new file mode 100644 index 0000000..89a0d40 --- /dev/null +++ b/docker/Dockerfile.docreader @@ -0,0 +1,160 @@ +# ========================= +# 构建阶段 +# ========================= +FROM python:3.10.18-bookworm AS builder + +# 切换 apt 源到清华 +RUN sed -i 's@http://deb.debian.org@https://mirrors.tuna.tsinghua.edu.cn@g' /etc/apt/sources.list.d/debian.sources && \ + sed -i 's@http://security.debian.org@https://mirrors.tuna.tsinghua.edu.cn@g' /etc/apt/sources.list.d/debian.sources + +WORKDIR /app + +# 安装构建依赖 +RUN apt-get update && apt-get install -y \ + gcc \ + python3-dev \ + libjpeg-dev \ + zlib1g-dev \ + libpq-dev \ + libffi-dev \ + libgl1 \ + libglib2.0-0 \ + wget \ + antiword \ + curl \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# 检查是否存在本地protoc安装包，如果存在则离线安装，否则在线安装,其他安装包按需求添加 +ARG TARGETARCH +COPY packages/ /app/packages/ +RUN echo "检查本地protoc安装包..." && \ + # 根据目标架构选择正确的protoc包名 + case ${TARGETARCH} in \ + "amd64") PROTOC_ARCH="x86_64" ;; \ + "arm64") PROTOC_ARCH="aarch_64" ;; \ + "arm") PROTOC_ARCH="arm" ;; \ + *) echo "Unsupported architecture for protoc: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + PROTOC_PACKAGE="protoc-3.19.4-linux-${PROTOC_ARCH}.zip" && \ + if [ -f "/app/packages/${PROTOC_PACKAGE}" ]; then \ + echo "发现本地protoc安装包，将进行离线安装"; \ + # 离线安装：使用本地包（精确路径避免歧义） + cp /app/packages/${PROTOC_PACKAGE} /app/ && \ + unzip -o /app/${PROTOC_PACKAGE} -d /usr/local && \ + chmod +x /usr/local/bin/protoc && \ + rm -f /app/${PROTOC_PACKAGE}; \ + else \ + echo "未发现本地protoc安装包，将进行在线安装"; \ + # 在线安装：从网络下载 + curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/${PROTOC_PACKAGE} && \ + unzip -o ${PROTOC_PACKAGE} -d /usr/local && \ + chmod +x /usr/local/bin/protoc && \ + rm -f ${PROTOC_PACKAGE}; \ + fi + +# 复制依赖文件 +COPY services/docreader/requirements.txt . + +# 安装依赖 +RUN pip cache purge && pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + +# 预下载 PP-OCRv4 模型 +RUN mkdir -p /root/.paddleocr/whl/det/ch && \ + mkdir -p /root/.paddleocr/whl/rec/ch && \ + mkdir -p /root/.paddleocr/whl/cls/ch && \ + # 下载检测模型 + wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_det_infer.tar \ + -O /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \ + tar -xf /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar -C /root/.paddleocr/whl/det/ch/ && \ + # 下载识别模型 + wget https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar \ + -O /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \ + tar -xf /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar -C /root/.paddleocr/whl/rec/ch/ && \ + # 下载文本方向分类模型（用于判断文本是否需要旋转） + wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_cls_infer.tar \ + -O /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar && \ + tar -xf /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar -C /root/.paddleocr/whl/cls/ && \ + # 清理压缩包 + rm -f /root/.paddleocr/whl/det/ch/ch_PP-OCRv4_det_infer.tar && \ + rm -f /root/.paddleocr/whl/rec/ch/ch_PP-OCRv4_rec_infer.tar && \ + rm -f /root/.paddleocr/whl/cls/ch_ppocr_mobile_v2.0_cls_infer.tar + +# 复制源代码和生成脚本 +COPY services/docreader/src/ /app/src/ +COPY services/docreader/scripts/ /app/scripts/ + +# 确保模型目录存在 +RUN ls -la /root/.paddleocr/whl/ + +# 生成 protobuf 代码 +RUN chmod +x /app/scripts/generate_proto.sh && bash /app/scripts/generate_proto.sh + + +# ========================= +# 运行阶段 +# ========================= +FROM python:3.10.18-bookworm AS runner + +# 切换 apt 源到清华 +RUN sed -i 's@http://deb.debian.org@https://mirrors.tuna.tsinghua.edu.cn@g' /etc/apt/sources.list.d/debian.sources && \ + sed -i 's@http://security.debian.org@https://mirrors.tuna.tsinghua.edu.cn@g' /etc/apt/sources.list.d/debian.sources + +WORKDIR /app + +# 安装运行时依赖 +RUN apt-get update && apt-get install -y \ + libjpeg62-turbo \ + libpq5 \ + wget \ + gnupg \ + libgl1 \ + libglib2.0-0 \ + antiword \ + vim \ + tar \ + dpkg \ + libxinerama1 \ + libfontconfig1 \ + libdbus-glib-1-2 \ + libcairo2 \ + libcups2 \ + libglu1-mesa \ + libsm6 \ + libreoffice \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# 安装 grpc_health_probe +ARG TARGETARCH +RUN GRPC_HEALTH_PROBE_VERSION=v0.4.24 && \ + # 根据目标架构选择正确的二进制文件 + case ${TARGETARCH} in \ + "amd64") ARCH="amd64" ;; \ + "arm64") ARCH="arm64" ;; \ + "arm") ARCH="arm" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${ARCH} && \ + chmod +x /bin/grpc_health_probe + +# 从构建阶段复制已安装的依赖和生成的代码 +COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin +COPY --from=builder /root/.paddleocr /root/.paddleocr + +# 安装 Playwright 浏览器 +RUN python -m playwright install webkit +RUN python -m playwright install-deps webkit + +COPY --from=builder /app/src /app/src + +# 设置 Python 路径 +ENV PYTHONPATH=/app/src +RUN cd /app/src && python -m download_deps + +# 暴露 gRPC 端口 +EXPOSE 50051 + +# 直接运行 Python 服务（日志输出到 stdout/stderr） +CMD ["python", "/app/src/server/server.py"] \ No newline at end of file diff --git a/docker/config/supervisord.conf b/docker/config/supervisord.conf new file mode 100644 index 0000000..44039e2 --- /dev/null +++ b/docker/config/supervisord.conf @@ -0,0 +1,31 @@ +[supervisord] +nodaemon=true +logfile=/var/log/supervisord.log +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=info +pidfile=/var/run/supervisord.pid +user=root + +[program:WeKnora] +command=/app/WeKnora +directory=/app +autostart=true +autorestart=true +startretries=5 +redirect_stderr=true +stdout_logfile=/var/log/WeKnora.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=10 +environment=CGO_ENABLED=1 +user=appuser + +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock \ No newline at end of file diff --git a/docker/docker-compose.minio.yml b/docker/docker-compose.minio.yml new file mode 100644 index 0000000..a78b0ab --- /dev/null +++ b/docker/docker-compose.minio.yml @@ -0,0 +1,29 @@ +version: "3.8" + +services: + minio: + image: minio/minio:latest + container_name: WeKnora-minio + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio-data:/data + environment: + - MINIO_ROOT_USER=${MINIO_ACCESS_KEY_ID} + - MINIO_ROOT_PASSWORD=${MINIO_SECRET_ACCESS_KEY} + command: server --console-address ":9001" /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - WeKnora-network + +volumes: + minio-data: + +networks: + WeKnora-network: + external: true \ No newline at end of file diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..ce1e014 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,2105 @@ +# WeKnora API 文档 + +## 目录 + +- [概述](#概述) +- [基础信息](#基础信息) +- [认证机制](#认证机制) +- [错误处理](#错误处理) +- [API 概览](#api-概览) +- [API 详细说明](#api-详细说明) + - [租户管理 API](#租户管理api) + - [知识库管理 API](#知识库管理api) + - [知识管理 API](#知识管理api) + - [模型管理 API](#模型管理api) + - [分块管理 API](#分块管理api) + - [会话管理 API](#会话管理api) + - [聊天功能 API](#聊天功能api) + - [消息管理 API](#消息管理api) + - [评估功能 API](#评估功能api) + +## 概述 + +WeKnora 提供了一系列 RESTful API，用于创建和管理知识库、检索知识，以及进行基于知识的问答。本文档详细描述了这些 API 的使用方式。 + +## 基础信息 + +- **基础 URL**: `/api/v1` +- **响应格式**: JSON +- **认证方式**: API Key + +## 认证机制 + +所有 API 请求需要在 HTTP 请求头中包含 `X-API-Key` 进行身份认证： + +``` +X-API-Key: your_api_key +``` + +为便于问题追踪和调试，建议每个请求的 HTTP 请求头中添加 `X-Request-ID`： + +``` +X-Request-ID: unique_request_id +``` + +### 获取 API Key + +获取 API Key 有以下方式： + +**创建租户时获取**：通过 `POST /api/v1/tenants` 接口创建新租户时，响应中会自动返回生成的 API Key。 + +请妥善保管您的 API Key，避免泄露。API Key 代表您的账户身份，拥有完整的 API 访问权限。 + +## 错误处理 + +所有 API 使用标准的 HTTP 状态码表示请求状态，并返回统一的错误响应格式： + +```json +{ + "success": false, + "error": { + "code": "错误代码", + "message": "错误信息", + "details": "错误详情" + } +} +``` + +## API 概览 + +WeKnora API 按功能分为以下几类： + +1. **租户管理**：创建和管理租户账户 +2. **知识库管理**：创建、查询和管理知识库 +3. **知识管理**：上传、检索和管理知识内容 +4. **模型管理**：配置和管理各种AI模型 +5. **分块管理**：管理知识的分块内容 +6. **会话管理**：创建和管理对话会话 +7. **聊天功能**：基于知识库进行问答 +8. **消息管理**：获取和管理对话消息 +9. **评估功能**：评估模型性能 + +## API 详细说明 + +以下是每个API的详细说明和示例。 + +### 租户管理API + +| 方法 | 路径 | 描述 | +| ------ | -------------- | --------------------- | +| POST | `/tenants` | 创建新租户 | +| GET | `/tenants/:id` | 获取指定租户信息 | +| PUT | `/tenants/:id` | 更新租户信息 | +| DELETE | `/tenants/:id` | 删除租户 | +| GET | `/tenants` | 获取租户列表 | + +#### POST `/tenants` - 创建新租户 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/tenants' \ +--header 'Content-Type: application/json' \ +--data '{ + "name": "weknora", + "description": "weknora tenants", + "business": "wechat", + "retriever_engines": { + "engines": [ + { + "retriever_type": "keywords", + "retriever_engine_type": "postgres" + }, + { + "retriever_type": "vector", + "retriever_engine_type": "postgres" + } + ] + } +}' +``` + +**响应**: + +```json +{ + "data": { + "id": 10000, + "name": "weknora", + "description": "weknora tenants", + "api_key": "sk-aaLRAgvCRJcmtiL2vLMeB1FB5UV0Q-qB7DlTE1pJ9KA93XZG", + "status": "active", + "retriever_engines": { + "engines": [ + { + "retriever_engine_type": "postgres", + "retriever_type": "keywords" + }, + { + "retriever_engine_type": "postgres", + "retriever_type": "vector" + } + ] + }, + "business": "wechat", + "storage_quota": 10737418240, + "storage_used": 0, + "created_at": "2025-08-11T20:37:28.396980093+08:00", + "updated_at": "2025-08-11T20:37:28.396980301+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### GET `/tenants/:id` - 获取指定租户信息 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/tenants/10000' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-aaLRAgvCRJcmtiL2vLMeB1FB5UV0Q-qB7DlTE1pJ9KA93XZG' +``` + +**响应**: + +```json +{ + "data": { + "id": 10000, + "name": "weknora", + "description": "weknora tenants", + "api_key": "sk-aaLRAgvCRJcmtiL2vLMeB1FB5UV0Q-qB7DlTE1pJ9KA93XZG", + "status": "active", + "retriever_engines": { + "engines": [ + { + "retriever_engine_type": "postgres", + "retriever_type": "keywords" + }, + { + "retriever_engine_type": "postgres", + "retriever_type": "vector" + } + ] + }, + "business": "wechat", + "storage_quota": 10737418240, + "storage_used": 0, + "created_at": "2025-08-11T20:37:28.39698+08:00", + "updated_at": "2025-08-11T20:37:28.405693+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### PUT `/tenants/:id` - 更新租户信息 + +注意 API Key 会变更 + +**请求**: + +```curl +curl --location --request PUT 'http://localhost:8080/api/v1/tenants/10000' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-KREi84yPtahKxMtIMOW-Cxx2dxb9xROpUuDSpi3vbiC1QVDe' \ +--data '{ + "name": "weknora new", + "description": "weknora tenants new", + "status": "active", + "retriever_engines": { + "engines": [ + { + "retriever_engine_type": "postgres", + "retriever_type": "keywords" + }, + { + "retriever_engine_type": "postgres", + "retriever_type": "vector" + } + ] + }, + "business": "wechat", + "storage_quota": 10737418240 +}' +``` + +**响应**: + +```json +{ + "data": { + "id": 10000, + "name": "weknora new", + "description": "weknora tenants new", + "api_key": "sk-IKtd9JGV4-aPGQ6RiL8YJu9Vzb3-ae4lgFkjFJZmhvUn2mLu", + "status": "active", + "retriever_engines": { + "engines": [ + { + "retriever_engine_type": "postgres", + "retriever_type": "keywords" + }, + { + "retriever_engine_type": "postgres", + "retriever_type": "vector" + } + ] + }, + "business": "wechat", + "storage_quota": 10737418240, + "storage_used": 0, + "created_at": "0001-01-01T00:00:00Z", + "updated_at": "2025-08-11T20:49:02.13421034+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### DELETE `/tenants/:id` - 删除租户 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/tenants/10000' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-IKtd9JGV4-aPGQ6RiL8YJu9Vzb3-ae4lgFkjFJZmhvUn2mLu' +``` + +**响应**: + +```json +{ + "message": "Tenant deleted successfully", + "success": true +} +``` + +#### GET `/tenants` - 获取租户列表 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/tenants' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-An7_t_izCKFIJ4iht9Xjcjnj_MC48ILvwezEDki9ScfIa7KA' +``` + +**响应**: + +```json +{ + "data": { + "items": [ + { + "id": 10002, + "name": "weknora", + "description": "weknora tenants", + "api_key": "sk-An7_t_izCKFIJ4iht9Xjcjnj_MC48ILvwezEDki9ScfIa7KA", + "status": "active", + "retriever_engines": { + "engines": [ + { + "retriever_engine_type": "postgres", + "retriever_type": "keywords" + }, + { + "retriever_engine_type": "postgres", + "retriever_type": "vector" + } + ] + }, + "business": "wechat", + "storage_quota": 10737418240, + "storage_used": 0, + "created_at": "2025-08-11T20:52:58.05679+08:00", + "updated_at": "2025-08-11T20:52:58.060495+08:00", + "deleted_at": null + } + ] + }, + "success": true +} +``` + + + +### 知识库管理API + +| 方法 | 路径 | 描述 | +| ------ | ------------------------------------ | ------------------------ | +| POST | `/knowledge-bases` | 创建知识库 | +| GET | `/knowledge-bases` | 获取知识库列表 | +| GET | `/knowledge-bases/:id` | 获取知识库详情 | +| PUT | `/knowledge-bases/:id` | 更新知识库 | +| DELETE | `/knowledge-bases/:id` | 删除知识库 | +| GET | `/knowledge-bases/:id/hybrid-search` | 混合搜索知识库内容 | +| POST | `/knowledge-bases/copy` | 拷贝知识库 | + +#### POST `/knowledge-bases` - 创建知识库 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-bases' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--data '{ + "name": "weknora", + "description": "weknora description", + "chunking_config": { + "chunk_size": 1000, + "chunk_overlap": 200, + "separators": [ + "." + ], + "enable_multimodal": true + }, + "image_processing_config": { + "model_id": "f2083ad7-63e3-486d-a610-e6c56e58d72e" + }, + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "rerank_model_id": "b30171a1-787b-426e-a293-735cd5ac16c0", + "vlm_model_id": "f2083ad7-63e3-486d-a610-e6c56e58d72e", + "vlm_config": { + "model_name": "qwen2.5vl:3b", + "interface_type": "ollama", + "base_url": "", + "api_key": "" + }, + "cos_config": { + "secret_id": "", + "secret_key": "", + "region": "", + "bucket_name": "", + "app_id": "", + "path_prefix": "" + } +}' +``` + +**响应**: + +```json +{ + "data": { + "id": "b5829e4a-3845-4624-a7fb-ea3b35e843b0", + "name": "weknora", + "description": "weknora description", + "tenant_id": 1, + "chunking_config": { + "chunk_size": 1000, + "chunk_overlap": 200, + "separators": [ + "." + ], + "enable_multimodal": true + }, + "image_processing_config": { + "model_id": "f2083ad7-63e3-486d-a610-e6c56e58d72e" + }, + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "rerank_model_id": "b30171a1-787b-426e-a293-735cd5ac16c0", + "vlm_model_id": "f2083ad7-63e3-486d-a610-e6c56e58d72e", + "vlm_config": { + "model_name": "qwen2.5vl:3b", + "base_url": "", + "api_key": "", + "interface_type": "ollama" + }, + "cos_config": { + "secret_id": "", + "secret_key": "", + "region": "", + "bucket_name": "", + "app_id": "", + "path_prefix": "" + }, + "created_at": "2025-08-12T11:30:09.206238645+08:00", + "updated_at": "2025-08-12T11:30:09.206238854+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### GET `/knowledge-bases` - 获取知识库列表 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-bases' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "kb-00000001", + "name": "Default Knowledge Base", + "description": "System Default Knowledge Base", + "tenant_id": 1, + "chunking_config": { + "chunk_size": 1000, + "chunk_overlap": 200, + "separators": [ + "\n\n", + "\n", + "。", + "！", + "？", + ";", + "；" + ], + "enable_multimodal": true + }, + "image_processing_config": { + "model_id": "" + }, + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "rerank_model_id": "b30171a1-787b-426e-a293-735cd5ac16c0", + "vlm_model_id": "f2083ad7-63e3-486d-a610-e6c56e58d72e", + "vlm_config": { + "model_name": "qwen2.5vl:3b", + "base_url": "http://host.docker.internal:11435/v1", + "api_key": "", + "interface_type": "ollama" + }, + "cos_config": { + "secret_id": "", + "secret_key": "", + "region": "", + "bucket_name": "", + "app_id": "", + "path_prefix": "" + }, + "created_at": "2025-08-11T20:10:41.817794+08:00", + "updated_at": "2025-08-12T11:23:00.593097+08:00", + "deleted_at": null + } + ], + "success": true +} +``` + +#### GET `/knowledge-bases/:id` - 获取知识库详情 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' +``` + +**响应**: + +```json +{ + "data": { + "id": "kb-00000001", + "name": "Default Knowledge Base", + "description": "System Default Knowledge Base", + "tenant_id": 1, + "chunking_config": { + "chunk_size": 1000, + "chunk_overlap": 200, + "separators": [ + "\n\n", + "\n", + "。", + "！", + "？", + ";", + "；" + ], + "enable_multimodal": true + }, + "image_processing_config": { + "model_id": "" + }, + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "rerank_model_id": "b30171a1-787b-426e-a293-735cd5ac16c0", + "vlm_model_id": "f2083ad7-63e3-486d-a610-e6c56e58d72e", + "vlm_config": { + "model_name": "qwen2.5vl:3b", + "base_url": "http://host.docker.internal:11435/v1", + "api_key": "", + "interface_type": "ollama" + }, + "cos_config": { + "secret_id": "", + "secret_key": "", + "region": "", + "bucket_name": "", + "app_id": "", + "path_prefix": "" + }, + "created_at": "2025-08-11T20:10:41.817794+08:00", + "updated_at": "2025-08-12T11:23:00.593097+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### PUT `/knowledge-bases/:id` - 更新知识库 + +**请求**: + +```curl +curl --location --request PUT 'http://localhost:8080/api/v1/knowledge-bases/b5829e4a-3845-4624-a7fb-ea3b35e843b0' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--data '{ + "name": "weknora new", + "description": "weknora description new", + "config": { + "chunking_config": { + "chunk_size": 1000, + "chunk_overlap": 200, + "separators": [ + "\n\n", + "\n", + "。", + "！", + "？", + ";", + "；" + ], + "enable_multimodal": true + }, + "image_processing_config": { + "model_id": "" + } + } +}' +``` + +**响应**: + +```json +{ + "data": { + "id": "b5829e4a-3845-4624-a7fb-ea3b35e843b0", + "name": "weknora new", + "description": "weknora description new", + "tenant_id": 1, + "chunking_config": { + "chunk_size": 1000, + "chunk_overlap": 200, + "separators": [ + "\n\n", + "\n", + "。", + "！", + "？", + ";", + "；" + ], + "enable_multimodal": true + }, + "image_processing_config": { + "model_id": "" + }, + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "rerank_model_id": "b30171a1-787b-426e-a293-735cd5ac16c0", + "vlm_model_id": "f2083ad7-63e3-486d-a610-e6c56e58d72e", + "vlm_config": { + "model_name": "qwen2.5vl:3b", + "base_url": "", + "api_key": "", + "interface_type": "ollama" + }, + "cos_config": { + "secret_id": "", + "secret_key": "", + "region": "", + "bucket_name": "", + "app_id": "", + "path_prefix": "" + }, + "created_at": "2025-08-12T11:30:09.206238+08:00", + "updated_at": "2025-08-12T11:36:09.083577609+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### DELETE `/knowledge-bases/:id` - 删除知识库 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/knowledge-bases/b5829e4a-3845-4624-a7fb-ea3b35e843b0' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' +``` + +**响应**: + +```json +{ + "message": "Knowledge base deleted successfully", + "success": true +} +``` + +#### GET `/knowledge-bases/:id/hybrid-search` - 混合搜索知识库内容 + +**请求**: + +```curl +curl --location --request GET 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001/hybrid-search' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--data '{ + "query_text": "彗星", + "vector_threshold": 0.1, + "keyword_threshold": 0.1, + "match_count": 1 +}' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "7d955251-3f79-4fd5-a6aa-02f81e044091", + "content": "有几位后来xxxxx", + "knowledge_id": "a6790b93-4700-4676-bd48-0d4804e1456b", + "chunk_index": 3, + "knowledge_title": "彗星.txt", + "start_at": 2287, + "end_at": 2760, + "seq": 3, + "score": 0.7402352891601821, + "match_type": 2, + "sub_chunk_id": null, + "metadata": {}, + "chunk_type": "text", + "parent_chunk_id": "", + "image_info": "", + "knowledge_filename": "彗星.txt", + "knowledge_source": "" + } + ], + "success": true +} +``` + + + +### 知识管理API + +| 方法 | 路径 | 描述 | +| ------ | ------------------------------------- | ------------------------ | +| POST | `/knowledge-bases/:id/knowledge/file` | 从文件创建知识 | +| POST | `/knowledge-bases/:id/knowledge/url` | 从 URL 创建知识 | +| GET | `/knowledge-bases/:id/knowledge` | 获取知识库下的知识列表 | +| GET | `/knowledge/:id` | 获取知识详情 | +| DELETE | `/knowledge/:id` | 删除知识 | +| GET | `/knowledge/:id/download` | 下载知识文件 | +| PUT | `/knowledge/:id` | 更新知识 | +| PUT | `/knowledge/image/:id/:chunk_id` | 更新图像分块信息 | +| GET | `/knowledge/batch` | 批量获取知识 | + +#### POST `/knowledge-bases/:id/knowledge/file` - 从文件创建知识 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001/knowledge/file' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--form 'file=@"/Users/xxxx/tests/彗星.txt"' \ +--form 'enable_multimodel="true"' +``` + +**响应**: + +```json +{ + "data": { + "id": "4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "type": "file", + "title": "彗星.txt", + "description": "", + "source": "", + "parse_status": "processing", + "enable_status": "disabled", + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "file_name": "彗星.txt", + "file_type": "txt", + "file_size": 7710, + "file_hash": "d69476ddbba45223a5e97e786539952c", + "file_path": "data/files/1/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5/1754970756171067621.txt", + "storage_size": 0, + "metadata": null, + "created_at": "2025-08-12T11:52:36.168632288+08:00", + "updated_at": "2025-08-12T11:52:36.173612121+08:00", + "processed_at": null, + "error_message": "", + "deleted_at": null + }, + "success": true +} +``` + +#### POST `/knowledge-bases/:id/knowledge/url` - 从 URL 创建知识 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001/knowledge/url' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' \ +--data '{ + "url":"https://github.com/Tencent/WeKnora", + "enable_multimodel":true +}' +``` + +**响应**: + +```json +{ + "data": { + "id": "9c8af585-ae15-44ce-8f73-45ad18394651", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "type": "url", + "title": "", + "description": "", + "source": "https://github.com/Tencent/WeKnora", + "parse_status": "processing", + "enable_status": "disabled", + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "file_name": "", + "file_type": "", + "file_size": 0, + "file_hash": "", + "file_path": "", + "storage_size": 0, + "metadata": null, + "created_at": "2025-08-12T11:55:05.709266776+08:00", + "updated_at": "2025-08-12T11:55:05.712918234+08:00", + "processed_at": null, + "error_message": "", + "deleted_at": null + }, + "success": true +} +``` + +#### GET `/knowledge-bases/:id/knowledge?page=&page_size` - 获取知识库下的知识列表 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-bases/kb-00000001/knowledge?page_size=1&page=1' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "9c8af585-ae15-44ce-8f73-45ad18394651", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "type": "url", + "title": "", + "description": "", + "source": "https://github.com/Tencent/WeKnora", + "parse_status": "pending", + "enable_status": "disabled", + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "file_name": "", + "file_type": "", + "file_size": 0, + "file_hash": "", + "file_path": "", + "storage_size": 0, + "metadata": null, + "created_at": "2025-08-12T11:55:05.709266+08:00", + "updated_at": "2025-08-12T11:55:05.709266+08:00", + "processed_at": null, + "error_message": "", + "deleted_at": null + } + ], + "page": 1, + "page_size": 1, + "success": true, + "total": 2 +} +``` + +注：parse_status 包含 `pending/processing/failed/completed` 四种状态 + +#### GET `/knowledge/:id` - 获取知识详情 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "data": { + "id": "4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "type": "file", + "title": "彗星.txt", + "description": "彗星是由冰和尘埃构成的太阳系小天体，接近太阳时会形成彗发和彗尾。其轨道周期差异大，来源包括柯伊伯带和奥尔特云。彗星与小行星的区别逐渐模糊，部分彗星已失去挥发物质，类似小行星。截至2019年，已知彗星超6600颗，数量庞大。彗星在古代被视为凶兆，现代研究揭示其复杂结构与起源。", + "source": "", + "parse_status": "completed", + "enable_status": "enabled", + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "file_name": "彗星.txt", + "file_type": "txt", + "file_size": 7710, + "file_hash": "d69476ddbba45223a5e97e786539952c", + "file_path": "data/files/1/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5/1754970756171067621.txt", + "storage_size": 33689, + "metadata": null, + "created_at": "2025-08-12T11:52:36.168632+08:00", + "updated_at": "2025-08-12T11:52:53.376871+08:00", + "processed_at": "2025-08-12T11:52:53.376573+08:00", + "error_message": "", + "deleted_at": null + }, + "success": true +} +``` + +#### GET `/knowledge/batch` - 批量获取知识 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge/batch?ids=9c8af585-ae15-44ce-8f73-45ad18394651&ids=4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "9c8af585-ae15-44ce-8f73-45ad18394651", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "type": "url", + "title": "", + "description": "", + "source": "https://github.com/Tencent/WeKnora", + "parse_status": "pending", + "enable_status": "disabled", + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "file_name": "", + "file_type": "", + "file_size": 0, + "file_hash": "", + "file_path": "", + "storage_size": 0, + "metadata": null, + "created_at": "2025-08-12T11:55:05.709266+08:00", + "updated_at": "2025-08-12T11:55:05.709266+08:00", + "processed_at": null, + "error_message": "", + "deleted_at": null + }, + { + "id": "4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "type": "file", + "title": "彗星.txt", + "description": "彗星是由冰和尘埃构成的太阳系小天体，接近太阳时会形成彗发和彗尾。其轨道周期差异大，来源包括柯伊伯带和奥尔特云。彗星与小行星的区别逐渐模糊，部分彗星已失去挥发物质，类似小行星。截至2019年，已知彗星超6600颗，数量庞大。彗星在古代被视为凶兆，现代研究揭示其复杂结构与起源。", + "source": "", + "parse_status": "completed", + "enable_status": "enabled", + "embedding_model_id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "file_name": "彗星.txt", + "file_type": "txt", + "file_size": 7710, + "file_hash": "d69476ddbba45223a5e97e786539952c", + "file_path": "data/files/1/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5/1754970756171067621.txt", + "storage_size": 33689, + "metadata": null, + "created_at": "2025-08-12T11:52:36.168632+08:00", + "updated_at": "2025-08-12T11:52:53.376871+08:00", + "processed_at": "2025-08-12T11:52:53.376573+08:00", + "error_message": "", + "deleted_at": null + } + ], + "success": true +} +``` + +#### DELETE `/knowledge/:id` - 删除知识 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/knowledge/9c8af585-ae15-44ce-8f73-45ad18394651' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "message": "Deleted successfully", + "success": true +} +``` + +#### GET `/knowledge/:id/download` - 下载知识文件 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5/download' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +``` +attachment +``` + + + +### 模型管理API + +| 方法 | 路径 | 描述 | +| ------ | --------------------- | --------------------- | +| POST | `/models` | 创建模型 | +| GET | `/models` | 获取模型列表 | +| GET | `/models/:id` | 获取模型详情 | +| PUT | `/models/:id` | 更新模型 | +| DELETE | `/models/:id` | 删除模型 | + +#### POST `/models` - 创建模型 + +创建对话模型（KnowledgeQA）请求体: + +```curl +curl --location 'http://localhost:8080/api/v1/models' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--data '{ + "name": "qwen3:8b", + "type": "KnowledgeQA", + "source": "local", + "description": "LLM Model for Knowledge QA", + "parameters": { + "base_url": "", + "api_key": "" + }, + "is_default": false +}' +``` + +创建嵌入模型（Embedding）请求体: + +```curl +curl --location 'http://localhost:8080/api/v1/models' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--data '{ + "name": "nomic-embed-text:latest", + "type": "Embedding", + "source": "local", + "description": "Embedding Model", + "parameters": { + "base_url": "", + "api_key": "", + "embedding_parameters": { + "dimension": 768, + "truncate_prompt_tokens": 0 + } + }, + "is_default": false +}' +``` + +创建排序模型（Rerank）请求体: + +```curl +curl --location 'http://localhost:8080/api/v1/models' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--data '{ + "name": "linux6200/bge-reranker-v2-m3:latest", + "type": "Rerank", + "source": "local", + "description": "Rerank Model for Knowledge QA", + "parameters": { + "base_url": "", + "api_key": "" + }, + "is_default": false +}' +``` + +**响应**: + +```json +{ + "data": { + "id": "09c5a1d6-ee8b-4657-9a17-d3dcbd5c70cb", + "tenant_id": 1, + "name": "nomic-embed-text:latest3", + "type": "Embedding", + "source": "local", + "description": "Embedding Model", + "parameters": { + "base_url": "", + "api_key": "", + "embedding_parameters": { + "dimension": 768, + "truncate_prompt_tokens": 0 + } + }, + "is_default": false, + "status": "downloading", + "created_at": "2025-08-12T10:39:01.454591766+08:00", + "updated_at": "2025-08-12T10:39:01.454591766+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### GET `/models` - 获取模型列表 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/models' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "tenant_id": 1, + "name": "nomic-embed-text:latest", + "type": "Embedding", + "source": "local", + "description": "Embedding Model", + "parameters": { + "base_url": "", + "api_key": "", + "embedding_parameters": { + "dimension": 768, + "truncate_prompt_tokens": 0 + } + }, + "is_default": true, + "status": "active", + "created_at": "2025-08-11T20:10:41.813832+08:00", + "updated_at": "2025-08-11T20:10:41.822354+08:00", + "deleted_at": null + }, + { + "id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "tenant_id": 1, + "name": "qwen3:8b", + "type": "KnowledgeQA", + "source": "local", + "description": "LLM Model for Knowledge QA", + "parameters": { + "base_url": "", + "api_key": "", + "embedding_parameters": { + "dimension": 0, + "truncate_prompt_tokens": 0 + } + }, + "is_default": true, + "status": "active", + "created_at": "2025-08-11T20:10:41.811761+08:00", + "updated_at": "2025-08-11T20:10:41.825381+08:00", + "deleted_at": null + } + ], + "success": true +} +``` + +#### GET `/models/:id` - 获取模型详情 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/models/dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' +``` + +**响应**: + +```json +{ + "data": { + "id": "dff7bc94-7885-4dd1-bfd5-bd96e4df2fc3", + "tenant_id": 1, + "name": "nomic-embed-text:latest", + "type": "Embedding", + "source": "local", + "description": "Embedding Model", + "parameters": { + "base_url": "", + "api_key": "", + "embedding_parameters": { + "dimension": 768, + "truncate_prompt_tokens": 0 + } + }, + "is_default": true, + "status": "active", + "created_at": "2025-08-11T20:10:41.813832+08:00", + "updated_at": "2025-08-11T20:10:41.822354+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### PUT `/models/:id` - 更新模型 + +**请求**: + +```curl +curl --location --request PUT 'http://localhost:8080/api/v1/models/8fdc464d-8eaa-44d4-a85b-094b28af5330' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--data '{ + "name": "linux6200/bge-reranker-v2-m3:latest", + "description": "Rerank Model for Knowledge QA new", + "parameters": { + "base_url": "", + "api_key": "" + }, + "is_default": false +}' +``` + +**响应**: + +```json +{ + "data": { + "id": "8fdc464d-8eaa-44d4-a85b-094b28af5330", + "tenant_id": 1, + "name": "linux6200/bge-reranker-v2-m3:latest", + "type": "Rerank", + "source": "local", + "description": "Rerank Model for Knowledge QA new", + "parameters": { + "base_url": "", + "api_key": "", + "embedding_parameters": { + "dimension": 0, + "truncate_prompt_tokens": 0 + } + }, + "is_default": false, + "status": "active", + "created_at": "2025-08-12T10:57:39.512681+08:00", + "updated_at": "2025-08-12T11:00:27.271678+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### DELETE `/models/:id` - 删除模型 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/models/8fdc464d-8eaa-44d4-a85b-094b28af5330' \ +--header 'Content-Type: application/json' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' +``` + +**响应**: + +```json +{ + "message": "Model deleted", + "success": true +} +``` + + + +### 分块管理API + +| 方法 | 路径 | 描述 | +| ------ | --------------------------- | ------------------------ | +| GET | `/chunks/:knowledge_id` | 获取知识的分块列表 | +| DELETE | `/chunks/:knowledge_id/:id` | 删除分块 | +| DELETE | `/chunks/:knowledge_id` | 删除知识下的所有分块 | + +#### GET `/chunks/:knowledge_id?page=&page_size=` - 获取知识的分块列表 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/chunks/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5?page=1&page_size=1' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "df10b37d-cd05-4b14-ba8a-e1bd0eb3bbd7", + "tenant_id": 0, + "knowledge_id": "4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5", + "knowledge_base_id": "kb-00000001", + "content": "彗星xxxx", + "chunk_index": 0, + "is_enabled": true, + "start_at": 0, + "end_at": 964, + "pre_chunk_id": "", + "next_chunk_id": "", + "chunk_type": "text", + "parent_chunk_id": "", + "relation_chunks": null, + "indirect_relation_chunks": null, + "image_info": "", + "created_at": "0001-01-01T00:00:00Z", + "updated_at": "0001-01-01T00:00:00Z", + "deleted_at": null + } + ], + "page": 1, + "page_size": 1, + "success": true, + "total": 5 +} +``` + +#### DELETE `/chunks/:knowledge_id/:id` - 删除分块 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/chunks/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5/df10b37d-cd05-4b14-ba8a-e1bd0eb3bbd7' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "message": "Chunk deleted", + "success": true +} +``` + +#### DELETE `/chunks/:knowledge_id` - 删除知识下的所有分块 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/chunks/4c4e7c1a-09cf-485b-a7b5-24b8cdc5acf5' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "message": "All chunks under knowledge deleted", + "success": true +} +``` + + + +### 会话管理API + +| 方法 | 路径 | 描述 | +| ------ | --------------------------------------- | --------------------- | +| POST | `/sessions` | 创建会话 | +| GET | `/sessions/:id` | 获取会话详情 | +| GET | `/sessions` | 获取租户的会话列表 | +| PUT | `/sessions/:id` | 更新会话 | +| DELETE | `/sessions/:id` | 删除会话 | +| POST | `/sessions/:session_id/generate_title` | 生成会话标题 | +| GET | `/sessions/continue-stream/:session_id` | 继续未完成的会话 | + +#### POST `/sessions` - 创建会话 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/sessions' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' \ +--data '{ + "knowledge_base_id": "kb-00000001", + "session_strategy": { + "max_rounds": 5, + "enable_rewrite": true, + "fallback_strategy": "FIXED_RESPONSE", + "fallback_response": "对不起，我无法回答这个问题", + "embedding_top_k": 10, + "keyword_threshold": 0.5, + "vector_threshold": 0.7, + "rerank_model_id": "排序模型ID", + "rerank_top_k": 3, + "rerank_threshold": 0.7, + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_parameters": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话。xxx", + "context_template": "你是一个专业的智能信息检索助手xxx", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + }, + "no_match_prefix": "\n\nNO_MATCH" + } +}' +``` + +**响应**: + +```json +{ + "data": { + "id": "411d6b70-9a85-4d03-bb74-aab0fd8bd12f", + "title": "", + "description": "", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "max_rounds": 5, + "enable_rewrite": true, + "fallback_strategy": "FIXED_RESPONSE", + "fallback_response": "对不起，我无法回答这个问题", + "embedding_top_k": 10, + "keyword_threshold": 0.5, + "vector_threshold": 0.7, + "rerank_model_id": "排序模型ID", + "rerank_top_k": 3, + "rerank_threshold": 0.7, + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_parameters": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话。xxx", + "context_template": "你是一个专业的智能信息检索助手xxx", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + }, + "created_at": "2025-08-12T12:26:19.611616669+08:00", + "updated_at": "2025-08-12T12:26:19.611616919+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### GET `/sessions/:id` - 获取会话详情 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/sessions/ceb9babb-1e30-41d7-817d-fd584954304b' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "data": { + "id": "ceb9babb-1e30-41d7-817d-fd584954304b", + "title": "模型优化策略", + "description": "", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "max_rounds": 5, + "enable_rewrite": true, + "fallback_strategy": "fixed", + "fallback_response": "抱歉，我无法回答这个问题。", + "embedding_top_k": 10, + "keyword_threshold": 0.3, + "vector_threshold": 0.5, + "rerank_model_id": "", + "rerank_top_k": 5, + "rerank_threshold": 0.7, + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_parameters": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话", + "context_template": "你是一个专业的智能信息检索助手", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + }, + "created_at": "2025-08-12T10:24:38.308596+08:00", + "updated_at": "2025-08-12T10:25:41.317761+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### GET `/sessions?page=&page_size=` - 获取租户的会话列表 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/sessions?page=1&page_size=1' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "411d6b70-9a85-4d03-bb74-aab0fd8bd12f", + "title": "", + "description": "", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "max_rounds": 5, + "enable_rewrite": true, + "fallback_strategy": "FIXED_RESPONSE", + "fallback_response": "对不起，我无法回答这个问题", + "embedding_top_k": 10, + "keyword_threshold": 0.5, + "vector_threshold": 0.7, + "rerank_model_id": "排序模型ID", + "rerank_top_k": 3, + "rerank_threshold": 0.7, + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_parameters": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话。xxx", + "context_template": "你是一个专业的智能信息检索助手xxx", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + }, + "created_at": "2025-08-12T12:26:19.611616+08:00", + "updated_at": "2025-08-12T12:26:19.611616+08:00", + "deleted_at": null + } + ], + "page": 1, + "page_size": 1, + "success": true, + "total": 2 +} +``` + +#### PUT `/sessions/:id` - 更新会话 + +**请求**: + +```curl +curl --location --request PUT 'http://localhost:8080/api/v1/sessions/411d6b70-9a85-4d03-bb74-aab0fd8bd12f' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' \ +--data '{ + "title": "weknora", + "description": "weknora description", + "knowledge_base_id": "kb-00000001", + "max_rounds": 5, + "enable_rewrite": true, + "fallback_strategy": "FIXED_RESPONSE", + "fallback_response": "对不起，我无法回答这个问题", + "embedding_top_k": 10, + "keyword_threshold": 0.5, + "vector_threshold": 0.7, + "rerank_model_id": "排序模型ID", + "rerank_top_k": 3, + "rerank_threshold": 0.7, + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_parameters": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话。xxx", + "context_template": "你是一个专业的智能信息检索助手xxx", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + } +}' +``` + +**响应**: + +```json +{ + "data": { + "id": "411d6b70-9a85-4d03-bb74-aab0fd8bd12f", + "title": "weknora", + "description": "weknora description", + "tenant_id": 1, + "knowledge_base_id": "kb-00000001", + "max_rounds": 5, + "enable_rewrite": true, + "fallback_strategy": "FIXED_RESPONSE", + "fallback_response": "对不起，我无法回答这个问题", + "embedding_top_k": 10, + "keyword_threshold": 0.5, + "vector_threshold": 0.7, + "rerank_model_id": "排序模型ID", + "rerank_top_k": 3, + "rerank_threshold": 0.7, + "summary_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_parameters": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话。xxx", + "context_template": "你是一个专业的智能信息检索助手xxx", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + }, + "created_at": "0001-01-01T00:00:00Z", + "updated_at": "2025-08-12T14:20:56.738424351+08:00", + "deleted_at": null + }, + "success": true +} +``` + +#### DELETE `/sessions/:id` - 删除会话 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/sessions/411d6b70-9a85-4d03-bb74-aab0fd8bd12f' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "message": "Session deleted successfully", + "success": true +} +``` + +#### POST `/sessions/:session_id/generate_title` - 生成会话标题 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/sessions/ceb9babb-1e30-41d7-817d-fd584954304b/generate_title' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' \ +--data '{ + "messages": [ + { + "role": "user", + "content": "你好，我想了解关于人工智能的知识" + }, + { + "role": "assistant", + "content": "人工智能是计算机科学的一个分支..." + } + ] +}' +``` + +**响应**: + +```json +{ + "data": "模型优化策略", + "success": true +} +``` + +#### GET `/sessions/continue-stream/:session_id` - 继续未完成的会话 + +**查询参数**: +- `message_id`: 从 `/messages/:session_id/load` 接口中获取的 `is_completed` 为 `false` 的消息 ID + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/sessions/continue-stream/ceb9babb-1e30-41d7-817d-fd584954304b?message_id=b8b90eeb-7dd5-4cf9-81c6-5ebcbd759451' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应格式**: +服务器端事件流（Server-Sent Events），与 `/knowledge-chat/:session_id` 返回结果一致 + + + +### 聊天功能API + +| 方法 | 路径 | 描述 | +| ---- | ----------------------------- | ------------------------ | +| POST | `/knowledge-chat/:session_id` | 基于知识库的问答 | +| POST | `/knowledge-search` | 基于知识库的搜索知识 | + +#### POST `/knowledge-chat/:session_id` - 基于知识库的问答 + +**请求**: + +```curl +curl --location 'http://localhost:8080/api/v1/knowledge-chat/ceb9babb-1e30-41d7-817d-fd584954304b' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' \ +--data '{ + "query": "彗尾的形状" +}' +``` + +**响应格式**: +服务器端事件流（Server-Sent Events，Content-Type: text/event-stream） + +**响应**: + +``` +event: message +data: {"id":"3475c004-0ada-4306-9d30-d7f5efce50d2","response_type":"references","content":"","done":false,"knowledge_references":[{"id":"c8347bef-127f-4a22-b962-edf5a75386ec","content":"彗星xxx。","knowledge_id":"a6790b93-4700-4676-bd48-0d4804e1456b","chunk_index":0,"knowledge_title":"彗星.txt","start_at":0,"end_at":2760,"seq":0,"score":4.038836479187012,"match_type":3,"sub_chunk_id":["688821f0-40bf-428e-8cb6-541531ebeb76","c1e9903e-2b4d-4281-be15-0149288d45c2","7d955251-3f79-4fd5-a6aa-02f81e044091"],"metadata":{},"chunk_type":"text","parent_chunk_id":"","image_info":"","knowledge_filename":"彗星.txt","knowledge_source":""},{"id":"fa3aadee-cadb-4a84-9941-c839edc3e626","content":"# 文档名称\n彗星.txt\n\n# 摘要\n彗星是由冰和尘埃构成的太阳系小天体，接近太阳时会释放气体形成彗发和彗尾。其轨道周期差异大，来源包括柯伊伯带和奥尔特云。彗星与小行星的区别逐渐模糊，部分彗星已失去挥发物质，类似小行星。目前已知彗星数量众多，且存在系外彗星。彗星在古代被视为凶兆，现代研究揭示其复杂结构与起源。","knowledge_id":"a6790b93-4700-4676-bd48-0d4804e1456b","chunk_index":6,"knowledge_title":"彗星.txt","start_at":0,"end_at":0,"seq":6,"score":0.6131043121858466,"match_type":3,"sub_chunk_id":null,"metadata":{},"chunk_type":"summary","parent_chunk_id":"c8347bef-127f-4a22-b962-edf5a75386ec","image_info":"","knowledge_filename":"彗星.txt","knowledge_source":""}]} + +event: message +data: {"id":"3475c004-0ada-4306-9d30-d7f5efce50d2","response_type":"answer","content":"表现为","done":false,"knowledge_references":null} + +event: message +data: {"id":"3475c004-0ada-4306-9d30-d7f5efce50d2","response_type":"answer","content":"结构","done":false,"knowledge_references":null} + +event: message +data: {"id":"3475c004-0ada-4306-9d30-d7f5efce50d2","response_type":"answer","content":"。","done":false,"knowledge_references":null} + +event: message +data: {"id":"3475c004-0ada-4306-9d30-d7f5efce50d2","response_type":"answer","content":"","done":true,"knowledge_references":null} +``` + + + +### 消息管理API + +| 方法 | 路径 | 描述 | +| ------ | ---------------------------- | ------------------------ | +| GET | `/messages/:session_id/load` | 获取最近的会话消息列表 | +| DELETE | `/messages/:session_id/:id` | 删除消息 | + +#### GET `/messages/:session_id/load?before_time=2025-04-18T11:57:31.310671+08:00&limit=20` - 获取最近的会话消息列表 + +**查询参数**: + +- `before_time`: 上一次拉取的最早一条消息的 created_at 字段，为空拉取最近的消息 +- `limit`: 每页条数(默认 20) + +**请求**: + +```curl +curl --location --request GET 'http://localhost:8080/api/v1/messages/ceb9babb-1e30-41d7-817d-fd584954304b/load?limit=3&before_time=2030-08-12T14%3A35%3A42.123456789Z' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' \ +--data '{ + "query": "彗尾的形状" +}' +``` + +**响应**: + +```json +{ + "data": [ + { + "id": "b8b90eeb-7dd5-4cf9-81c6-5ebcbd759451", + "session_id": "ceb9babb-1e30-41d7-817d-fd584954304b", + "request_id": "hCA8SDjxcAvv", + "content": "\n好的", + "role": "assistant", + "knowledge_references": [ + { + "id": "c8347bef-127f-4a22-b962-edf5a75386ec", + "content": "彗星xxx", + "knowledge_id": "a6790b93-4700-4676-bd48-0d4804e1456b", + "chunk_index": 0, + "knowledge_title": "彗星.txt", + "start_at": 0, + "end_at": 2760, + "seq": 0, + "score": 4.038836479187012, + "match_type": 4, + "sub_chunk_id": [ + "688821f0-40bf-428e-8cb6-541531ebeb76", + "c1e9903e-2b4d-4281-be15-0149288d45c2", + "7d955251-3f79-4fd5-a6aa-02f81e044091" + ], + "metadata": {}, + "chunk_type": "text", + "parent_chunk_id": "", + "image_info": "", + "knowledge_filename": "彗星.txt", + "knowledge_source": "" + }, + { + "id": "fa3aadee-cadb-4a84-9941-c839edc3e626", + "content": "# 文档名称\n彗星.txt\n\n# 摘要\n彗星是由冰和尘埃构成的太阳系小天体，接近太阳时会释放气体形成彗发和彗尾。其轨道周期差异大，来源包括柯伊伯带和奥尔特云。彗星与小行星的区别逐渐模糊，部分彗星已失去挥发物质，类似小行星。目前已知彗星数量众多，且存在系外彗星。彗星在古代被视为凶兆，现代研究揭示其复杂结构与起源。", + "knowledge_id": "a6790b93-4700-4676-bd48-0d4804e1456b", + "chunk_index": 6, + "knowledge_title": "彗星.txt", + "start_at": 0, + "end_at": 0, + "seq": 6, + "score": 0.6131043121858466, + "match_type": 0, + "sub_chunk_id": null, + "metadata": {}, + "chunk_type": "summary", + "parent_chunk_id": "c8347bef-127f-4a22-b962-edf5a75386ec", + "image_info": "", + "knowledge_filename": "彗星.txt", + "knowledge_source": "" + } + ], + "is_completed": true, + "created_at": "2025-08-12T10:24:38.370548+08:00", + "updated_at": "2025-08-12T10:25:40.416382+08:00", + "deleted_at": null + }, + { + "id": "7fa136ae-a045-424e-baac-52113d92ae94", + "session_id": "ceb9babb-1e30-41d7-817d-fd584954304b", + "request_id": "3475c004-0ada-4306-9d30-d7f5efce50d2", + "content": "彗尾的形状", + "role": "user", + "knowledge_references": [], + "is_completed": true, + "created_at": "2025-08-12T14:30:39.732246+08:00", + "updated_at": "2025-08-12T14:30:39.733277+08:00", + "deleted_at": null + }, + { + "id": "9bcafbcf-a758-40af-a9a3-c4d8e0f49439", + "session_id": "ceb9babb-1e30-41d7-817d-fd584954304b", + "request_id": "3475c004-0ada-4306-9d30-d7f5efce50d2", + "content": "\n好的", + "role": "assistant", + "knowledge_references": [ + { + "id": "c8347bef-127f-4a22-b962-edf5a75386ec", + "content": "彗星xxx", + "knowledge_id": "a6790b93-4700-4676-bd48-0d4804e1456b", + "chunk_index": 0, + "knowledge_title": "彗星.txt", + "start_at": 0, + "end_at": 2760, + "seq": 0, + "score": 4.038836479187012, + "match_type": 3, + "sub_chunk_id": [ + "688821f0-40bf-428e-8cb6-541531ebeb76", + "c1e9903e-2b4d-4281-be15-0149288d45c2", + "7d955251-3f79-4fd5-a6aa-02f81e044091" + ], + "metadata": {}, + "chunk_type": "text", + "parent_chunk_id": "", + "image_info": "", + "knowledge_filename": "彗星.txt", + "knowledge_source": "" + }, + { + "id": "fa3aadee-cadb-4a84-9941-c839edc3e626", + "content": "# 文档名称\n彗星.txt\n\n# 摘要\n彗星是由冰和尘埃构成的太阳系小天体，接近太阳时会释放气体形成彗发和彗尾。其轨道周期差异大，来源包括柯伊伯带和奥尔特云。彗星与小行星的区别逐渐模糊，部分彗星已失去挥发物质，类似小行星。目前已知彗星数量众多，且存在系外彗星。彗星在古代被视为凶兆，现代研究揭示其复杂结构与起源。", + "knowledge_id": "a6790b93-4700-4676-bd48-0d4804e1456b", + "chunk_index": 6, + "knowledge_title": "彗星.txt", + "start_at": 0, + "end_at": 0, + "seq": 6, + "score": 0.6131043121858466, + "match_type": 3, + "sub_chunk_id": null, + "metadata": {}, + "chunk_type": "summary", + "parent_chunk_id": "c8347bef-127f-4a22-b962-edf5a75386ec", + "image_info": "", + "knowledge_filename": "彗星.txt", + "knowledge_source": "" + } + ], + "is_completed": true, + "created_at": "2025-08-12T14:30:39.735108+08:00", + "updated_at": "2025-08-12T14:31:17.829926+08:00", + "deleted_at": null + } + ], + "success": true +} +``` + +#### DELETE `/messages/:session_id/:id` - 删除消息 + +**请求**: + +```curl +curl --location --request DELETE 'http://localhost:8080/api/v1/messages/ceb9babb-1e30-41d7-817d-fd584954304b/9bcafbcf-a758-40af-a9a3-c4d8e0f49439' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "message": "Message deleted successfully", + "success": true +} +``` + + + +### 评估功能API + +| 方法 | 路径 | 描述 | +| ---- | ------------- | --------------------- | +| GET | `/evaluation` | 获取评估任务 | +| POST | `/evaluation` | 创建评估任务 | + +#### GET `/evaluation` - 获取评估任务 + +**请求参数**: +- `task_id`: 从 `POST /evaluation` 接口中获取到的任务 ID +- `X-API-Key`: 用户 API Key + +**请求**: + +```bash +curl --location 'http://localhost:8080/api/v1/evaluation?task_id=c34563ad-b09f-4858-b72e-e92beb80becb' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' +``` + +**响应**: + +```json +{ + "data": { + "task": { + "id": "c34563ad-b09f-4858-b72e-e92beb80becb", + "tenant_id": 1, + "dataset_id": "default", + "start_time": "2025-08-12T14:54:26.221804768+08:00", + "status": 2, + "total": 1, + "finished": 1 + }, + "params": { + "session_id": "", + "knowledge_base_id": "2ef57434-8c8d-4442-b967-2f7fc578a2fc", + "vector_threshold": 0.5, + "keyword_threshold": 0.3, + "embedding_top_k": 10, + "vector_database": "", + "rerank_model_id": "b30171a1-787b-426e-a293-735cd5ac16c0", + "rerank_top_k": 5, + "rerank_threshold": 0.7, + "chat_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_config": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话。", + "context_template": "你是一个专业的智能信息检索助手", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + }, + "fallback_strategy": "", + "fallback_response": "抱歉，我无法回答这个问题。" + }, + "metric": { + "retrieval_metrics": { + "precision": 0, + "recall": 0, + "ndcg3": 0, + "ndcg10": 0, + "mrr": 0, + "map": 0 + }, + "generation_metrics": { + "bleu1": 0.037656734016532384, + "bleu2": 0.04067392145167686, + "bleu4": 0.048963321289052536, + "rouge1": 0, + "rouge2": 0, + "rougel": 0 + } + } + }, + "success": true +} +``` + +#### POST `/evaluation` - 创建评估任务 + +**请求参数**: +- `dataset_id`: 评估使用的数据集，暂时只支持官方测试数据集 `default` +- `knowledge_base_id`: 评估使用的知识库 +- `chat_id`: 评估使用的对话模型 +- `rerank_id`: 评估使用的重排序模型 + +**请求**: + +```bash +curl --location 'http://localhost:8080/api/v1/evaluation' \ +--header 'X-API-Key: sk-vQHV2NZI_LK5W7wHQvH3yGYExX8YnhaHwZipUYbiZKCYJbBQ' \ +--header 'Content-Type: application/json' \ +--data '{ + "dataset_id": "default", + "knowledge_base_id": "kb-00000001", + "chat_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "rerank_id": "b30171a1-787b-426e-a293-735cd5ac16c0" +}' +``` + +**响应**: + +```json +{ + "data": { + "task": { + "id": "c34563ad-b09f-4858-b72e-e92beb80becb", + "tenant_id": 1, + "dataset_id": "default", + "start_time": "2025-08-12T14:54:26.221804768+08:00", + "status": 1 + }, + "params": { + "session_id": "", + "knowledge_base_id": "2ef57434-8c8d-4442-b967-2f7fc578a2fc", + "vector_threshold": 0.5, + "keyword_threshold": 0.3, + "embedding_top_k": 10, + "vector_database": "", + "rerank_model_id": "b30171a1-787b-426e-a293-735cd5ac16c0", + "rerank_top_k": 5, + "rerank_threshold": 0.7, + "chat_model_id": "8aea788c-bb30-4898-809e-e40c14ffb48c", + "summary_config": { + "max_tokens": 0, + "repeat_penalty": 1, + "top_k": 0, + "top_p": 0, + "frequency_penalty": 0, + "presence_penalty": 0, + "prompt": "这是用户和助手之间的对话。", + "context_template": "你是一个专业的智能信息检索助手，xxx", + "no_match_prefix": "\n\nNO_MATCH", + "temperature": 0.3, + "seed": 0, + "max_completion_tokens": 2048 + }, + "fallback_strategy": "", + "fallback_response": "抱歉，我无法回答这个问题。" + } + }, + "success": true +} +``` + + \ No newline at end of file diff --git a/docs/QA.md b/docs/QA.md new file mode 100644 index 0000000..fd24773 --- /dev/null +++ b/docs/QA.md @@ -0,0 +1,100 @@ +# 常见问题 + +## 1. 如何查看日志？ +```bash +docker compose logs -f app docreader postgres +``` + +## 2. 如何启动和停止服务？ +```bash +# 启动服务 +./scripts/start_all.sh + +# 停止服务 +./scripts/start_all.sh --stop + +# 清空数据库 +./scripts/start_all.sh --stop && make clean-db +``` + +## 3. 服务启动后无法正常上传文档？ + +通常是Embedding模型和对话模型没有正确被设置导致。按照以下步骤进行排查 + +1. 查看`.env`配置中的模型信息是否配置完整，其中如果使用ollama访问本地模型，需要确保本地ollama服务正常运行，同时在`.env`中的如下环境变量需要正确设置: +```bash +# LLM Model +INIT_LLM_MODEL_NAME=your_llm_model +# Embedding Model +INIT_EMBEDDING_MODEL_NAME=your_embedding_model +# Embedding模型向量维度 +INIT_EMBEDDING_MODEL_DIMENSION=your_embedding_model_dimension +# Embedding模型的ID，通常是一个字符串 +INIT_EMBEDDING_MODEL_ID=your_embedding_model_id +``` + +如果是通过remote api访问模型，则需要额外提供对应的`BASE_URL`和`API_KEY`: +```bash +# LLM模型的访问地址 +INIT_LLM_MODEL_BASE_URL=your_llm_model_base_url +# LLM模型的API密钥，如果需要身份验证，可以设置 +INIT_LLM_MODEL_API_KEY=your_llm_model_api_key +# Embedding模型的访问地址 +INIT_EMBEDDING_MODEL_BASE_URL=your_embedding_model_base_url +# Embedding模型的API密钥，如果需要身份验证，可以设置 +INIT_EMBEDDING_MODEL_API_KEY=your_embedding_model_api_key +``` + +当需要重排序功能时，需要额外配置Rerank模型，具体配置如下： +```bash +# 使用的Rerank模型名称 +INIT_RERANK_MODEL_NAME=your_rerank_model_name +# Rerank模型的访问地址 +INIT_RERANK_MODEL_BASE_URL=your_rerank_model_base_url +# Rerank模型的API密钥，如果需要身份验证，可以设置 +INIT_RERANK_MODEL_API_KEY=your_rerank_model_api_key +``` + +2. 查看主服务日志，是否有`ERROR`日志输出 + +## 4. 如何开启多模态功能？ +1. 确保 `.env` 如下配置被正确设置: +```bash +# VLM_MODEL_NAME 使用的多模态模型名称 +VLM_MODEL_NAME=your_vlm_model_name + +# VLM_MODEL_BASE_URL 使用的多模态模型访问地址 +VLM_MODEL_BASE_URL=your_vlm_model_base_url + +# VLM_MODEL_API_KEY 使用的多模态模型API密钥 +VLM_MODEL_API_KEY=your_vlm_model_api_key +``` +注：多模态大模型当前仅支持remote api访问，固需要提供`VLM_MODEL_BASE_URL`和`VLM_MODEL_API_KEY` + +2. 解析后的文件需要上传到COS中，确保 `.env` 中 `COS` 信息正确设置： +```bash +# 腾讯云COS的访问密钥ID +COS_SECRET_ID=your_cos_secret_id + +# 腾讯云COS的密钥 +COS_SECRET_KEY=your_cos_secret_key + +# 腾讯云COS的区域，例如 ap-guangzhou +COS_REGION=your_cos_region + +# 腾讯云COS的桶名称 +COS_BUCKET_NAME=your_cos_bucket_name + +# 腾讯云COS的应用ID +COS_APP_ID=your_cos_app_id + +# 腾讯云COS的路径前缀，用于存储文件 +COS_PATH_PREFIX=your_cos_path_prefix +``` +重要：务必将COS中文件的权限设置为**公有读**，否则文档解析模块无法正常解析文件 + +3. 查看文档解析模块日志，查看OCR和Caption是否正确解析和打印 + + +## P.S. +如果以上方式未解决问题，请在issue中描述您的问题，并提供必要的日志信息辅助我们进行问题排查 \ No newline at end of file diff --git a/docs/WeKnora.md b/docs/WeKnora.md new file mode 100644 index 0000000..d5d2caf --- /dev/null +++ b/docs/WeKnora.md @@ -0,0 +1,319 @@ +## 介绍 +WeKora 是一个可立即在生产环境投入的企业级RAG框架，实现智能文档理解和检索功能。该系统采用模块化设计，将文档理解、向量存储、推理文件等功能分离。 + +![arc](./images/arc.png) + +--- + +## PipeLine +WeKnora 处理文档需要多个步骤：插入-》知识提取-》索引-》检索-》生成，整个流程支持多种检索方法， + +![](./images/pipeline2.jpeg) + + +以用户上传的一张住宿流水单pdf文件为例，详细介绍下其数据流： + +### 1. 接收请求与初始化 ++ **请求识别**: 系统收到一个请求，并为其分配了唯一的 `request_id=Lkq0OGLYu2fV`，用于追踪整个处理流程。 ++ **租户与会话验证**: + - 系统首先验证了租户信息（ID: 1, Name: Default Tenant）。 + - 接着开始处理一个知识库问答（Knowledge QA）请求，该请求属于会话 `1f241340-ae75-40a5-8731-9a3a82e34fdd`。 ++ **用户问题**: 用户的原始问题是：“**入住的房型是什么**”。 ++ **消息创建**: 系统为用户的提问和即将生成的回答分别创建了消息记录，ID 分别为 `703ddf09-...` 和 `6f057649-...`。 + +### 2. 知识库问答流程启动 +系统正式调用知识库问答服务，并定义了将要按顺序执行的完整处理管道（Pipeline），包含以下9个事件： +`[rewrite_query, preprocess_query, chunk_search, chunk_rerank, chunk_merge, filter_top_k, into_chat_message, chat_completion_stream, stream_filter]` + +--- + +### 3. 事件执行详情 +#### 事件 1: `rewrite_query` - 问题改写 ++ **目的**: 为了让检索更精确，系统需要结合上下文来理解用户的真实意图。 ++ **操作**: + 1. 系统检索了当前会话最近的20条历史消息（实际检索到8条）作为上下文。 + 2. 调用了一个名为 `deepseek-r1:7b` 的本地大语言模型。 + 3. 模型根据聊天历史分析出提问者是“Liwx”，并将原问题“入住的房型是什么”改写得更具体。 ++ **结果**: 问题被成功改写为：“**Liwx本次入住的房型是什么**”。 + +#### 事件 2: `preprocess_query` - 问题预处理 ++ **目的**: 将改写后的问题进行分词，转换为适合搜索引擎处理的关键词序列。 ++ **操作**: 对改写后的问题进行了分词处理。 ++ **结果**: 生成了一串关键词：“`需要 改写 用户 问题 入住 房型 根据 提供 信息 入住 人 Liwx 选择 房型 双床 房 因此 改写 后 完整 问题 为 Liwx 本次 入住 房型`”。 + +#### 事件 3: `chunk_search` - 知识区块检索 +这是最核心的**检索（Retrieval）**步骤，系统执行了两次混合搜索（Hybrid Search）。 + ++ **第一次搜索 (使用改写后的完整问句)**: + - **向量检索**: + 1. 加载嵌入模型 `bge-m3:latest` 将问句转换为一个1024维的向量。 + 2. 在PostgreSQL数据库中进行向量相似度搜索，找到了2个相关的知识区块（chunk），ID 分别为 `e3bf6599-...` 和 `3989c6ce-...`。 + - **关键词检索**: + 1. 同时，系统也进行了关键词搜索。 + 2. 同样找到了上述2个知识区块。 + - **结果合并**: 两种方法找到的4个结果（实际是2个重复的）被去重，最终得到2个唯一的知识区块。 ++ **第二次搜索 (使用预处理后的关键词序列)**: + - 系统使用分词后的关键词重复了上述的**向量检索**和**关键词检索**过程。 + - 最终也得到了相同的2个知识区块。 ++ **最终结果**: 经过两次搜索和结果合并，系统锁定了2个最相关的知识区块，并将它们的内容提取出来，准备用于生成答案。 + +#### 事件 4: `chunk_rerank` - 结果重排序 ++ **目的**: 使用一个更强大的模型对初步检索出的结果进行更精细的排序，以提高最终答案的质量。 ++ **操作**: 日志显示 `Rerank model ID is empty, skipping reranking`。这意味着系统配置了重排序步骤，但没有指定具体的重排序模型，因此**跳过了此步骤**。 + +#### 事件 5: `chunk_merge` - 区块合并 ++ **目的**: 将内容上相邻或相关的知识区块进行合并，形成更完整的上下文。 ++ **操作**: 系统分析了检索到的2个区块，并尝试进行合并。根据日志，最终处理后仍然是2个独立的区块，但已按相关性分数排好序。 + +#### 事件 6: `filter_top_k` - Top-K 过滤 ++ **目的**: 仅保留最相关的K个结果，防止过多无关信息干扰语言模型。 ++ **操作**: 系统配置保留前5个（Top-K = 5）最相关的区块。由于当前只有2个区块，它们全部通过了此过滤器。 + +#### 事件 7 & 8: `into_chat_message` & `chat_completion_stream` - 生成回答 +这是**生成（Generation）**步骤。 + ++ **目的**: 基于检索到的信息，生成自然流畅的回答。 ++ **操作**: + 1. 系统将检索到的2个知识区块的内容、用户的原始问题以及聊天历史整合在一起，形成一个完整的提示（Prompt）。 + 2. 再次调用 `deepseek-r1:7b` 大语言模型，并以**流式（Stream）**的方式请求生成答案。流式输出可以实现打字机效果，提升用户体验。 + +#### 事件 9: `stream_filter` - 流式输出过滤 ++ **目的**: 对模型生成的实时文本流进行后处理，过滤掉不需要的特殊标记或内容。 ++ **操作**: + - 系统设置了一个过滤器，用于移除模型在思考过程中可能产生的内部标记，如 `` 和 ``。 + - 日志显示，模型输出的第一个词块是 ` 根据`，过滤器成功拦截并移除了 `` 标记，只将“根据”及之后的内容传递下去。 + +### 4. 完成与响应 ++ **发送引用**: 在生成答案的同时，系统将作为依据的2个知识区块作为“参考内容”发送给前端，以便用户查证来源。 ++ **更新消息**: 当模型生成完所有内容后，系统将完整的回答更新到之前创建的消息记录（ID: `6f057649-...`）中。 ++ **请求结束**: 服务器返回 `200` 成功状态码，标志着本次从提问到回答的完整流程结束。 + +### 总结 +这个日志完整地记录了一次典型的RAG流程：系统通过**问题改写**和**预处理**来精确理解用户意图，接着利用**向量与关键词混合检索**从知识库中找到相关信息，虽然跳过了**重排序**，但依然执行了**合并**与**过滤**，最后将检索到的知识作为上下文，交由大语言模型**生成**流畅、准确的回答，并通过**流式过滤**保证了输出的纯净性。 + +## 文档解析切分 +代码实现了一个独立的、通过gRPC通信的微服务，专门负责文档内容的深度解析、分块和多模态信息提取。它正是“异步处理”阶段的核心执行者。 + +### **整体架构** +这是一个基于Python的gRPC服务，其核心职责是接收文件（或URL），并将其解析成结构化的、可供后续处理（如向量化）的文本块（Chunks）。 + ++ `server.py`: 服务的入口和网络层。它负责启动一个多进程、多线程的gRPC服务器，接收来自Go后端的请求，并将解析结果返回。 ++ `parser.py`: 设计模式中的**外观（Facade）模式**。它提供了一个统一的`Parser`类，屏蔽了内部多种具体解析器（如PDF、DOCX、Markdown等）的复杂性。外部调用者（`server.py`）只需与这个`Parser`类交互。 ++ `base_parser.py`: 解析器的基类，定义了所有具体解析器共享的核心逻辑和抽象方法。这是整个解析流程的“大脑”，包含了最复杂的文本分块、图片处理、OCR和图像描述生成等功能。 + +--- + +### **详细工作流程** +当Go后端启动异步任务时，它会携带文件内容和配置信息，向这个Python服务发起一次gRPC调用。以下是完整的处理流程： + +#### **第一步：请求接收与分发 (**`server.py`** & **`parser.py`**) +1. **gRPC服务入口 (**`server.py: serve`**)**: + - 服务通过`serve()`函数启动。它会根据环境变量（`GRPC_WORKER_PROCESSES`, `GRPC_MAX_WORKERS`）启动一个**多进程、多线程**的服务器，以充分利用CPU资源，提高并发处理能力。 + - 每个工作进程都监听在指定的端口（如50051），准备接收请求。 +2. **请求处理 (**`server.py: ReadFromFile`**)**: + - 当Go后端发起`ReadFromFile`请求时，其中一个工作进程会接收到该请求。 + - 该方法首先会解析请求中的参数，包括： + * `file_name`, `file_type`, `file_content`：文件的基本信息和二进制内容。 + * `read_config`: 一个包含所有解析配置的复杂对象，如`chunk_size`（分块大小）、`chunk_overlap`（重叠大小）、`enable_multimodal`（是否启用多模态处理）、`storage_config`（对象存储配置）、`vlm_config`（视觉语言模型配置）等。 + - 它将这些配置整合成一个`ChunkingConfig`数据对象。 + - 最关键的一步是调用 `self.parser.parse_file(...)`，将解析任务交给`Parser`外观类处理。 +3. **解析器选择 (**`parser.py: Parser.parse_file`**)**: + - `Parser`类接收到任务后，首先调用`get_parser(file_type)`方法。 + - 该方法会根据文件类型（例如 `'pdf'`）在一个字典 `self.parsers` 中查找对应的具体解析器类（例如 `PDFParser`）。 + - 找到后，它会**实例化**这个`PDFParser`类，并将`ChunkingConfig`等所有配置信息传递给构造函数。 + +#### **第二步：核心解析与分块 (**`base_parser.py`**)** +它触及了整个流程的核心：**如何保证信息的上下文完整性和原始顺序**。 + +根据 `base_parser.py` 代码，**最终切分出的 Chunk 中的文本、表格和图像是按照它们在原始文档中的出现顺序来保存的**。 + +这个顺序得以保证，主要归功于 `BaseParser` 中几个设计精巧的方法相互协作。我们来详细追踪一下这个流程。 + +整个顺序的保证可以分为三个阶段： + +1. **阶段一：统一的文本流创建 (**`pdf_parser.py`**)**: + - 在 `parse_into_text` 方法中，您的代码会**逐页**处理PDF。 + - 在每一页内部，它会按照一定的逻辑（先提取非表格文本，再附加表格，最后附加图像占位符）将所有内容**拼接成一个长字符串** (`page_content_parts`)。 + - **关键点**: 虽然在这个阶段，文本、表格和图像占位符的拼接顺序可能不是100%精确到字符级别，但它保证了**同一页的内容会在一起**，并且大致遵循了从上到下的阅读顺序。 + - 最后，所有页面的内容被 `"\n\n--- Page Break ---\n\n"` 连接起来，形成一个**包含了所有信息（文本、Markdown表格、图像占位符）的、单一的、有序的文本流 (**`final_text`**)**。 +2. **阶段二：原子化与保护 (**`_split_into_units`**)**: + - 这个单一的 `final_text` 被传递给 `_split_into_units` 方法。 + - 这个方法是**保证结构完整性的关键**。它使用正则表达式，将**整个Markdown表格**和**整个Markdown图像占位符**识别为**不可分割的原子单元 (atomic units)**。 + - 它会将这些原子单元（表格、图片）和它们之间的普通文本块，按照它们在 `final_text` 中出现的**原始顺序**，切分成一个列表 (`units`)。 + - **结果**: 我们现在有了一个列表，例如 `['一些文本', '![...](...)', '另一些文本', '|...|...|\n|---|---|\n...', '更多文本']`。这个列表中的元素顺序**完全等同于它们在原始文档中的顺序**。 +3. **阶段三：顺序分块 (**`chunk_text`**)**: + - `chunk_text` 方法接收到这个**有序的 **`units`** 列表**。 + - 它的工作机制非常简单直接：它会**按顺序**遍历这个列表中的每一个单元（`unit`）。 + - 它将这些单元**依次添加**到一个临时的 `current_chunk` 列表中，直到这个块的长度接近 `chunk_size` 的上限。 + - 当一个块满了之后，它就被保存下来，然后开始一个新的块（可能会带有上一个块的重叠部分）。 + - **关键点**: 因为 `chunk_text` **严格按照 **`units`** 列表的顺序进行处理**，所以它永远不会打乱表格、文本和图像之间的相对顺序。一个在文档中先出现的表格，也必然会出现在一个序号更靠前的 Chunk 中。 +4. **阶段四：图像信息附加 (**`process_chunks_images`**)**: + - 在文本块被切分好之后，`process_chunks_images` 方法会被调用。 + - 它会处理**每一个**已经生成好的 Chunk。 + - 在每个 Chunk 内部，它会找到图像占位符，然后进行AI处理。 + - 最后，它会将处理好的图像信息（包含永久URL、OCR文本、图像描述等）附加到**该 Chunk 自己**的 `.images` 属性中。 + - **关键点**: 这个过程**不会改变 Chunk 的顺序或其 **`.content`** 的内容**。它只是为已经存在的、顺序正确的 Chunk 附加额外的信息。 + +#### **第三步：多模态处理（如果启用） (**`base_parser.py`**)** +如果 `enable_multimodal` 为 `True`，在文本分块完成后，会进入最复杂的多模态处理阶段。 + +1. **并发任务启动 (**`BaseParser.process_chunks_images`**)**: + - 该方法使用`asyncio`（Python的异步I/O框架）来**并发处理所有文本块中的图片**，以极大地提升效率。 + - 它为每个`Chunk`创建一个异步任务`process_chunk_images_async`。 +2. **处理单个块中的图片 (**`BaseParser.process_chunk_images_async`**)**: + - **提取图片引用**: 首先，使用正则表达式 `extract_images_from_chunk` 从当前块的文本中找到所有的图片引用（例如，`![alt text](image.png)`）。 + - **图片持久化**: 对于找到的每个图片，并发地调用 `download_and_upload_image`。这个函数负责： + * 从其原始位置（可能是PDF内部、本地路径或远程URL）获取图片数据。 + * 将图片**上传到配置好的对象存储（COS/MinIO）**。这一步至关重要，它将临时的、不稳定的图片引用转换成一个持久化、可通过URL公开访问的地址。 + * 返回持久化的URL和图片对象（PIL Image）。 + - **并发AI处理**: 将所有成功上传的图片收集起来，调用`process_multiple_images`。 + * 该方法内部使用`asyncio.Semaphore`来限制并发数量（例如最多同时处理5张图片），防止瞬间消耗过多内存或触发模型API的速率限制。 + * 对于每张图片，它会调用`process_image_async`。 +3. **处理单张图片 (**`BaseParser.process_image_async`**)**: + - **OCR**: 调用`perform_ocr`，它会使用一个OCR引擎（如`PaddleOCR`）来识别图片中的所有文字。 + - **图像描述 (Caption)**: 调用`get_image_caption`，它会将图片数据（转为Base64）发送给配置的视觉语言模型（VLM），生成对图片内容的自然语言描述。 + - 该方法返回 `(ocr_text, caption, 持久化URL)`。 +4. **结果聚合**: + - 所有图片处理完成后，包含持久化URL、OCR文本和图像描述的结构化信息，会被附加到对应`Chunk`对象的 `.images` 字段上。 + +#### **第四步：返回结果 (**`server.py`**)** +1. **数据转换 (**`server.py: _convert_chunk_to_proto`**)**: + - 当`parser.parse_file`执行完毕后，它返回一个包含所有处理过的`Chunk`对象的列表（`ParseResult`）。 + - `ReadFromFile`方法接收到这个结果，并调用`_convert_chunk_to_proto`，将Python的`Chunk`对象（包括其内部的图片信息）转换成gRPC定义的Protobuf消息格式。 +2. **响应返回**: + - 最后，gRPC服务器将这个包含所有分块和多模态信息的`ReadResponse`消息发送回给调用方——Go后端服务。 + +至此，Go后端就拿到了结构化、信息丰富的文档数据，可以进行下一步的向量化和索引存储了。 + + +## 部署 +支持Docker 镜像本地部署，并通过API端口提供接口服务 + +## 性能和监控 +Weknora包含丰富的监控和测试组件： + ++ 分布式跟踪：集成Jaeger用于跟踪请求在服务架构中的完整执行路。本质上，Jaeger是一种帮助用户“看见”请求在分布式系统中完整生命周期的技术。 ++ 健康监控：监控服务处在健康状态 ++ 可扩展性：通过容器化部署，可通过多个服务满足大规模并发请求 + +## QA +### 问题1: 在检索过程的执行了两次混合搜索的目的是什么？以及第一次和第二次搜索有什么不同？ +这是一个非常好的观察。系统执行两次混合搜索是为了**最大化检索的准确性和召回率**，本质上是一种**查询扩展（Query Expansion）和多策略检索**的组合方法。 + +#### 目的 +通过两种不同形式的查询（原始改写句 vs. 分词后的关键词序列）去搜索，系统可以结合两种查询方式的优点： + ++ **语义检索的深度**: 使用完整的句子进行搜索，能更好地利用向量模型（如`bge-m3`）对句子整体含义的理解能力，找到语义上最接近的知识区块。 ++ **关键词检索的广度**: 使用分词后的关键词进行搜索，能确保即使知识区块的表述方式与原问题不同，但只要包含了核心关键词，就有机会被命中。这对于传统的关键词匹配算法（如BM25）尤其有效。 + +简单来说，就是**用两种不同的“问法”去问同一个问题**，然后将两边的结果汇总起来，确保最相关的知识不会被遗漏。 + +#### 两次搜索的不同点 +它们最核心的不同在于**输入的查询文本（Query Text）**： + +1. **第一次混合搜索** + - **输入**: 使用的是经过`rewrite_query`事件后生成的、**语法完整的自然语言问句**。 + - **日志证据**: + +```plain +INFO [2025-08-29 09:46:36.896] [request_id=Lkq0OGLYu2fV] knowledgebase.go:266[HybridSearch] | Hybrid search parameters, knowledge base ID: kb-00000001, query text: 需要改写的用户问题是：“入住的房型是什么”。根据提供的信息，入住人Liwx选择的房型是双床房。因此，改写后的完整问题为： “Liwx本次入住的房型是什么” +``` + +2. **第二次混合搜索** + - **输入**: 使用的是经过`preprocess_query`事件处理后生成的、**由空格隔开的关键词序列**。 + - **日志证据**: + +```plain +INFO [2025-08-29 09:46:37.257] [request_id=Lkq0OGLYu2fV] knowledgebase.go:266[HybridSearch] | Hybrid search parameters, knowledge base ID: kb-00000001, query text: 需要 改写 用户 问题 入住 房型 根据 提供 信息 入住 人 Liwx 选择 房型 双床 房 因此 改写 后 完整 问题 为 Liwx 本次 入住 房型 +``` + +最终，系统将这两次搜索的结果进行去重和合并（日志中显示每次都找到2个结果，去重后总共还是2个），从而得到一个更可靠的知识集合，用于后续的答案生成。 + + + +### 问题2：重排序模型分析 +Reranker（重排器）是目前RAG领域中非常先进的技术，它们在工作原理和适用场景上有着显著的区别。 + +简单来说，它们代表了从“**专门的判别模型**”到“**利用大语言模型（LLM）进行判别**”再到“**深度挖掘LLM内部信息进行判别**”的演进。 + +以下是它们的详细区别： + + + +#### 1. Normal Reranker (常规重排器 / 交叉编码器) +这是最经典也是最主流的重排方法。 + ++ **模型类型**: **序列分类模型 (Sequence Classification Model)**。本质上是一个**交叉编码器 (Cross-Encoder)**，通常基于BERT、RoBERTa等双向编码器架构。`BAAI/bge-reranker-base/large/v2-m3` 都属于这一类。 ++ **工作原理**: + 1. 它将**查询（Query）**和**待排序的文档（Passage）**拼接成一个单一的输入序列，例如：`[CLS] what is panda? [SEP] The giant panda is a bear species endemic to China. [SEP]`。 + 2. 这个拼接后的序列被完整地送入模型中。模型内部的自注意力机制（Self-Attention）可以同时分析查询和文档中的每一个词，并计算它们之间**细粒度的交互关系**。 + 3. 模型最终输出一个**单一的分数（Logit）**，这个分数直接代表了查询和文档的相关性。分数越高，相关性越强。 ++ **关键特性**: + - **优点**: 由于查询和文档在模型内部进行了充分的、深度的交互，其**准确度通常非常高**，是衡量Reranker性能的黄金标准。 + - **缺点**: **速度较慢**。因为它必须为**每一个“查询-文档”对**都独立执行一次完整的、代价高昂的计算。如果初步检索返回了100个文档，它就需要运行100次。 + + + +#### 2. LLM-based Reranker (基于LLM的重排器) +这种方法创造性地利用了通用大语言模型（LLM）的能力来进行重排。 + ++ **模型类型**: **因果语言模型 (Causal Language Model)**，即我们常说的GPT、Llama、Gemma这类用于生成文本的LLM。`BAAI/bge-reranker-v2-gemma` 就是一个典型的例子。 ++ **工作原理**: + 1. 它**不是直接输出一个分数**，而是将重排任务**转化为一个问答或文本生成任务**。 + 2. 它通过一个精心设计的**提示（Prompt）**来组织输入，例如：`"Given a query A and a passage B, determine whether the passage contains an answer to the query by providing a prediction of either 'Yes' or 'No'. A: {query} B: {passage}"`。 + 3. 它将这个完整的Prompt喂给LLM，然后**观察LLM在最后生成“Yes”这个词的概率**。 + 4. 这个**生成“Yes”的概率（或其Logit值）就被当作是相关性分数**。如果模型非常确信答案是“Yes”，说明它认为文档B包含了查询A的答案，即相关性高。 ++ **关键特性**: + - **优点**: 能够利用LLM强大的**语义理解、推理和世界知识**，对于需要深度理解和推理才能判断相关性的复杂查询，效果可能更好。 + - **缺点**: 计算开销可能非常大（取决于LLM的大小），并且性能**高度依赖于Prompt的设计**。 + + + +#### 3. LLM-based Layerwise Reranker (基于LLM分层信息的重排器) +这是第二种方法的“威力加强版”，是一种更前沿、更复杂的探究性技术。 + ++ **模型类型**: 同样是**因果语言模型 (Causal Language Model)**，例如`BAAI/bge-reranker-v2-minicpm-layerwise`。 ++ **工作原理**: + 1. 输入部分与第二种方法完全相同，也是使用“Yes/No”的Prompt。 + 2. 核心区别在于**分数的提取方式**。它不再仅仅依赖LLM**最后一层**的输出（即最终的预测结果）。 + 3. 它认为LLM在逐层处理信息的过程中，不同深度的网络层（Layer）可能捕获了不同层次的语义相关性信息。因此，它会从**模型的多个中间层**提取出关于“Yes”这个词的预测Logit。 + 4. 代码中的 `cutoff_layers=[28]` 参数就是告诉模型：“请把第28层的输出给我”。最终，你会得到一个或多个来自不同网络层的分数，这些分数可以被平均或以其他方式组合，形成一个更鲁棒的最终相关性判断。 ++ **关键特性**: + - **优点**: 理论上可以获得**更丰富、更全面的相关性信号**，可能达到比只看最后一层更高的精度，是目前探索性能极限的一种方法。 + - **缺点**: **复杂度最高**，需要对模型进行特定的修改才能提取中间层信息（代码中的`trust_remote_code=True`就是一个信号），计算开销也很大。 + +#### 总结对比 +| 特性 | 1. Normal Reranker (常规) | 2. LLM-based Reranker (基于LLM) | 3. LLM-based Layerwise Reranker (基于LLM分层) | +| :--- | :--- | :--- | :--- | +| **底层模型** | 交叉编码器 (如BERT) | 因果语言模型 (如Gemma) | 因果语言模型 (如MiniCPM) | +| **工作原理** | 计算Query和Passage的深度交互，直接输出相关分 | 将排序任务转为"Yes/No"预测，用"Yes"的概率作为分数 | 与2类似，但从LLM的多个中间层提取"Yes"的概率 | +| **输出** | 单一的相关性分数 | 单一的相关性分数（来自最后一层） | 多个相关性分数（来自不同层） | +| **优点** | **速度与精度的最佳平衡点**，成熟稳定 | 利用LLM的推理能力，处理复杂问题 | 理论上精度最高，信号更丰富 | +| **缺点** | 相比向量检索慢 | 计算开销大，依赖Prompt设计 | **复杂度最高**，计算开销最大 | +| **推荐场景** | **大多数生产环境的首选**，效果好，易于部署 | 对答案质量有极致要求，且计算资源充足的场景 | 学术研究或追求SOTA（State-of-the-art）性能的场景 | + + +#### 使用建议 +1. **开始阶段**: 强烈建议您**从 **`Normal Reranker`** 开始**，例如 `BAAI/bge-reranker-v2-m3`。它是目前综合表现最好的模型之一，能显著提升您的RAG系统性能，并且相对容易集成和部署。 +2. **进阶探索**: 如果您发现常规Reranker在处理某些非常微妙或需要复杂推理的查询时表现不佳，并且您拥有充足的GPU资源，可以尝试 `LLM-based Reranker`。 +3. **前沿研究**: `Layerwise Reranker` 更适合研究人员或希望在特定任务上压榨出最后一点性能的专家。 + + +### 问题3：粗过滤或细过滤后的知识（带重排）如何组装发送给大模型的？ +这一块主要是设计提示词，典型的指令细节，其核心任务是根据上下文回答用户问题。组装上下文时需要指定 +关键约束：必须严格按照所提供文档回答，禁止使用你自己的知识回答 +未知情况处理： 如果文档中没有足够的信息来回答问题，请告知“根据所掌握的资料，无法回答这个问题” +引用要求：在回答时，如果引用了某个文档内容，请在句子末尾加上文档编号 + + + + + + + + + + + + + diff --git a/docs/WeKnora.pdf b/docs/WeKnora.pdf new file mode 100644 index 0000000..f98a9f2 Binary files /dev/null and b/docs/WeKnora.pdf differ diff --git a/docs/images/answer.png b/docs/images/answer.png new file mode 100644 index 0000000..f601a8a Binary files /dev/null and b/docs/images/answer.png differ diff --git a/docs/images/arc.png b/docs/images/arc.png new file mode 100644 index 0000000..5f1cc5f Binary files /dev/null and b/docs/images/arc.png differ diff --git a/docs/images/config.png b/docs/images/config.png new file mode 100644 index 0000000..66a010b Binary files /dev/null and b/docs/images/config.png differ diff --git a/docs/images/graph1.png b/docs/images/graph1.png new file mode 100644 index 0000000..b878fc2 Binary files /dev/null and b/docs/images/graph1.png differ diff --git a/docs/images/graph2.png b/docs/images/graph2.png new file mode 100644 index 0000000..549812f Binary files /dev/null and b/docs/images/graph2.png differ diff --git a/docs/images/knowledges.png b/docs/images/knowledges.png new file mode 100644 index 0000000..6bab844 Binary files /dev/null and b/docs/images/knowledges.png differ diff --git a/docs/images/logo.png b/docs/images/logo.png new file mode 100644 index 0000000..158aba2 Binary files /dev/null and b/docs/images/logo.png differ diff --git a/docs/images/pipeline.jpg b/docs/images/pipeline.jpg new file mode 100644 index 0000000..abf9c55 Binary files /dev/null and b/docs/images/pipeline.jpg differ diff --git a/docs/images/pipeline.png b/docs/images/pipeline.png new file mode 100644 index 0000000..ae9c316 Binary files /dev/null and b/docs/images/pipeline.png differ diff --git a/docs/images/pipeline2.jpeg b/docs/images/pipeline2.jpeg new file mode 100644 index 0000000..ca89273 Binary files /dev/null and b/docs/images/pipeline2.jpeg differ diff --git a/docs/images/qa.png b/docs/images/qa.png new file mode 100644 index 0000000..ce06a87 Binary files /dev/null and b/docs/images/qa.png differ diff --git "a/docs/\344\275\277\347\224\250\345\205\266\344\273\226\345\220\221\351\207\217\346\225\260\346\215\256\345\272\223.md" "b/docs/\344\275\277\347\224\250\345\205\266\344\273\226\345\220\221\351\207\217\346\225\260\346\215\256\345\272\223.md" new file mode 100644 index 0000000..b013586 --- /dev/null +++ "b/docs/\344\275\277\347\224\250\345\205\266\344\273\226\345\220\221\351\207\217\346\225\260\346\215\256\345\272\223.md" @@ -0,0 +1,190 @@ +### 如何集成新的向量数据库 + +本文提供了向 WeKnora 项目添加新向量数据库支持的完整指南。通过实现标准化接口和遵循结构化流程，开发者可以高效地集成自定义向量数据库。 + +### 集成流程 + +#### 1. 实现基础检索引擎接口 + +首先需要实现 `interfaces` 包中的 `RetrieveEngine` 接口，定义检索引擎的核心能力： + +```go +type RetrieveEngine interface { + // 返回检索引擎的类型标识 + EngineType() types.RetrieverEngineType + + // 执行检索操作，返回匹配结果 + Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) + + // 返回该引擎支持的检索类型列表 + Support() []types.RetrieverType +} +``` + +#### 2. 实现存储层接口 + +实现 `RetrieveEngineRepository` 接口，扩展基础检索引擎能力，添加索引管理功能： + +```go +type RetrieveEngineRepository interface { + // 保存单个索引信息 + Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error + + // 批量保存多个索引信息 + BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error + + // 估算索引存储所需空间 + EstimateStorageSize(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) int64 + + // 通过分块ID列表删除索引 + DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error + + // 复制索引数据，避免重新计算嵌入向量 + CopyIndices( + ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, + ) error + + // 通过知识ID列表删除索引 + DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error + + // 继承RetrieveEngine接口 + RetrieveEngine +} +``` + +#### 3. 实现服务层接口 + +创建实现 `RetrieveEngineService` 接口的服务，负责处理索引创建和管理的业务逻辑： + +```go +type RetrieveEngineService interface { + // 创建单个索引 + Index(ctx context.Context, + embedder embedding.Embedder, + indexInfo *types.IndexInfo, + retrieverTypes []types.RetrieverType, + ) error + + // 批量创建索引 + BatchIndex(ctx context.Context, + embedder embedding.Embedder, + indexInfoList []*types.IndexInfo, + retrieverTypes []types.RetrieverType, + ) error + + // 估算索引存储空间 + EstimateStorageSize(ctx context.Context, + embedder embedding.Embedder, + indexInfoList []*types.IndexInfo, + retrieverTypes []types.RetrieverType, + ) int64 + + // 复制索引数据 + CopyIndices( + ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, + ) error + + // 删除索引 + DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error + DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error + + // 继承RetrieveEngine接口 + RetrieveEngine +} +``` + +#### 4. 添加环境变量配置 + +在环境配置中添加新数据库的必要连接参数： + +``` +# 在RETRIEVE_DRIVER中添加新数据库驱动名称（多个驱动用逗号分隔） +RETRIEVE_DRIVER=postgres,elasticsearch_v8,your_database + +# 新数据库的连接参数 +YOUR_DATABASE_ADDR=your_database_host:port +YOUR_DATABASE_USERNAME=username +YOUR_DATABASE_PASSWORD=password +# 其他必要的连接参数... +``` + +#### 5. 注册检索引擎 + +在 `internal/container/container.go` 文件的 `initRetrieveEngineRegistry` 函数中添加新数据库的初始化与注册逻辑： + +```go +func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.RetrieveEngineRegistry, error) { + registry := retriever.NewRetrieveEngineRegistry() + retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") + log := logger.GetLogger(context.Background()) + + // 已有的PostgreSQL和Elasticsearch初始化代码... + + // 添加新向量数据库的初始化代码 + if slices.Contains(retrieveDriver, "your_database") { + // 初始化数据库客户端 + client, err := your_database.NewClient(your_database.Config{ + Addresses: []string{os.Getenv("YOUR_DATABASE_ADDR")}, + Username: os.Getenv("YOUR_DATABASE_USERNAME"), + Password: os.Getenv("YOUR_DATABASE_PASSWORD"), + // 其他连接参数... + }) + + if err != nil { + log.Errorf("Create your_database client failed: %v", err) + } else { + // 创建检索引擎仓库 + yourDatabaseRepo := your_database.NewYourDatabaseRepository(client, cfg) + + // 注册检索引擎 + if err := registry.Register( + retriever.NewKVHybridRetrieveEngine( + yourDatabaseRepo, types.YourDatabaseRetrieverEngineType, + ), + ); err != nil { + log.Errorf("Register your_database retrieve engine failed: %v", err) + } else { + log.Infof("Register your_database retrieve engine success") + } + } + } + + return registry, nil +} +``` + +#### 6. 定义检索引擎类型常量 + +在 `internal/types/retriever.go` 文件中添加新的检索引擎类型常量： + +```go +// RetrieverEngineType 定义检索引擎类型 +const ( + ElasticsearchRetrieverEngineType RetrieverEngineType = "elasticsearch" + PostgresRetrieverEngineType RetrieverEngineType = "postgres" + YourDatabaseRetrieverEngineType RetrieverEngineType = "your_database" // 添加新数据库类型 +) +``` + +## 参考实现示例 + +建议参考现有的 PostgreSQL 和 Elasticsearch 实现作为开发模板。这些实现位于以下目录： + +- PostgreSQL: `internal/application/repository/retriever/postgres/` +- ElasticsearchV7: `internal/application/repository/retriever/elasticsearch/v7/` +- ElasticsearchV8: `internal/application/repository/retriever/elasticsearch/v8/` + +通过遵循以上步骤和参考现有实现，你可以成功集成新的向量数据库到 WeKnora 系统中，扩展其向量检索能力。 + + + diff --git a/frontend/.dockerignore b/frontend/.dockerignore new file mode 100644 index 0000000..9e31322 --- /dev/null +++ b/frontend/.dockerignore @@ -0,0 +1,8 @@ +node_modules +dist +.git +.gitignore +README.md +.vscode +*.log +.DS_Store \ No newline at end of file diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..8ee54e8 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,30 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +.DS_Store +dist +dist-ssr +coverage +*.local + +/cypress/videos/ +/cypress/screenshots/ + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +*.tsbuildinfo diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..f1fe3e7 --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,36 @@ +# 构建阶段 +FROM node:24-alpine as build-stage + +WORKDIR /app + +# 设置环境变量，忽略类型检查错误 +ENV NODE_OPTIONS="--max-old-space-size=4096" +ENV VITE_IS_DOCKER=true + +# 复制依赖文件 +COPY package*.json ./ + +# 安装依赖 +RUN corepack enable +RUN pnpm install + +# 复制项目文件 +COPY . . + +# 构建应用 +RUN pnpm run build + +# 生产阶段 +FROM nginx:stable-alpine as production-stage + +# 复制构建产物到nginx服务目录 +COPY --from=build-stage /app/dist /usr/share/nginx/html + +# 复制nginx配置文件 +COPY nginx.conf /etc/nginx/conf.d/default.conf + +# 暴露端口 +EXPOSE 80 + +# 启动nginx +CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/frontend/env.d.ts b/frontend/env.d.ts new file mode 100644 index 0000000..5b580fc --- /dev/null +++ b/frontend/env.d.ts @@ -0,0 +1,6 @@ +/// +// 配置这个文件是 解决错误：找不到模块“@/views/login/index.vue”或其相应的类型声明。ts(2307) +// 这段代码告诉 TypeScript，所有以 .vue 结尾的文件都是 Vue 组件，可以通过 import 语句进行导入。这样做通常可以解决无法识别模块的问题。 +declare module '*.vue' { + import { Component } from 'vue'; const component: Component; export default component; +} \ No newline at end of file diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..e2ab8ef --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,22 @@ + + + + WeKnora + + + + + + + + + + + + + +
+ + + + diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 0000000..e855553 --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,52 @@ +server { + listen 80; + server_name localhost; + client_max_body_size 50M; + + # 安全头配置 + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + # 错误日志配置 + error_log /var/log/nginx/error.log warn; + access_log /var/log/nginx/access.log; + + # 前端静态文件 + location / { + root /usr/share/nginx/html; + index index.html; + try_files $uri $uri/ /index.html; + } + + # API请求代理到后端服务 + location /api/ { + proxy_pass http://app:8080/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # 连接和重试配置 + proxy_connect_timeout 30s; # 连接超时时间 + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_next_upstream_tries 3; # 重试次数 + proxy_next_upstream_timeout 30s; # 重试超时时间 + + # SSE 相关配置 + proxy_http_version 1.1; # 使用 HTTP/1.1 + proxy_set_header Connection ""; # 禁用 Connection: close，保持连接打开 + chunked_transfer_encoding off; # 关闭分块传输编码 + proxy_buffering off; # 关闭缓冲 + proxy_cache off; # 关闭缓存 + proxy_read_timeout 3600s; # 增加读取超时时间 + proxy_send_timeout 3600s; # 增加发送超时时间 + } + + # 错误页面 + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json new file mode 100644 index 0000000..87b6c68 --- /dev/null +++ b/frontend/package-lock.json @@ -0,0 +1,3821 @@ +{ + "name": "knowledage-base", + "version": "0.1.3", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "knowledage-base", + "version": "0.1.3", + "dependencies": { + "@microsoft/fetch-event-source": "^2.0.1", + "@types/dompurify": "^3.0.5", + "axios": "^1.8.4", + "dompurify": "^3.2.6", + "marked": "^5.1.2", + "pagefind": "^1.1.1", + "pinia": "^3.0.1", + "tdesign-icons-vue-next": "^0.4.1", + "tdesign-vue-next": "^1.11.5", + "vue": "^3.5.13", + "vue-router": "^4.5.0", + "webpack": "^5.94.0" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.1", + "@types/marked": "^5.0.2", + "@types/node": "^22.14.0", + "@vitejs/plugin-vue": "6.0.0", + "@vitejs/plugin-vue-jsx": "5.0.1", + "@vue/tsconfig": "^0.7.0", + "less": "^4.3.0", + "less-loader": "^12.2.0", + "npm-run-all2": "^7.0.2", + "typescript": "~5.8.0", + "vite": "7.0.4", + "vue-tsc": "^2.2.8" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://mirrors.tencent.com/npm/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.0", + "resolved": "https://mirrors.tencent.com/npm/@babel/compat-data/-/compat-data-7.28.0.tgz", + "integrity": "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.0", + "resolved": "https://mirrors.tencent.com/npm/@babel/core/-/core-7.28.0.tgz", + "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", + "dev": true, + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.0", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.27.3", + "@babel/helpers": "^7.27.6", + "@babel/parser": "^7.28.0", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.0", + "@babel/types": "^7.28.0", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.0", + "resolved": "https://mirrors.tencent.com/npm/@babel/generator/-/generator-7.28.0.tgz", + "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.0", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-annotate-as-pure": { + "version": "7.27.3", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.27.3.tgz", + "integrity": "sha512-fXSwMQqitTGeHLBC08Eq5yXz2m37E4pJX1qAU1+2cNedz/ifv/bVXft90VeSav5nFO61EcNgwr0aJxbyPaWBPg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.27.3" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-create-class-features-plugin": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-create-class-features-plugin/-/helper-create-class-features-plugin-7.27.1.tgz", + "integrity": "sha512-QwGAmuvM17btKU5VqXfb+Giw4JcN0hjuufz3DYnpeVDvZLAObloM77bhMXiqry3Iio+Ai4phVRDwl6WU10+r5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.27.1", + "@babel/helper-member-expression-to-functions": "^7.27.1", + "@babel/helper-optimise-call-expression": "^7.27.1", + "@babel/helper-replace-supers": "^7.27.1", + "@babel/helper-skip-transparent-expression-wrappers": "^7.27.1", + "@babel/traverse": "^7.27.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-member-expression-to-functions": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.27.1.tgz", + "integrity": "sha512-E5chM8eWjTp/aNoVpcbfM7mLxu9XGLWYise2eBKGQomAk/Mb4XoxyqXTZbuTohbsl8EKqdlMhnDI2CCLfcs9wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.27.3", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", + "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.27.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-optimise-call-expression": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.27.1.tgz", + "integrity": "sha512-URMGH08NzYFhubNSGJrpUEphGKQwMQYBySzat5cAByY1/YgIRkULnIy3tAMeszlL/so2HbeilYloUmSpd7GdVw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-replace-supers": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-replace-supers/-/helper-replace-supers-7.27.1.tgz", + "integrity": "sha512-7EHz6qDZc8RYS5ElPoShMheWvEgERonFCs7IAonWLLUTXW59DP14bCZt89/GKyreYn8g3S83m21FelHKbeDCKA==", + "dev": true, + "dependencies": { + "@babel/helper-member-expression-to-functions": "^7.27.1", + "@babel/helper-optimise-call-expression": "^7.27.1", + "@babel/traverse": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-skip-transparent-expression-wrappers": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-skip-transparent-expression-wrappers/-/helper-skip-transparent-expression-wrappers-7.27.1.tgz", + "integrity": "sha512-Tub4ZKEXqbPjXgWLl2+3JpQAYBJ8+ikpQ2Ocj/q/r0LwE3UhENh7EUabyHjz2kCEsrRY83ew2DQdHluuiDQFzg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", + "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.27.6", + "resolved": "https://mirrors.tencent.com/npm/@babel/helpers/-/helpers-7.27.6.tgz", + "integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.27.6" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.0", + "resolved": "https://mirrors.tencent.com/npm/@babel/parser/-/parser-7.28.0.tgz", + "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", + "dependencies": { + "@babel/types": "^7.28.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", + "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.27.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", + "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-typescript": { + "version": "7.28.0", + "resolved": "https://mirrors.tencent.com/npm/@babel/plugin-transform-typescript/-/plugin-transform-typescript-7.28.0.tgz", + "integrity": "sha512-4AEiDEBPIZvLQaWlc9liCavE0xRM0dNca41WtBeM3jgFptfUOSG9z0uteLhq6+3rq+WB6jIvUwKDTpXEHPJ2Vg==", + "dev": true, + "dependencies": { + "@babel/helper-annotate-as-pure": "^7.27.3", + "@babel/helper-create-class-features-plugin": "^7.27.1", + "@babel/helper-plugin-utils": "^7.27.1", + "@babel/helper-skip-transparent-expression-wrappers": "^7.27.1", + "@babel/plugin-syntax-typescript": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.27.6", + "resolved": "https://mirrors.tencent.com/npm/@babel/runtime/-/runtime-7.27.6.tgz", + "integrity": "sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://mirrors.tencent.com/npm/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.0", + "resolved": "https://mirrors.tencent.com/npm/@babel/traverse/-/traverse-7.28.0.tgz", + "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.0", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.0", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.0", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.1", + "resolved": "https://mirrors.tencent.com/npm/@babel/types/-/types-7.28.1.tgz", + "integrity": "sha512-x0LvFTekgSX+83TI28Y9wYPUfzrnl2aT5+5QLnO6v7mSJYtEEevuDRN0F0uSHRk1G1IWZC43o00Y0xDDrpBGPQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/aix-ppc64/-/aix-ppc64-0.25.6.tgz", + "integrity": "sha512-ShbM/3XxwuxjFiuVBHA+d3j5dyac0aEVVq1oluIDf71hUw0aRF59dV/efUsIwFnR6m8JNM2FjZOzmaZ8yG61kw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/android-arm/-/android-arm-0.25.6.tgz", + "integrity": "sha512-S8ToEOVfg++AU/bHwdksHNnyLyVM+eMVAOf6yRKFitnwnbwwPNqKr3srzFRe7nzV69RQKb5DgchIX5pt3L53xg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/android-arm64/-/android-arm64-0.25.6.tgz", + "integrity": "sha512-hd5zdUarsK6strW+3Wxi5qWws+rJhCCbMiC9QZyzoxfk5uHRIE8T287giQxzVpEvCwuJ9Qjg6bEjcRJcgfLqoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/android-x64/-/android-x64-0.25.6.tgz", + "integrity": "sha512-0Z7KpHSr3VBIO9A/1wcT3NTy7EB4oNC4upJ5ye3R7taCc2GUdeynSLArnon5G8scPwaU866d3H4BCrE5xLW25A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/darwin-arm64/-/darwin-arm64-0.25.6.tgz", + "integrity": "sha512-FFCssz3XBavjxcFxKsGy2DYK5VSvJqa6y5HXljKzhRZ87LvEi13brPrf/wdyl/BbpbMKJNOr1Sd0jtW4Ge1pAA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/darwin-x64/-/darwin-x64-0.25.6.tgz", + "integrity": "sha512-GfXs5kry/TkGM2vKqK2oyiLFygJRqKVhawu3+DOCk7OxLy/6jYkWXhlHwOoTb0WqGnWGAS7sooxbZowy+pK9Yg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.6.tgz", + "integrity": "sha512-aoLF2c3OvDn2XDTRvn8hN6DRzVVpDlj2B/F66clWd/FHLiHaG3aVZjxQX2DYphA5y/evbdGvC6Us13tvyt4pWg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/freebsd-x64/-/freebsd-x64-0.25.6.tgz", + "integrity": "sha512-2SkqTjTSo2dYi/jzFbU9Plt1vk0+nNg8YC8rOXXea+iA3hfNJWebKYPs3xnOUf9+ZWhKAaxnQNUf2X9LOpeiMQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-arm/-/linux-arm-0.25.6.tgz", + "integrity": "sha512-SZHQlzvqv4Du5PrKE2faN0qlbsaW/3QQfUUc6yO2EjFcA83xnwm91UbEEVx4ApZ9Z5oG8Bxz4qPE+HFwtVcfyw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-arm64/-/linux-arm64-0.25.6.tgz", + "integrity": "sha512-b967hU0gqKd9Drsh/UuAm21Khpoh6mPBSgz8mKRq4P5mVK8bpA+hQzmm/ZwGVULSNBzKdZPQBRT3+WuVavcWsQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-ia32/-/linux-ia32-0.25.6.tgz", + "integrity": "sha512-aHWdQ2AAltRkLPOsKdi3xv0mZ8fUGPdlKEjIEhxCPm5yKEThcUjHpWB1idN74lfXGnZ5SULQSgtr5Qos5B0bPw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-loong64/-/linux-loong64-0.25.6.tgz", + "integrity": "sha512-VgKCsHdXRSQ7E1+QXGdRPlQ/e08bN6WMQb27/TMfV+vPjjTImuT9PmLXupRlC90S1JeNNW5lzkAEO/McKeJ2yg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-mips64el/-/linux-mips64el-0.25.6.tgz", + "integrity": "sha512-WViNlpivRKT9/py3kCmkHnn44GkGXVdXfdc4drNmRl15zVQ2+D2uFwdlGh6IuK5AAnGTo2qPB1Djppj+t78rzw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-ppc64/-/linux-ppc64-0.25.6.tgz", + "integrity": "sha512-wyYKZ9NTdmAMb5730I38lBqVu6cKl4ZfYXIs31Baf8aoOtB4xSGi3THmDYt4BTFHk7/EcVixkOV2uZfwU3Q2Jw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-riscv64/-/linux-riscv64-0.25.6.tgz", + "integrity": "sha512-KZh7bAGGcrinEj4qzilJ4hqTY3Dg2U82c8bv+e1xqNqZCrCyc+TL9AUEn5WGKDzm3CfC5RODE/qc96OcbIe33w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-s390x/-/linux-s390x-0.25.6.tgz", + "integrity": "sha512-9N1LsTwAuE9oj6lHMyyAM+ucxGiVnEqUdp4v7IaMmrwb06ZTEVCIs3oPPplVsnjPfyjmxwHxHMF8b6vzUVAUGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/linux-x64/-/linux-x64-0.25.6.tgz", + "integrity": "sha512-A6bJB41b4lKFWRKNrWoP2LHsjVzNiaurf7wyj/XtFNTsnPuxwEBWHLty+ZE0dWBKuSK1fvKgrKaNjBS7qbFKig==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.6.tgz", + "integrity": "sha512-IjA+DcwoVpjEvyxZddDqBY+uJ2Snc6duLpjmkXm/v4xuS3H+3FkLZlDm9ZsAbF9rsfP3zeA0/ArNDORZgrxR/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/netbsd-x64/-/netbsd-x64-0.25.6.tgz", + "integrity": "sha512-dUXuZr5WenIDlMHdMkvDc1FAu4xdWixTCRgP7RQLBOkkGgwuuzaGSYcOpW4jFxzpzL1ejb8yF620UxAqnBrR9g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.6.tgz", + "integrity": "sha512-l8ZCvXP0tbTJ3iaqdNf3pjaOSd5ex/e6/omLIQCVBLmHTlfXW3zAxQ4fnDmPLOB1x9xrcSi/xtCWFwCZRIaEwg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/openbsd-x64/-/openbsd-x64-0.25.6.tgz", + "integrity": "sha512-hKrmDa0aOFOr71KQ/19JC7az1P0GWtCN1t2ahYAf4O007DHZt/dW8ym5+CUdJhQ/qkZmI1HAF8KkJbEFtCL7gw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.6.tgz", + "integrity": "sha512-+SqBcAWoB1fYKmpWoQP4pGtx+pUUC//RNYhFdbcSA16617cchuryuhOCRpPsjCblKukAckWsV+aQ3UKT/RMPcA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/sunos-x64/-/sunos-x64-0.25.6.tgz", + "integrity": "sha512-dyCGxv1/Br7MiSC42qinGL8KkG4kX0pEsdb0+TKhmJZgCUDBGmyo1/ArCjNGiOLiIAgdbWgmWgib4HoCi5t7kA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/win32-arm64/-/win32-arm64-0.25.6.tgz", + "integrity": "sha512-42QOgcZeZOvXfsCBJF5Afw73t4veOId//XD3i+/9gSkhSV6Gk3VPlWncctI+JcOyERv85FUo7RxuxGy+z8A43Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/win32-ia32/-/win32-ia32-0.25.6.tgz", + "integrity": "sha512-4AWhgXmDuYN7rJI6ORB+uU9DHLq/erBbuMoAuB4VWJTu5KtCgcKYPynF0YI1VkBNuEfjNlLrFr9KZPJzrtLkrQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/@esbuild/win32-x64/-/win32-x64-0.25.6.tgz", + "integrity": "sha512-NgJPHHbEpLQgDH2MjQu90pzW/5vvXIZ7KOnPyNBm92A6WgZ/7b6fJyUBjoumLqeOQQGqY2QjQxRo97ah4Sj0cA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.12", + "resolved": "https://mirrors.tencent.com/npm/@jridgewell/gen-mapping/-/gen-mapping-0.3.12.tgz", + "integrity": "sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://mirrors.tencent.com/npm/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.10", + "resolved": "https://mirrors.tencent.com/npm/@jridgewell/source-map/-/source-map-0.3.10.tgz", + "integrity": "sha512-0pPkgz9dY+bijgistcTTJ5mR+ocqRXLuhXHYdzoMmmoJ2C9S46RCm2GMUbatPEUK9Yjy26IrAy8D/M00lLkv+Q==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.4", + "resolved": "https://mirrors.tencent.com/npm/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz", + "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.29", + "resolved": "https://mirrors.tencent.com/npm/@jridgewell/trace-mapping/-/trace-mapping-0.3.29.tgz", + "integrity": "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@microsoft/fetch-event-source": { + "version": "2.0.1", + "resolved": "https://mirrors.tencent.com/npm/@microsoft/fetch-event-source/-/fetch-event-source-2.0.1.tgz", + "integrity": "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==" + }, + "node_modules/@pagefind/darwin-arm64": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/@pagefind/darwin-arm64/-/darwin-arm64-1.3.0.tgz", + "integrity": "sha512-365BEGl6ChOsauRjyVpBjXybflXAOvoMROw3TucAROHIcdBvXk9/2AmEvGFU0r75+vdQI4LJdJdpH4Y6Yqaj4A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/darwin-x64": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/@pagefind/darwin-x64/-/darwin-x64-1.3.0.tgz", + "integrity": "sha512-zlGHA23uuXmS8z3XxEGmbHpWDxXfPZ47QS06tGUq0HDcZjXjXHeLG+cboOy828QIV5FXsm9MjfkP5e4ZNbOkow==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@pagefind/linux-arm64": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/@pagefind/linux-arm64/-/linux-arm64-1.3.0.tgz", + "integrity": "sha512-8lsxNAiBRUk72JvetSBXs4WRpYrQrVJXjlRRnOL6UCdBN9Nlsz0t7hWstRk36+JqHpGWOKYiuHLzGYqYAqoOnQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/linux-x64": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/@pagefind/linux-x64/-/linux-x64-1.3.0.tgz", + "integrity": "sha512-hAvqdPJv7A20Ucb6FQGE6jhjqy+vZ6pf+s2tFMNtMBG+fzcdc91uTw7aP/1Vo5plD0dAOHwdxfkyw0ugal4kcQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@pagefind/windows-x64": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/@pagefind/windows-x64/-/windows-x64-1.3.0.tgz", + "integrity": "sha512-BR1bIRWOMqkf8IoU576YDhij1Wd/Zf2kX/kCI0b2qzCKC8wcc2GQJaaRMCpzvCCrmliO4vtJ6RITp/AnoYUUmQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@popperjs/core": { + "version": "2.11.8", + "resolved": "https://mirrors.tencent.com/npm/@popperjs/core/-/core-2.11.8.tgz", + "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.19", + "resolved": "https://mirrors.tencent.com/npm/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.19.tgz", + "integrity": "sha512-3FL3mnMbPu0muGOCaKAhhFEYmqv9eTfPSJRJmANrCwtgK8VuxpsZDGK+m0LYAGoyO8+0j5uRe4PeyPDK1yA/hA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.45.0.tgz", + "integrity": "sha512-2o/FgACbji4tW1dzXOqAV15Eu7DdgbKsF2QKcxfG4xbh5iwU7yr5RRP5/U+0asQliSYv5M4o7BevlGIoSL0LXg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.45.0.tgz", + "integrity": "sha512-PSZ0SvMOjEAxwZeTx32eI/j5xSYtDCRxGu5k9zvzoY77xUNssZM+WV6HYBLROpY5CkXsbQjvz40fBb7WPwDqtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.45.0.tgz", + "integrity": "sha512-BA4yPIPssPB2aRAWzmqzQ3y2/KotkLyZukVB7j3psK/U3nVJdceo6qr9pLM2xN6iRP/wKfxEbOb1yrlZH6sYZg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.45.0.tgz", + "integrity": "sha512-Pr2o0lvTwsiG4HCr43Zy9xXrHspyMvsvEw4FwKYqhli4FuLE5FjcZzuQ4cfPe0iUFCvSQG6lACI0xj74FDZKRA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.45.0.tgz", + "integrity": "sha512-lYE8LkE5h4a/+6VnnLiL14zWMPnx6wNbDG23GcYFpRW1V9hYWHAw9lBZ6ZUIrOaoK7NliF1sdwYGiVmziUF4vA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.45.0.tgz", + "integrity": "sha512-PVQWZK9sbzpvqC9Q0GlehNNSVHR+4m7+wET+7FgSnKG3ci5nAMgGmr9mGBXzAuE5SvguCKJ6mHL6vq1JaJ/gvw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.45.0.tgz", + "integrity": "sha512-hLrmRl53prCcD+YXTfNvXd776HTxNh8wPAMllusQ+amcQmtgo3V5i/nkhPN6FakW+QVLoUUr2AsbtIRPFU3xIA==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.45.0.tgz", + "integrity": "sha512-XBKGSYcrkdiRRjl+8XvrUR3AosXU0NvF7VuqMsm7s5nRy+nt58ZMB19Jdp1RdqewLcaYnpk8zeVs/4MlLZEJxw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.45.0.tgz", + "integrity": "sha512-fRvZZPUiBz7NztBE/2QnCS5AtqLVhXmUOPj9IHlfGEXkapgImf4W9+FSkL8cWqoAjozyUzqFmSc4zh2ooaeF6g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.45.0.tgz", + "integrity": "sha512-Btv2WRZOcUGi8XU80XwIvzTg4U6+l6D0V6sZTrZx214nrwxw5nAi8hysaXj/mctyClWgesyuxbeLylCBNauimg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loongarch64-gnu": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.45.0.tgz", + "integrity": "sha512-Li0emNnwtUZdLwHjQPBxn4VWztcrw/h7mgLyHiEI5Z0MhpeFGlzaiBHpSNVOMB/xucjXTTcO+dhv469Djr16KA==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.45.0.tgz", + "integrity": "sha512-sB8+pfkYx2kvpDCfd63d5ScYT0Fz1LO6jIb2zLZvmK9ob2D8DeVqrmBDE0iDK8KlBVmsTNzrjr3G1xV4eUZhSw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.45.0.tgz", + "integrity": "sha512-5GQ6PFhh7E6jQm70p1aW05G2cap5zMOvO0se5JMecHeAdj5ZhWEHbJ4hiKpfi1nnnEdTauDXxPgXae/mqjow9w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.45.0.tgz", + "integrity": "sha512-N/euLsBd1rekWcuduakTo/dJw6U6sBP3eUq+RXM9RNfPuWTvG2w/WObDkIvJ2KChy6oxZmOSC08Ak2OJA0UiAA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.45.0.tgz", + "integrity": "sha512-2l9sA7d7QdikL0xQwNMO3xURBUNEWyHVHfAsHsUdq+E/pgLTUcCE+gih5PCdmyHmfTDeXUWVhqL0WZzg0nua3g==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.45.0.tgz", + "integrity": "sha512-XZdD3fEEQcwG2KrJDdEQu7NrHonPxxaV0/w2HpvINBdcqebz1aL+0vM2WFJq4DeiAVT6F5SUQas65HY5JDqoPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.45.0.tgz", + "integrity": "sha512-7ayfgvtmmWgKWBkCGg5+xTQ0r5V1owVm67zTrsEY1008L5ro7mCyGYORomARt/OquB9KY7LpxVBZes+oSniAAQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.45.0.tgz", + "integrity": "sha512-B+IJgcBnE2bm93jEW5kHisqvPITs4ddLOROAcOc/diBgrEiQJJ6Qcjby75rFSmH5eMGrqJryUgJDhrfj942apQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.45.0.tgz", + "integrity": "sha512-+CXwwG66g0/FpWOnP/v1HnrGVSOygK/osUbu3wPRy8ECXjoYKjRAyfxYpDQOfghC5qPJYLPH0oN4MCOjwgdMug==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.45.0.tgz", + "integrity": "sha512-SRf1cytG7wqcHVLrBc9VtPK4pU5wxiB/lNIkNmW2ApKXIg+RpqwHfsaEK+e7eH4A1BpI6BX/aBWXxZCIrJg3uA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@tsconfig/node22": { + "version": "22.0.2", + "resolved": "https://mirrors.tencent.com/npm/@tsconfig/node22/-/node22-22.0.2.tgz", + "integrity": "sha512-Kmwj4u8sDRDrMYRoN9FDEcXD8UpBSaPQQ24Gz+Gamqfm7xxn+GBR7ge/Z7pK8OXNGyUzbSwJj+TH6B+DS/epyA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/dompurify": { + "version": "3.0.5", + "resolved": "https://mirrors.tencent.com/npm/@types/dompurify/-/dompurify-3.0.5.tgz", + "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==", + "license": "MIT", + "dependencies": { + "@types/trusted-types": "*" + } + }, + "node_modules/@types/eslint": { + "version": "9.6.1", + "resolved": "https://mirrors.tencent.com/npm/@types/eslint/-/eslint-9.6.1.tgz", + "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", + "license": "MIT", + "dependencies": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "node_modules/@types/eslint-scope": { + "version": "3.7.7", + "resolved": "https://mirrors.tencent.com/npm/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", + "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", + "license": "MIT", + "dependencies": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://mirrors.tencent.com/npm/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://mirrors.tencent.com/npm/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" + }, + "node_modules/@types/lodash": { + "version": "4.17.20", + "resolved": "https://mirrors.tencent.com/npm/@types/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA==", + "license": "MIT" + }, + "node_modules/@types/lodash-es": { + "version": "4.17.12", + "resolved": "https://mirrors.tencent.com/npm/@types/lodash-es/-/lodash-es-4.17.12.tgz", + "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", + "license": "MIT", + "dependencies": { + "@types/lodash": "*" + } + }, + "node_modules/@types/marked": { + "version": "5.0.2", + "resolved": "https://mirrors.tencent.com/npm/@types/marked/-/marked-5.0.2.tgz", + "integrity": "sha512-OucS4KMHhFzhz27KxmWg7J+kIYqyqoW5kdIEI319hqARQQUTqhao3M/F+uFnDXD0Rg72iDDZxZNxq5gvctmLlg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.16.3", + "resolved": "https://mirrors.tencent.com/npm/@types/node/-/node-22.16.3.tgz", + "integrity": "sha512-sr4Xz74KOUeYadexo1r8imhRtlVXcs+j3XK3TcoiYk7B1t3YRVJgtaD3cwX73NYb71pmVuMLNRhJ9XKdoDB74g==", + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/sortablejs": { + "version": "1.15.8", + "resolved": "https://mirrors.tencent.com/npm/@types/sortablejs/-/sortablejs-1.15.8.tgz", + "integrity": "sha512-b79830lW+RZfwaztgs1aVPgbasJ8e7AXtZYHTELNXZPsERt4ymJdjV4OccDbHQAvHrCcFpbF78jkm0R6h/pZVg==", + "license": "MIT" + }, + "node_modules/@types/tinycolor2": { + "version": "1.4.6", + "resolved": "https://mirrors.tencent.com/npm/@types/tinycolor2/-/tinycolor2-1.4.6.tgz", + "integrity": "sha512-iEN8J0BoMnsWBqjVbWH/c0G0Hh7O21lpR2/+PrvAVgWdzL7eexIFm4JN/Wn10PTcmNdtS6U67r499mlWMXOxNw==" + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://mirrors.tencent.com/npm/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT" + }, + "node_modules/@types/validator": { + "version": "13.15.2", + "resolved": "https://mirrors.tencent.com/npm/@types/validator/-/validator-13.15.2.tgz", + "integrity": "sha512-y7pa/oEJJ4iGYBxOpfAKn5b9+xuihvzDVnC/OSvlVnGxVg0pOqmjiMafiJ1KVNQEaPZf9HsEp5icEwGg8uIe5Q==", + "license": "MIT" + }, + "node_modules/@vitejs/plugin-vue": { + "version": "6.0.0", + "resolved": "https://mirrors.tencent.com/npm/@vitejs/plugin-vue/-/plugin-vue-6.0.0.tgz", + "integrity": "sha512-iAliE72WsdhjzTOp2DtvKThq1VBC4REhwRcaA+zPAAph6I+OQhUXv+Xu2KS7ElxYtb7Zc/3R30Hwv1DxEo7NXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "1.0.0-beta.19" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0", + "vue": "^3.2.25" + } + }, + "node_modules/@vitejs/plugin-vue-jsx": { + "version": "5.0.1", + "resolved": "https://mirrors.tencent.com/npm/@vitejs/plugin-vue-jsx/-/plugin-vue-jsx-5.0.1.tgz", + "integrity": "sha512-X7qmQMXbdDh+sfHUttXokPD0cjPkMFoae7SgbkF9vi3idGUKmxLcnU2Ug49FHwiKXebfzQRIm5yK3sfCJzNBbg==", + "dev": true, + "dependencies": { + "@babel/core": "^7.27.7", + "@babel/plugin-transform-typescript": "^7.27.1", + "@rolldown/pluginutils": "^1.0.0-beta.21", + "@vue/babel-plugin-jsx": "^1.4.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0", + "vue": "^3.0.0" + } + }, + "node_modules/@vitejs/plugin-vue-jsx/node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.27", + "resolved": "https://mirrors.tencent.com/npm/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", + "integrity": "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@volar/language-core": { + "version": "2.4.15", + "resolved": "https://mirrors.tencent.com/npm/@volar/language-core/-/language-core-2.4.15.tgz", + "integrity": "sha512-3VHw+QZU0ZG9IuQmzT68IyN4hZNd9GchGPhbD9+pa8CVv7rnoOZwo7T8weIbrRmihqy3ATpdfXFnqRrfPVK6CA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/source-map": "2.4.15" + } + }, + "node_modules/@volar/source-map": { + "version": "2.4.15", + "resolved": "https://mirrors.tencent.com/npm/@volar/source-map/-/source-map-2.4.15.tgz", + "integrity": "sha512-CPbMWlUN6hVZJYGcU/GSoHu4EnCHiLaXI9n8c9la6RaI9W5JHX+NqG+GSQcB0JdC2FIBLdZJwGsfKyBB71VlTg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@volar/typescript": { + "version": "2.4.15", + "resolved": "https://mirrors.tencent.com/npm/@volar/typescript/-/typescript-2.4.15.tgz", + "integrity": "sha512-2aZ8i0cqPGjXb4BhkMsPYDkkuc2ZQ6yOpqwAuNwUoncELqoy5fRgOQtLR9gB0g902iS0NAkvpIzs27geVyVdPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@volar/language-core": "2.4.15", + "path-browserify": "^1.0.1", + "vscode-uri": "^3.0.8" + } + }, + "node_modules/@vue/babel-helper-vue-transform-on": { + "version": "1.4.0", + "resolved": "https://mirrors.tencent.com/npm/@vue/babel-helper-vue-transform-on/-/babel-helper-vue-transform-on-1.4.0.tgz", + "integrity": "sha512-mCokbouEQ/ocRce/FpKCRItGo+013tHg7tixg3DUNS+6bmIchPt66012kBMm476vyEIJPafrvOf4E5OYj3shSw==", + "dev": true + }, + "node_modules/@vue/babel-plugin-jsx": { + "version": "1.4.0", + "resolved": "https://mirrors.tencent.com/npm/@vue/babel-plugin-jsx/-/babel-plugin-jsx-1.4.0.tgz", + "integrity": "sha512-9zAHmwgMWlaN6qRKdrg1uKsBKHvnUU+Py+MOCTuYZBoZsopa90Di10QRjB+YPnVss0BZbG/H5XFwJY1fTxJWhA==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.26.5", + "@babel/plugin-syntax-jsx": "^7.25.9", + "@babel/template": "^7.26.9", + "@babel/traverse": "^7.26.9", + "@babel/types": "^7.26.9", + "@vue/babel-helper-vue-transform-on": "1.4.0", + "@vue/babel-plugin-resolve-type": "1.4.0", + "@vue/shared": "^3.5.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + } + } + }, + "node_modules/@vue/babel-plugin-resolve-type": { + "version": "1.4.0", + "resolved": "https://mirrors.tencent.com/npm/@vue/babel-plugin-resolve-type/-/babel-plugin-resolve-type-1.4.0.tgz", + "integrity": "sha512-4xqDRRbQQEWHQyjlYSgZsWj44KfiF6D+ktCuXyZ8EnVDYV3pztmXJDf1HveAjUAXxAnR8daCQT51RneWWxtTyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-plugin-utils": "^7.26.5", + "@babel/parser": "^7.26.9", + "@vue/compiler-sfc": "^3.5.13" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@vue/compiler-core": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/compiler-core/-/compiler-core-3.5.17.tgz", + "integrity": "sha512-Xe+AittLbAyV0pabcN7cP7/BenRBNcteM4aSDCtRvGw0d9OL+HG1u/XHLY/kt1q4fyMeZYXyIYrsHuPSiDPosA==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.27.5", + "@vue/shared": "3.5.17", + "entities": "^4.5.0", + "estree-walker": "^2.0.2", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-dom": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/compiler-dom/-/compiler-dom-3.5.17.tgz", + "integrity": "sha512-+2UgfLKoaNLhgfhV5Ihnk6wB4ljyW1/7wUIog2puUqajiC29Lp5R/IKDdkebh9jTbTogTbsgB+OY9cEWzG95JQ==", + "dependencies": { + "@vue/compiler-core": "3.5.17", + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/compiler-sfc/-/compiler-sfc-3.5.17.tgz", + "integrity": "sha512-rQQxbRJMgTqwRugtjw0cnyQv9cP4/4BxWfTdRBkqsTfLOHWykLzbOc3C4GGzAmdMDxhzU/1Ija5bTjMVrddqww==", + "dependencies": { + "@babel/parser": "^7.27.5", + "@vue/compiler-core": "3.5.17", + "@vue/compiler-dom": "3.5.17", + "@vue/compiler-ssr": "3.5.17", + "@vue/shared": "3.5.17", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.17", + "postcss": "^8.5.6", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/compiler-ssr/-/compiler-ssr-3.5.17.tgz", + "integrity": "sha512-hkDbA0Q20ZzGgpj5uZjb9rBzQtIHLS78mMilwrlpWk2Ep37DYntUz0PonQ6kr113vfOEdM+zTBuJDaceNIW0tQ==", + "dependencies": { + "@vue/compiler-dom": "3.5.17", + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/compiler-vue2": { + "version": "2.7.16", + "resolved": "https://mirrors.tencent.com/npm/@vue/compiler-vue2/-/compiler-vue2-2.7.16.tgz", + "integrity": "sha512-qYC3Psj9S/mfu9uVi5WvNZIzq+xnXMhOwbTFKKDD7b1lhpnn71jXSFdTQ+WsIEk0ONCd7VV2IMm7ONl6tbQ86A==", + "dev": true, + "license": "MIT", + "dependencies": { + "de-indent": "^1.0.2", + "he": "^1.2.0" + } + }, + "node_modules/@vue/devtools-api": { + "version": "7.7.7", + "resolved": "https://mirrors.tencent.com/npm/@vue/devtools-api/-/devtools-api-7.7.7.tgz", + "integrity": "sha512-lwOnNBH2e7x1fIIbVT7yF5D+YWhqELm55/4ZKf45R9T8r9dE2AIOy8HKjfqzGsoTHFbWbr337O4E0A0QADnjBg==", + "license": "MIT", + "dependencies": { + "@vue/devtools-kit": "^7.7.7" + } + }, + "node_modules/@vue/devtools-kit": { + "version": "7.7.7", + "resolved": "https://mirrors.tencent.com/npm/@vue/devtools-kit/-/devtools-kit-7.7.7.tgz", + "integrity": "sha512-wgoZtxcTta65cnZ1Q6MbAfePVFxfM+gq0saaeytoph7nEa7yMXoi6sCPy4ufO111B9msnw0VOWjPEFCXuAKRHA==", + "license": "MIT", + "dependencies": { + "@vue/devtools-shared": "^7.7.7", + "birpc": "^2.3.0", + "hookable": "^5.5.3", + "mitt": "^3.0.1", + "perfect-debounce": "^1.0.0", + "speakingurl": "^14.0.1", + "superjson": "^2.2.2" + } + }, + "node_modules/@vue/devtools-shared": { + "version": "7.7.7", + "resolved": "https://mirrors.tencent.com/npm/@vue/devtools-shared/-/devtools-shared-7.7.7.tgz", + "integrity": "sha512-+udSj47aRl5aKb0memBvcUG9koarqnxNM5yjuREvqwK6T3ap4mn3Zqqc17QrBFTqSMjr3HK1cvStEZpMDpfdyw==", + "license": "MIT", + "dependencies": { + "rfdc": "^1.4.1" + } + }, + "node_modules/@vue/language-core": { + "version": "2.2.12", + "resolved": "https://mirrors.tencent.com/npm/@vue/language-core/-/language-core-2.2.12.tgz", + "integrity": "sha512-IsGljWbKGU1MZpBPN+BvPAdr55YPkj2nB/TBNGNC32Vy2qLG25DYu/NBN2vNtZqdRbTRjaoYrahLrToim2NanA==", + "dev": true, + "dependencies": { + "@volar/language-core": "2.4.15", + "@vue/compiler-dom": "^3.5.0", + "@vue/compiler-vue2": "^2.7.16", + "@vue/shared": "^3.5.0", + "alien-signals": "^1.0.3", + "minimatch": "^9.0.3", + "muggle-string": "^0.4.1", + "path-browserify": "^1.0.1" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@vue/reactivity": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/reactivity/-/reactivity-3.5.17.tgz", + "integrity": "sha512-l/rmw2STIscWi7SNJp708FK4Kofs97zc/5aEPQh4bOsReD/8ICuBcEmS7KGwDj5ODQLYWVN2lNibKJL1z5b+Lw==", + "license": "MIT", + "dependencies": { + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/runtime-core/-/runtime-core-3.5.17.tgz", + "integrity": "sha512-QQLXa20dHg1R0ri4bjKeGFKEkJA7MMBxrKo2G+gJikmumRS7PTD4BOU9FKrDQWMKowz7frJJGqBffYMgQYS96Q==", + "dependencies": { + "@vue/reactivity": "3.5.17", + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/runtime-dom/-/runtime-dom-3.5.17.tgz", + "integrity": "sha512-8El0M60TcwZ1QMz4/os2MdlQECgGoVHPuLnQBU3m9h3gdNRW9xRmI8iLS4t/22OQlOE6aJvNNlBiCzPHur4H9g==", + "license": "MIT", + "dependencies": { + "@vue/reactivity": "3.5.17", + "@vue/runtime-core": "3.5.17", + "@vue/shared": "3.5.17", + "csstype": "^3.1.3" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/server-renderer/-/server-renderer-3.5.17.tgz", + "integrity": "sha512-BOHhm8HalujY6lmC3DbqF6uXN/K00uWiEeF22LfEsm9Q93XeJ/plHTepGwf6tqFcF7GA5oGSSAAUock3VvzaCA==", + "dependencies": { + "@vue/compiler-ssr": "3.5.17", + "@vue/shared": "3.5.17" + }, + "peerDependencies": { + "vue": "3.5.17" + } + }, + "node_modules/@vue/shared": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/@vue/shared/-/shared-3.5.17.tgz", + "integrity": "sha512-CabR+UN630VnsJO/jHWYBC1YVXyMq94KKp6iF5MQgZJs5I8cmjw6oVMO1oDbtBkENSHSSn/UadWlW/OAgdmKrg==", + "license": "MIT" + }, + "node_modules/@vue/tsconfig": { + "version": "0.7.0", + "resolved": "https://mirrors.tencent.com/npm/@vue/tsconfig/-/tsconfig-0.7.0.tgz", + "integrity": "sha512-ku2uNz5MaZ9IerPPUyOHzyjhXoX2kVJaVf7hL315DC17vS6IiZRmmCPfggNbU16QTvM80+uYYy3eYJB59WCtvg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "typescript": "5.x", + "vue": "^3.4.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + }, + "vue": { + "optional": true + } + } + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/ast/-/ast-1.14.1.tgz", + "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/helper-numbers": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.13.2", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz", + "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==" + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.13.2", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz", + "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz", + "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.13.2", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz", + "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.13.2", + "@webassemblyjs/helper-api-error": "1.13.2", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.13.2", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz", + "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==" + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz", + "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/wasm-gen": "1.14.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.13.2", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz", + "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==", + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.13.2", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/leb128/-/leb128-1.13.2.tgz", + "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==", + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.13.2", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/utf8/-/utf8-1.13.2.tgz", + "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz", + "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/helper-wasm-section": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-opt": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1", + "@webassemblyjs/wast-printer": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz", + "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz", + "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz", + "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-api-error": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.14.1", + "resolved": "https://mirrors.tencent.com/npm/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz", + "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://mirrors.tencent.com/npm/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==" + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://mirrors.tencent.com/npm/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "license": "Apache-2.0" + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://mirrors.tencent.com/npm/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-phases": { + "version": "1.0.4", + "resolved": "https://mirrors.tencent.com/npm/acorn-import-phases/-/acorn-import-phases-1.0.4.tgz", + "integrity": "sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==", + "license": "MIT", + "engines": { + "node": ">=10.13.0" + }, + "peerDependencies": { + "acorn": "^8.14.0" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://mirrors.tencent.com/npm/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://mirrors.tencent.com/npm/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://mirrors.tencent.com/npm/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/alien-signals": { + "version": "1.0.13", + "resolved": "https://mirrors.tencent.com/npm/alien-signals/-/alien-signals-1.0.13.tgz", + "integrity": "sha512-OGj9yyTnJEttvzhTUWuscOvtqxq5vrhF7vL9oS0xJ2mK0ItPYP1/y+vCFebfxoEyAz0++1AIwJ5CMr+Fk3nDmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://mirrors.tencent.com/npm/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://mirrors.tencent.com/npm/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.10.0", + "resolved": "https://mirrors.tencent.com/npm/axios/-/axios-1.10.0.tgz", + "integrity": "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://mirrors.tencent.com/npm/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/birpc": { + "version": "2.5.0", + "resolved": "https://mirrors.tencent.com/npm/birpc/-/birpc-2.5.0.tgz", + "integrity": "sha512-VSWO/W6nNQdyP520F1mhf+Lc2f8pjGQOtoHHm7Ze8Go1kX7akpVIrtTa0fn+HB0QJEDVacl6aO08YE0PgXfdnQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://mirrors.tencent.com/npm/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.25.1", + "resolved": "https://mirrors.tencent.com/npm/browserslist/-/browserslist-4.25.1.tgz", + "integrity": "sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001726", + "electron-to-chromium": "^1.5.173", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://mirrors.tencent.com/npm/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://mirrors.tencent.com/npm/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001727", + "resolved": "https://mirrors.tencent.com/npm/caniuse-lite/-/caniuse-lite-1.0.30001727.tgz", + "integrity": "sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chrome-trace-event": { + "version": "1.0.4", + "resolved": "https://mirrors.tencent.com/npm/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz", + "integrity": "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==", + "license": "MIT", + "engines": { + "node": ">=6.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://mirrors.tencent.com/npm/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://mirrors.tencent.com/npm/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://mirrors.tencent.com/npm/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/copy-anything": { + "version": "2.0.6", + "resolved": "https://mirrors.tencent.com/npm/copy-anything/-/copy-anything-2.0.6.tgz", + "integrity": "sha512-1j20GZTsvKNkc4BY3NpMOM8tt///wY3FpIzozTOFO2ffuZcV61nojHXVKIy3WM+7ADCy5FVhdZYHYDdgTU0yJw==", + "dev": true, + "dependencies": { + "is-what": "^3.14.1" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://mirrors.tencent.com/npm/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cross-spawn/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://mirrors.tencent.com/npm/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true + }, + "node_modules/cross-spawn/node_modules/which": { + "version": "2.0.2", + "resolved": "https://mirrors.tencent.com/npm/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://mirrors.tencent.com/npm/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" + }, + "node_modules/dayjs": { + "version": "1.11.10", + "resolved": "https://mirrors.tencent.com/npm/dayjs/-/dayjs-1.11.10.tgz", + "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" + }, + "node_modules/de-indent": { + "version": "1.0.2", + "resolved": "https://mirrors.tencent.com/npm/de-indent/-/de-indent-1.0.2.tgz", + "integrity": "sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg==", + "dev": true + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://mirrors.tencent.com/npm/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "dev": true, + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://mirrors.tencent.com/npm/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dompurify": { + "version": "3.2.6", + "resolved": "https://mirrors.tencent.com/npm/dompurify/-/dompurify-3.2.6.tgz", + "integrity": "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://mirrors.tencent.com/npm/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.183", + "resolved": "https://mirrors.tencent.com/npm/electron-to-chromium/-/electron-to-chromium-1.5.183.tgz", + "integrity": "sha512-vCrDBYjQCAEefWGjlK3EpoSKfKbT10pR4XXPdn65q7snuNOZnthoVpBfZPykmDapOKfoD+MMIPG8ZjKyyc9oHA==" + }, + "node_modules/enhanced-resolve": { + "version": "5.18.2", + "resolved": "https://mirrors.tencent.com/npm/enhanced-resolve/-/enhanced-resolve-5.18.2.tgz", + "integrity": "sha512-6Jw4sE1maoRJo3q8MsSIn2onJFbLTOjY9hlx4DZXmOKvLRd1Ok2kXmAGXaafL2+ijsJZ1ClYbl/pmqr9+k4iUQ==", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://mirrors.tencent.com/npm/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/errno": { + "version": "0.1.8", + "resolved": "https://mirrors.tencent.com/npm/errno/-/errno-0.1.8.tgz", + "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, + "optional": true, + "dependencies": { + "prr": "~1.0.1" + }, + "bin": { + "errno": "cli.js" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://mirrors.tencent.com/npm/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.7.0", + "resolved": "https://mirrors.tencent.com/npm/es-module-lexer/-/es-module-lexer-1.7.0.tgz", + "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==" + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://mirrors.tencent.com/npm/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://mirrors.tencent.com/npm/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.25.6", + "resolved": "https://mirrors.tencent.com/npm/esbuild/-/esbuild-0.25.6.tgz", + "integrity": "sha512-GVuzuUwtdsghE3ocJ9Bs8PNoF13HNQ5TXbEi2AhvVb8xU1Iwt9Fos9FEamfoee+u/TOsn7GUWc04lz46n2bbTg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.6", + "@esbuild/android-arm": "0.25.6", + "@esbuild/android-arm64": "0.25.6", + "@esbuild/android-x64": "0.25.6", + "@esbuild/darwin-arm64": "0.25.6", + "@esbuild/darwin-x64": "0.25.6", + "@esbuild/freebsd-arm64": "0.25.6", + "@esbuild/freebsd-x64": "0.25.6", + "@esbuild/linux-arm": "0.25.6", + "@esbuild/linux-arm64": "0.25.6", + "@esbuild/linux-ia32": "0.25.6", + "@esbuild/linux-loong64": "0.25.6", + "@esbuild/linux-mips64el": "0.25.6", + "@esbuild/linux-ppc64": "0.25.6", + "@esbuild/linux-riscv64": "0.25.6", + "@esbuild/linux-s390x": "0.25.6", + "@esbuild/linux-x64": "0.25.6", + "@esbuild/netbsd-arm64": "0.25.6", + "@esbuild/netbsd-x64": "0.25.6", + "@esbuild/openbsd-arm64": "0.25.6", + "@esbuild/openbsd-x64": "0.25.6", + "@esbuild/openharmony-arm64": "0.25.6", + "@esbuild/sunos-x64": "0.25.6", + "@esbuild/win32-arm64": "0.25.6", + "@esbuild/win32-ia32": "0.25.6", + "@esbuild/win32-x64": "0.25.6" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://mirrors.tencent.com/npm/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://mirrors.tencent.com/npm/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://mirrors.tencent.com/npm/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://mirrors.tencent.com/npm/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://mirrors.tencent.com/npm/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://mirrors.tencent.com/npm/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==" + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://mirrors.tencent.com/npm/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://mirrors.tencent.com/npm/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.0.6", + "resolved": "https://mirrors.tencent.com/npm/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fdir": { + "version": "6.4.6", + "resolved": "https://mirrors.tencent.com/npm/fdir/-/fdir-6.4.6.tgz", + "integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==", + "dev": true, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://mirrors.tencent.com/npm/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.3", + "resolved": "https://mirrors.tencent.com/npm/form-data/-/form-data-4.0.3.tgz", + "integrity": "sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://mirrors.tencent.com/npm/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://mirrors.tencent.com/npm/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://mirrors.tencent.com/npm/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://mirrors.tencent.com/npm/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://mirrors.tencent.com/npm/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "license": "BSD-2-Clause" + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://mirrors.tencent.com/npm/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://mirrors.tencent.com/npm/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://mirrors.tencent.com/npm/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://mirrors.tencent.com/npm/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://mirrors.tencent.com/npm/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://mirrors.tencent.com/npm/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://mirrors.tencent.com/npm/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/hookable": { + "version": "5.5.3", + "resolved": "https://mirrors.tencent.com/npm/hookable/-/hookable-5.5.3.tgz", + "integrity": "sha512-Yc+BQe8SvoXH1643Qez1zqLRmbA5rCL+sSmk6TVos0LWVfNIB7PGncdlId77WzLGSIB5KaWgTaNTs2lNVEI6VQ==", + "license": "MIT" + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://mirrors.tencent.com/npm/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dev": true, + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/image-size": { + "version": "0.5.5", + "resolved": "https://mirrors.tencent.com/npm/image-size/-/image-size-0.5.5.tgz", + "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", + "dev": true, + "optional": true, + "bin": { + "image-size": "bin/image-size.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-what": { + "version": "3.14.1", + "resolved": "https://mirrors.tencent.com/npm/is-what/-/is-what-3.14.1.tgz", + "integrity": "sha512-sNxgpk9793nzSs7bA6JQJGeIuRBQhAaNGG77kzYQgMkrID+lS6SlK07K5LaptscDlSaIgH+GPFzf+d75FVxozA==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "3.1.1", + "resolved": "https://mirrors.tencent.com/npm/isexe/-/isexe-3.1.1.tgz", + "integrity": "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ==", + "dev": true, + "engines": { + "node": ">=16" + } + }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://mirrors.tencent.com/npm/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://mirrors.tencent.com/npm/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://mirrors.tencent.com/npm/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "4.0.0", + "resolved": "https://mirrors.tencent.com/npm/json-parse-even-better-errors/-/json-parse-even-better-errors-4.0.0.tgz", + "integrity": "sha512-lR4MXjGNgkJc7tkQ97kb2nuEMnNCyU//XYVH0MKTGcXEiSudQ5MKGKen3C5QubYy0vmq+JGitUg92uuywGEwIA==", + "dev": true, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://mirrors.tencent.com/npm/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://mirrors.tencent.com/npm/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/less": { + "version": "4.3.0", + "resolved": "https://mirrors.tencent.com/npm/less/-/less-4.3.0.tgz", + "integrity": "sha512-X9RyH9fvemArzfdP8Pi3irr7lor2Ok4rOttDXBhlwDg+wKQsXOXgHWduAJE1EsF7JJx0w0bcO6BC6tCKKYnXKA==", + "dev": true, + "dependencies": { + "copy-anything": "^2.0.1", + "parse-node-version": "^1.0.1", + "tslib": "^2.3.0" + }, + "bin": { + "lessc": "bin/lessc" + }, + "engines": { + "node": ">=14" + }, + "optionalDependencies": { + "errno": "^0.1.1", + "graceful-fs": "^4.1.2", + "image-size": "~0.5.0", + "make-dir": "^2.1.0", + "mime": "^1.4.1", + "needle": "^3.1.0", + "source-map": "~0.6.0" + } + }, + "node_modules/less-loader": { + "version": "12.3.0", + "resolved": "https://mirrors.tencent.com/npm/less-loader/-/less-loader-12.3.0.tgz", + "integrity": "sha512-0M6+uYulvYIWs52y0LqN4+QM9TqWAohYSNTo4htE8Z7Cn3G/qQMEmktfHmyJT23k+20kU9zHH2wrfFXkxNLtVw==", + "dev": true, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "less": "^3.5.0 || ^4.0.0", + "webpack": "^5.0.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/loader-runner": { + "version": "4.3.0", + "resolved": "https://mirrors.tencent.com/npm/loader-runner/-/loader-runner-4.3.0.tgz", + "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "license": "MIT", + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://mirrors.tencent.com/npm/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://mirrors.tencent.com/npm/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/magic-string": { + "version": "0.30.17", + "resolved": "https://mirrors.tencent.com/npm/magic-string/-/magic-string-0.30.17.tgz", + "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0" + } + }, + "node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://mirrors.tencent.com/npm/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "optional": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "5.7.2", + "resolved": "https://mirrors.tencent.com/npm/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "dev": true, + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/marked": { + "version": "5.1.2", + "resolved": "https://mirrors.tencent.com/npm/marked/-/marked-5.1.2.tgz", + "integrity": "sha512-ahRPGXJpjMjwSOlBoTMZAK7ATXkli5qCPxZ21TG44rx1KEo44bii4ekgTDQPNRQ4Kh7JMb9Ub1PVk1NxRSsorg==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 16" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://mirrors.tencent.com/npm/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/memorystream": { + "version": "0.3.1", + "resolved": "https://mirrors.tencent.com/npm/memorystream/-/memorystream-0.3.1.tgz", + "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==", + "dev": true, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://mirrors.tencent.com/npm/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://mirrors.tencent.com/npm/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "license": "MIT", + "optional": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://mirrors.tencent.com/npm/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://mirrors.tencent.com/npm/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://mirrors.tencent.com/npm/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://mirrors.tencent.com/npm/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://mirrors.tencent.com/npm/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/muggle-string": { + "version": "0.4.1", + "resolved": "https://mirrors.tencent.com/npm/muggle-string/-/muggle-string-0.4.1.tgz", + "integrity": "sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://mirrors.tencent.com/npm/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/needle": { + "version": "3.3.1", + "resolved": "https://mirrors.tencent.com/npm/needle/-/needle-3.3.1.tgz", + "integrity": "sha512-6k0YULvhpw+RoLNiQCRKOl09Rv1dPLr8hHnVjHqdolKwDrdNyk+Hmrthi4lIGPPz3r39dLx0hsF5s40sZ3Us4Q==", + "dev": true, + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.3", + "sax": "^1.2.4" + }, + "bin": { + "needle": "bin/needle" + }, + "engines": { + "node": ">= 4.4.x" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://mirrors.tencent.com/npm/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://mirrors.tencent.com/npm/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "license": "MIT" + }, + "node_modules/npm-normalize-package-bin": { + "version": "4.0.0", + "resolved": "https://mirrors.tencent.com/npm/npm-normalize-package-bin/-/npm-normalize-package-bin-4.0.0.tgz", + "integrity": "sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm-run-all2": { + "version": "7.0.2", + "resolved": "https://mirrors.tencent.com/npm/npm-run-all2/-/npm-run-all2-7.0.2.tgz", + "integrity": "sha512-7tXR+r9hzRNOPNTvXegM+QzCuMjzUIIq66VDunL6j60O4RrExx32XUhlrS7UK4VcdGw5/Wxzb3kfNcFix9JKDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "cross-spawn": "^7.0.6", + "memorystream": "^0.3.1", + "minimatch": "^9.0.0", + "pidtree": "^0.6.0", + "read-package-json-fast": "^4.0.0", + "shell-quote": "^1.7.3", + "which": "^5.0.0" + }, + "bin": { + "npm-run-all": "bin/npm-run-all/index.js", + "npm-run-all2": "bin/npm-run-all/index.js", + "run-p": "bin/run-p/index.js", + "run-s": "bin/run-s/index.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0", + "npm": ">= 9" + } + }, + "node_modules/pagefind": { + "version": "1.3.0", + "resolved": "https://mirrors.tencent.com/npm/pagefind/-/pagefind-1.3.0.tgz", + "integrity": "sha512-8KPLGT5g9s+olKMRTU9LFekLizkVIu9tes90O1/aigJ0T5LmyPqTzGJrETnSw3meSYg58YH7JTzhTTW/3z6VAw==", + "license": "MIT", + "bin": { + "pagefind": "lib/runner/bin.cjs" + }, + "optionalDependencies": { + "@pagefind/darwin-arm64": "1.3.0", + "@pagefind/darwin-x64": "1.3.0", + "@pagefind/linux-arm64": "1.3.0", + "@pagefind/linux-x64": "1.3.0", + "@pagefind/windows-x64": "1.3.0" + } + }, + "node_modules/parse-node-version": { + "version": "1.0.1", + "resolved": "https://mirrors.tencent.com/npm/parse-node-version/-/parse-node-version-1.0.1.tgz", + "integrity": "sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://mirrors.tencent.com/npm/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", + "dev": true + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://mirrors.tencent.com/npm/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/perfect-debounce": { + "version": "1.0.0", + "resolved": "https://mirrors.tencent.com/npm/perfect-debounce/-/perfect-debounce-1.0.0.tgz", + "integrity": "sha512-xCy9V055GLEqoFaHoC1SoLIaLmWctgCUaBaWxDZ7/Zx4CTyX7cJQLJOok/orfjZAh9kEYpjJa4d0KcJmCbctZA==" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://mirrors.tencent.com/npm/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://mirrors.tencent.com/npm/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pidtree": { + "version": "0.6.0", + "resolved": "https://mirrors.tencent.com/npm/pidtree/-/pidtree-0.6.0.tgz", + "integrity": "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g==", + "dev": true, + "license": "MIT", + "bin": { + "pidtree": "bin/pidtree.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://mirrors.tencent.com/npm/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "optional": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pinia": { + "version": "3.0.3", + "resolved": "https://mirrors.tencent.com/npm/pinia/-/pinia-3.0.3.tgz", + "integrity": "sha512-ttXO/InUULUXkMHpTdp9Fj4hLpD/2AoJdmAbAeW2yu1iy1k+pkFekQXw5VpC0/5p51IOR/jDaDRfRWRnMMsGOA==", + "license": "MIT", + "dependencies": { + "@vue/devtools-api": "^7.7.2" + }, + "funding": { + "url": "https://github.com/sponsors/posva" + }, + "peerDependencies": { + "typescript": ">=4.4.4", + "vue": "^2.7.0 || ^3.5.11" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://mirrors.tencent.com/npm/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://mirrors.tencent.com/npm/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://mirrors.tencent.com/npm/prr/-/prr-1.0.1.tgz", + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true, + "optional": true + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://mirrors.tencent.com/npm/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/read-package-json-fast": { + "version": "4.0.0", + "resolved": "https://mirrors.tencent.com/npm/read-package-json-fast/-/read-package-json-fast-4.0.0.tgz", + "integrity": "sha512-qpt8EwugBWDw2cgE2W+/3oxC+KTez2uSVR8JU9Q36TXPAGCaozfQUs59v4j4GFpWTaw0i6hAZSvOmu1J0uOEUg==", + "dev": true, + "dependencies": { + "json-parse-even-better-errors": "^4.0.0", + "npm-normalize-package-bin": "^4.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://mirrors.tencent.com/npm/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rfdc": { + "version": "1.4.1", + "resolved": "https://mirrors.tencent.com/npm/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", + "license": "MIT" + }, + "node_modules/rollup": { + "version": "4.45.0", + "resolved": "https://mirrors.tencent.com/npm/rollup/-/rollup-4.45.0.tgz", + "integrity": "sha512-WLjEcJRIo7i3WDDgOIJqVI2d+lAC3EwvOGy+Xfq6hs+GQuAA4Di/H72xmXkOhrIWFg2PFYSKZYfH0f4vfKXN4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.45.0", + "@rollup/rollup-android-arm64": "4.45.0", + "@rollup/rollup-darwin-arm64": "4.45.0", + "@rollup/rollup-darwin-x64": "4.45.0", + "@rollup/rollup-freebsd-arm64": "4.45.0", + "@rollup/rollup-freebsd-x64": "4.45.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.45.0", + "@rollup/rollup-linux-arm-musleabihf": "4.45.0", + "@rollup/rollup-linux-arm64-gnu": "4.45.0", + "@rollup/rollup-linux-arm64-musl": "4.45.0", + "@rollup/rollup-linux-loongarch64-gnu": "4.45.0", + "@rollup/rollup-linux-powerpc64le-gnu": "4.45.0", + "@rollup/rollup-linux-riscv64-gnu": "4.45.0", + "@rollup/rollup-linux-riscv64-musl": "4.45.0", + "@rollup/rollup-linux-s390x-gnu": "4.45.0", + "@rollup/rollup-linux-x64-gnu": "4.45.0", + "@rollup/rollup-linux-x64-musl": "4.45.0", + "@rollup/rollup-win32-arm64-msvc": "4.45.0", + "@rollup/rollup-win32-ia32-msvc": "4.45.0", + "@rollup/rollup-win32-x64-msvc": "4.45.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://mirrors.tencent.com/npm/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://mirrors.tencent.com/npm/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/sax": { + "version": "1.4.1", + "resolved": "https://mirrors.tencent.com/npm/sax/-/sax-1.4.1.tgz", + "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==", + "dev": true, + "license": "ISC", + "optional": true + }, + "node_modules/schema-utils": { + "version": "4.3.2", + "resolved": "https://mirrors.tencent.com/npm/schema-utils/-/schema-utils-4.3.2.tgz", + "integrity": "sha512-Gn/JaSk/Mt9gYubxTtSn/QCV4em9mpAPiR1rqy/Ocu19u/G9J5WWdNoUT4SiV6mFC3y6cxyFcFwdzPM3FgxGAQ==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://mirrors.tencent.com/npm/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/serialize-javascript": { + "version": "6.0.2", + "resolved": "https://mirrors.tencent.com/npm/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://mirrors.tencent.com/npm/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://mirrors.tencent.com/npm/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.3", + "resolved": "https://mirrors.tencent.com/npm/shell-quote/-/shell-quote-1.8.3.tgz", + "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/sortablejs": { + "version": "1.15.6", + "resolved": "https://mirrors.tencent.com/npm/sortablejs/-/sortablejs-1.15.6.tgz", + "integrity": "sha512-aNfiuwMEpfBM/CN6LY0ibyhxPfPbyFeBTYJKCvzkJ2GkUpazIt3H+QIPAMHwqQ7tMKaHz1Qj+rJJCqljnf4p3A==", + "license": "MIT" + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://mirrors.tencent.com/npm/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://mirrors.tencent.com/npm/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://mirrors.tencent.com/npm/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/speakingurl": { + "version": "14.0.1", + "resolved": "https://mirrors.tencent.com/npm/speakingurl/-/speakingurl-14.0.1.tgz", + "integrity": "sha512-1POYv7uv2gXoyGFpBCmpDVSNV74IfsWlDW216UPjbWufNf+bSU6GdbDsxdcxtfwb4xlI3yxzOTKClUosxARYrQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/superjson": { + "version": "2.2.2", + "resolved": "https://mirrors.tencent.com/npm/superjson/-/superjson-2.2.2.tgz", + "integrity": "sha512-5JRxVqC8I8NuOUjzBbvVJAKNM8qoVuH0O77h4WInc/qC2q5IreqKxYwgkga3PfA22OayK2ikceb/B26dztPl+Q==", + "dependencies": { + "copy-anything": "^3.0.2" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/superjson/node_modules/copy-anything": { + "version": "3.0.5", + "resolved": "https://mirrors.tencent.com/npm/copy-anything/-/copy-anything-3.0.5.tgz", + "integrity": "sha512-yCEafptTtb4bk7GLEQoM8KVJpxAfdBJYaXyzQEgQQQgYrZiDp8SJmGKlYza6CYjEDNstAdNdKA3UuoULlEbS6w==", + "dependencies": { + "is-what": "^4.1.8" + }, + "engines": { + "node": ">=12.13" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, + "node_modules/superjson/node_modules/is-what": { + "version": "4.1.16", + "resolved": "https://mirrors.tencent.com/npm/is-what/-/is-what-4.1.16.tgz", + "integrity": "sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==", + "license": "MIT", + "engines": { + "node": ">=12.13" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, + "node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://mirrors.tencent.com/npm/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tapable": { + "version": "2.2.2", + "resolved": "https://mirrors.tencent.com/npm/tapable/-/tapable-2.2.2.tgz", + "integrity": "sha512-Re10+NauLTMCudc7T5WLFLAwDhQ0JWdrMK+9B2M8zR5hRExKmsRDCBA7/aV/pNJFltmBFO5BAMlQFi/vq3nKOg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/tdesign-icons-vue-next": { + "version": "0.4.1", + "resolved": "https://mirrors.tencent.com/npm/tdesign-icons-vue-next/-/tdesign-icons-vue-next-0.4.1.tgz", + "integrity": "sha512-uDPuTLRORnGcTyVGNoentNaK4V+ZcBmhYwcY3KqDaQQ5rrPeLMxu0ZVmgOEf0JtF2QZiqAxY7vodNEiLUdoRKA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.16.3" + }, + "peerDependencies": { + "vue": "^3.0.0" + } + }, + "node_modules/tdesign-vue-next": { + "version": "1.14.2", + "resolved": "https://mirrors.tencent.com/npm/tdesign-vue-next/-/tdesign-vue-next-1.14.2.tgz", + "integrity": "sha512-Wcg0SjT4W7JmMhwCI4/rnk8bJzcmjed/YAemVDaOY33U8PqUsU3DxxP/vJ2gbLH7NkFgLGVNpeAdTqYdmHnV4w==", + "dependencies": { + "@babel/runtime": "^7.22.6", + "@popperjs/core": "^2.11.8", + "@types/lodash-es": "^4.17.12", + "@types/sortablejs": "^1.15.1", + "@types/tinycolor2": "^1.4.3", + "@types/validator": "^13.7.17", + "dayjs": "1.11.10", + "lodash-es": "^4.17.21", + "mitt": "^3.0.1", + "sortablejs": "^1.15.0", + "tdesign-icons-vue-next": "^0.3.6", + "tinycolor2": "^1.6.0", + "validator": "^13.9.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "vue": ">=3.1.0" + } + }, + "node_modules/tdesign-vue-next/node_modules/tdesign-icons-vue-next": { + "version": "0.3.7", + "resolved": "https://mirrors.tencent.com/npm/tdesign-icons-vue-next/-/tdesign-icons-vue-next-0.3.7.tgz", + "integrity": "sha512-Q5ebVty/TCqhBa0l/17kkhjC0pBAOGvn7C35MAt1xS+johKVM9QEDOy9R6XEl332AiwQ37MwqioczqjYC30ckw==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.16.3" + }, + "peerDependencies": { + "vue": "^3.0.0" + } + }, + "node_modules/terser": { + "version": "5.43.1", + "resolved": "https://mirrors.tencent.com/npm/terser/-/terser-5.43.1.tgz", + "integrity": "sha512-+6erLbBm0+LROX2sPXlUYx/ux5PyE9K/a92Wrt6oA+WDAoFTdpHE5tCYCI5PNzq2y8df4rA+QgHLJuR4jNymsg==", + "license": "BSD-2-Clause", + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.14.0", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.3.14", + "resolved": "https://mirrors.tencent.com/npm/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz", + "integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.0", + "serialize-javascript": "^6.0.2", + "terser": "^5.31.1" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/tinycolor2": { + "version": "1.6.0", + "resolved": "https://mirrors.tencent.com/npm/tinycolor2/-/tinycolor2-1.6.0.tgz", + "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", + "license": "MIT" + }, + "node_modules/tinyglobby": { + "version": "0.2.14", + "resolved": "https://mirrors.tencent.com/npm/tinyglobby/-/tinyglobby-0.2.14.tgz", + "integrity": "sha512-tX5e7OM1HnYr2+a2C/4V0htOcSQcoSTH9KgJnVvNm5zm/cyEWKJ7j7YutsH9CxMdtOkkLFy2AHrMci9IM8IPZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.4.4", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://mirrors.tencent.com/npm/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true + }, + "node_modules/typescript": { + "version": "5.8.3", + "resolved": "https://mirrors.tencent.com/npm/typescript/-/typescript-5.8.3.tgz", + "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://mirrors.tencent.com/npm/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==" + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://mirrors.tencent.com/npm/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/validator": { + "version": "13.15.15", + "resolved": "https://mirrors.tencent.com/npm/validator/-/validator-13.15.15.tgz", + "integrity": "sha512-BgWVbCI72aIQy937xbawcs+hrVaN/CZ2UwutgaJ36hGqRrLNM+f5LUT/YPRbo8IV/ASeFzXszezV+y2+rq3l8A==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/vite": { + "version": "7.0.4", + "resolved": "https://mirrors.tencent.com/npm/vite/-/vite-7.0.4.tgz", + "integrity": "sha512-SkaSguuS7nnmV7mfJ8l81JGBFV7Gvzp8IzgE8A8t23+AxuNX61Q5H1Tpz5efduSN7NHC8nQXD3sKQKZAu5mNEA==", + "dev": true, + "dependencies": { + "esbuild": "^0.25.0", + "fdir": "^6.4.6", + "picomatch": "^4.0.2", + "postcss": "^8.5.6", + "rollup": "^4.40.0", + "tinyglobby": "^0.2.14" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vscode-uri": { + "version": "3.1.0", + "resolved": "https://mirrors.tencent.com/npm/vscode-uri/-/vscode-uri-3.1.0.tgz", + "integrity": "sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vue": { + "version": "3.5.17", + "resolved": "https://mirrors.tencent.com/npm/vue/-/vue-3.5.17.tgz", + "integrity": "sha512-LbHV3xPN9BeljML+Xctq4lbz2lVHCR6DtbpTf5XIO6gugpXUN49j2QQPcMj086r9+AkJ0FfUT8xjulKKBkkr9g==", + "license": "MIT", + "dependencies": { + "@vue/compiler-dom": "3.5.17", + "@vue/compiler-sfc": "3.5.17", + "@vue/runtime-dom": "3.5.17", + "@vue/server-renderer": "3.5.17", + "@vue/shared": "3.5.17" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/vue-router": { + "version": "4.5.1", + "resolved": "https://mirrors.tencent.com/npm/vue-router/-/vue-router-4.5.1.tgz", + "integrity": "sha512-ogAF3P97NPm8fJsE4by9dwSYtDwXIY1nFY9T6DyQnGHd1E2Da94w9JIolpe42LJGIl0DwOHBi8TcRPlPGwbTtw==", + "dependencies": { + "@vue/devtools-api": "^6.6.4" + }, + "funding": { + "url": "https://github.com/sponsors/posva" + }, + "peerDependencies": { + "vue": "^3.2.0" + } + }, + "node_modules/vue-router/node_modules/@vue/devtools-api": { + "version": "6.6.4", + "resolved": "https://mirrors.tencent.com/npm/@vue/devtools-api/-/devtools-api-6.6.4.tgz", + "integrity": "sha512-sGhTPMuXqZ1rVOk32RylztWkfXTRhuS7vgAKv0zjqk8gbsHkJ7xfFf+jbySxt7tWObEJwyKaHMikV/WGDiQm8g==", + "license": "MIT" + }, + "node_modules/vue-tsc": { + "version": "2.2.12", + "resolved": "https://mirrors.tencent.com/npm/vue-tsc/-/vue-tsc-2.2.12.tgz", + "integrity": "sha512-P7OP77b2h/Pmk+lZdJ0YWs+5tJ6J2+uOQPo7tlBnY44QqQSPYvS0qVT4wqDJgwrZaLe47etJLLQRFia71GYITw==", + "dev": true, + "dependencies": { + "@volar/typescript": "2.4.15", + "@vue/language-core": "2.2.12" + }, + "bin": { + "vue-tsc": "bin/vue-tsc.js" + }, + "peerDependencies": { + "typescript": ">=5.0.0" + } + }, + "node_modules/watchpack": { + "version": "2.4.4", + "resolved": "https://mirrors.tencent.com/npm/watchpack/-/watchpack-2.4.4.tgz", + "integrity": "sha512-c5EGNOiyxxV5qmTtAB7rbiXxi1ooX1pQKMLX/MIabJjRA0SJBQOjKF+KSVfHkr9U1cADPon0mRiVe/riyaiDUA==", + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/webpack": { + "version": "5.100.1", + "resolved": "https://mirrors.tencent.com/npm/webpack/-/webpack-5.100.1.tgz", + "integrity": "sha512-YJB/ESPUe2Locd0NKXmw72Dx8fZQk1gTzI6rc9TAT4+Sypbnhl8jd8RywB1bDsDF9Dy1RUR7gn3q/ZJTd0OZZg==", + "dependencies": { + "@types/eslint-scope": "^3.7.7", + "@types/estree": "^1.0.8", + "@types/json-schema": "^7.0.15", + "@webassemblyjs/ast": "^1.14.1", + "@webassemblyjs/wasm-edit": "^1.14.1", + "@webassemblyjs/wasm-parser": "^1.14.1", + "acorn": "^8.15.0", + "acorn-import-phases": "^1.0.3", + "browserslist": "^4.24.0", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.17.2", + "es-module-lexer": "^1.2.1", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.11", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^4.3.2", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.3.11", + "watchpack": "^2.4.1", + "webpack-sources": "^3.3.3" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-sources": { + "version": "3.3.3", + "resolved": "https://mirrors.tencent.com/npm/webpack-sources/-/webpack-sources-3.3.3.tgz", + "integrity": "sha512-yd1RBzSGanHkitROoPFd6qsrxt+oFhg/129YzheDGqeustzX0vTZJZsSsQjVQC4yzBQ56K55XU8gaNCtIzOnTg==", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/webpack/node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://mirrors.tencent.com/npm/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" + }, + "node_modules/which": { + "version": "5.0.0", + "resolved": "https://mirrors.tencent.com/npm/which/-/which-5.0.0.tgz", + "integrity": "sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ==", + "dev": true, + "dependencies": { + "isexe": "^3.1.1" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://mirrors.tencent.com/npm/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + } + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..10149fb --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,50 @@ +{ + "name": "knowledage-base", + "version": "0.1.3", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "build-with-types": "run-p type-check \"build-only {@}\" --", + "preview": "vite preview", + "build-only": "vite build", + "type-check": "vue-tsc --build" + }, + "dependencies": { + "@microsoft/fetch-event-source": "^2.0.1", + "@types/dompurify": "^3.0.5", + "axios": "^1.8.4", + "dompurify": "^3.2.6", + "marked": "^5.1.2", + "pagefind": "^1.1.1", + "pinia": "^3.0.1", + "tdesign-icons-vue-next": "^0.4.1", + "tdesign-vue-next": "^1.11.5", + "vue": "^3.5.13", + "vue-router": "^4.5.0", + "webpack": "^5.94.0" + }, + "devDependencies": { + "@tsconfig/node22": "^22.0.1", + "@types/marked": "^5.0.2", + "@types/node": "^22.14.0", + "@vitejs/plugin-vue": "6.0.0", + "@vitejs/plugin-vue-jsx": "5.0.1", + "@vue/tsconfig": "^0.7.0", + "less": "^4.3.0", + "less-loader": "^12.2.0", + "npm-run-all2": "^7.0.2", + "typescript": "~5.8.0", + "vite": "7.0.4", + "vue-tsc": "^2.2.8" + }, + "overrides": { + "lightningcss": "none", + "esbuild": "^0.25.0" + }, + "resolutions": { + "lightningcss": "none", + "esbuild": "^0.25.0" + } +} diff --git a/frontend/public/favicon.ico b/frontend/public/favicon.ico new file mode 100644 index 0000000..c8931ef Binary files /dev/null and b/frontend/public/favicon.ico differ diff --git a/frontend/src/App.vue b/frontend/src/App.vue new file mode 100644 index 0000000..e7d4d2b --- /dev/null +++ b/frontend/src/App.vue @@ -0,0 +1,23 @@ + + + diff --git a/frontend/src/api/auth/index.ts b/frontend/src/api/auth/index.ts new file mode 100644 index 0000000..85efb6a --- /dev/null +++ b/frontend/src/api/auth/index.ts @@ -0,0 +1,239 @@ +import { post, get, put } from '@/utils/request' + +// 用户登录接口 +export interface LoginRequest { + email: string + password: string +} + +export interface LoginResponse { + success: boolean + message?: string + user?: { + id: string + username: string + email: string + avatar?: string + tenant_id: number + is_active: boolean + created_at: string + updated_at: string + } + tenant?: { + id: number + name: string + description: string + api_key: string + status: string + business: string + storage_quota: number + storage_used: number + created_at: string + updated_at: string + } + token?: string + refresh_token?: string +} + +// 用户注册接口 +export interface RegisterRequest { + username: string + email: string + password: string +} + +export interface RegisterResponse { + success: boolean + message?: string + data?: { + user: { + id: string + username: string + email: string + } + tenant: { + id: string + name: string + api_key: string + } + } +} + +// 用户信息接口 +export interface UserInfo { + id: string + username: string + email: string + avatar?: string + tenant_id: string + created_at: string + updated_at: string +} + +// 租户信息接口 +export interface TenantInfo { + id: string + name: string + description?: string + api_key: string + status?: string + business?: string + owner_id: string + storage_quota?: number + storage_used?: number + created_at: string + updated_at: string + knowledge_bases?: KnowledgeBaseInfo[] +} + +// 知识库信息接口 +export interface KnowledgeBaseInfo { + id: string + name: string + description: string + tenant_id: string + created_at: string + updated_at: string + document_count?: number + chunk_count?: number +} + +// 模型信息接口 +export interface ModelInfo { + id: string + name: string + type: string + source: string + description?: string + is_default?: boolean + created_at: string + updated_at: string +} + +/** + * 用户登录 + */ +export async function login(data: LoginRequest): Promise { + try { + const response = await post('/api/v1/auth/login', data) + return response as unknown as LoginResponse + } catch (error: any) { + return { + success: false, + message: error.message || '登录失败' + } + } +} + +/** + * 用户注册 + */ +export async function register(data: RegisterRequest): Promise { + try { + const response = await post('/api/v1/auth/register', data) + return response as unknown as RegisterResponse + } catch (error: any) { + return { + success: false, + message: error.message || '注册失败' + } + } +} + +/** + * 获取当前用户信息 + */ +export async function getCurrentUser(): Promise<{ success: boolean; data?: { user: UserInfo; tenant: TenantInfo }; message?: string }> { + try { + const response = await get('/api/v1/auth/me') + return response as unknown as { success: boolean; data?: { user: UserInfo; tenant: TenantInfo }; message?: string } + } catch (error: any) { + return { + success: false, + message: error.message || '获取用户信息失败' + } + } +} + +/** + * 获取当前租户信息 + */ +export async function getCurrentTenant(): Promise<{ success: boolean; data?: TenantInfo; message?: string }> { + try { + const response = await get('/api/v1/auth/tenant') + return response as unknown as { success: boolean; data?: TenantInfo; message?: string } + } catch (error: any) { + return { + success: false, + message: error.message || '获取租户信息失败' + } + } +} + +/** + * 刷新Token + */ +export async function refreshToken(refreshToken: string): Promise<{ success: boolean; data?: { token: string; refreshToken: string }; message?: string }> { + try { + const response: any = await post('/api/v1/auth/refresh', { refreshToken }) + if (response && response.success) { + if (response.access_token || response.refresh_token) { + return { + success: true, + data: { + token: response.access_token, + refreshToken: response.refresh_token, + } + } + } + } + + // 其他情况直接返回原始消息 + return { + success: false, + message: response?.message || '刷新Token失败' + } + } catch (error: any) { + return { + success: false, + message: error.message || '刷新Token失败' + } + } +} + +/** + * 用户登出 + */ +export async function logout(): Promise<{ success: boolean; message?: string }> { + try { + await post('/api/v1/auth/logout', {}) + return { + success: true + } + } catch (error: any) { + return { + success: false, + message: error.message || '登出失败' + } + } +} + +/** + * 验证Token有效性 + */ +export async function validateToken(): Promise<{ success: boolean; valid?: boolean; message?: string }> { + try { + const response = await get('/api/v1/auth/validate') + return response as unknown as { success: boolean; valid?: boolean; message?: string } + } catch (error: any) { + return { + success: false, + valid: false, + message: error.message || 'Token验证失败' + } + } +} + + + + diff --git a/frontend/src/api/chat/index.ts b/frontend/src/api/chat/index.ts new file mode 100644 index 0000000..2d5fe31 --- /dev/null +++ b/frontend/src/api/chat/index.ts @@ -0,0 +1,31 @@ +import { get, post, put, del, postChat } from "../../utils/request"; + + + +export async function createSessions(data = {}) { + return post("/api/v1/sessions", data); +} + +export async function getSessionsList(page: number, page_size: number) { + return get(`/api/v1/sessions?page=${page}&page_size=${page_size}`); +} + +export async function generateSessionsTitle(session_id: string, data: any) { + return post(`/api/v1/sessions/${session_id}/generate_title`, data); +} + +export async function knowledgeChat(data: { session_id: string; query: string; }) { + return postChat(`/api/v1/knowledge-chat/${data.session_id}`, { query: data.query }); +} + +export async function getMessageList(data: { session_id: string; limit: number, created_at: string }) { + if (data.created_at) { + return get(`/api/v1/messages/${data.session_id}/load?before_time=${encodeURIComponent(data.created_at)}&limit=${data.limit}`); + } else { + return get(`/api/v1/messages/${data.session_id}/load?limit=${data.limit}`); + } +} + +export async function delSession(session_id: string) { + return del(`/api/v1/sessions/${session_id}`); +} \ No newline at end of file diff --git a/frontend/src/api/chat/streame.ts b/frontend/src/api/chat/streame.ts new file mode 100644 index 0000000..26cb490 --- /dev/null +++ b/frontend/src/api/chat/streame.ts @@ -0,0 +1,122 @@ +import { fetchEventSource } from '@microsoft/fetch-event-source' +import { ref, type Ref, onUnmounted, nextTick } from 'vue' +import { generateRandomString } from '@/utils/index'; + + + +interface StreamOptions { + // 请求方法 (默认POST) + method?: 'GET' | 'POST' + // 请求头 + headers?: Record + // 请求体自动序列化 + body?: Record + // 流式渲染间隔 (ms) + chunkInterval?: number +} + +export function useStream() { + // 响应式状态 + const output = ref('') // 显示内容 + const isStreaming = ref(false) // 流状态 + const isLoading = ref(false) // 初始加载 + const error = ref(null)// 错误信息 + let controller = new AbortController() + + // 流式渲染缓冲 + let buffer: string[] = [] + let renderTimer: number | null = null + + // 启动流式请求 + const startStream = async (params: { session_id: any; query: any; method: string; url: string }) => { + // 重置状态 + output.value = ''; + error.value = null; + isStreaming.value = true; + isLoading.value = true; + + // 获取API配置 + const apiUrl = import.meta.env.VITE_IS_DOCKER ? "" : "http://localhost:8080"; + + // 获取JWT Token + const token = localStorage.getItem('weknora_token'); + if (!token) { + error.value = "未找到登录令牌，请重新登录"; + stopStream(); + return; + } + + try { + let url = + params.method == "POST" + ? `${apiUrl}${params.url}/${params.session_id}` + : `${apiUrl}${params.url}/${params.session_id}?message_id=${params.query}`; + await fetchEventSource(url, { + method: params.method, + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${token}`, + "X-Request-ID": `${generateRandomString(12)}`, + }, + body: + params.method == "POST" + ? JSON.stringify({ query: params.query }) + : null, + signal: controller.signal, + openWhenHidden: true, + + onopen: async (res) => { + if (!res.ok) throw new Error(`HTTP ${res.status}`); + isLoading.value = false; + }, + + onmessage: (ev) => { + buffer.push(JSON.parse(ev.data)); // 数据存入缓冲 + // 执行自定义处理 + if (chunkHandler) { + chunkHandler(JSON.parse(ev.data)); + } + }, + + onerror: (err) => { + throw new Error(`流式连接失败: ${err}`); + }, + + onclose: () => { + stopStream(); + }, + }); + } catch (err) { + error.value = err instanceof Error ? err.message : String(err) + stopStream() + } + } + + let chunkHandler: ((data: any) => void) | null = null + // 注册块处理器 + const onChunk = (handler: () => void) => { + chunkHandler = handler + } + + + // 停止流 + const stopStream = () => { + controller.abort(); + controller = new AbortController(); // 重置控制器（如需重新发起） + isStreaming.value = false; + isLoading.value = false; + } + + // 组件卸载时自动清理 + onUnmounted(stopStream) + + return { + output, // 显示内容 + isStreaming, // 是否在流式传输中 + isLoading, // 初始连接状态 + error, + onChunk, + startStream, // 启动流 + stopStream // 手动停止 + } +} \ No newline at end of file diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts new file mode 100644 index 0000000..1fea566 --- /dev/null +++ b/frontend/src/api/initialization/index.ts @@ -0,0 +1,421 @@ +import { get, post } from '../../utils/request'; + +// 初始化配置数据类型 +export interface InitializationConfig { + llm: { + source: string; + modelName: string; + baseUrl?: string; + apiKey?: string; + }; + embedding: { + source: string; + modelName: string; + baseUrl?: string; + apiKey?: string; + dimension?: number; // 添加embedding维度字段 + }; + rerank: { + modelName: string; + baseUrl: string; + apiKey?: string; + enabled: boolean; + }; + multimodal: { + enabled: boolean; + storageType: 'cos' | 'minio'; + vlm?: { + modelName: string; + baseUrl: string; + apiKey?: string; + interfaceType?: string; // "ollama" or "openai" + }; + cos?: { + secretId: string; + secretKey: string; + region: string; + bucketName: string; + appId: string; + pathPrefix?: string; + }; + minio?: { + bucketName: string; + pathPrefix?: string; + }; + }; + documentSplitting: { + chunkSize: number; + chunkOverlap: number; + separators: string[]; + }; + // Frontend-only hint for storage selection UI + storageType?: 'cos' | 'minio'; + nodeExtract: { + enabled: boolean, + text: string, + tags: string[], + nodes: Node[], + relations: Relation[] + } +} + +// 下载任务状态类型 +export interface DownloadTask { + id: string; + modelName: string; + status: 'pending' | 'downloading' | 'completed' | 'failed'; + progress: number; + message: string; + startTime: string; + endTime?: string; +} + +// 根据知识库ID执行配置更新 +export function initializeSystemByKB(kbId: string, config: InitializationConfig): Promise { + return new Promise((resolve, reject) => { + console.log('开始知识库配置更新...', kbId, config); + post(`/api/v1/initialization/initialize/${kbId}`, config) + .then((response: any) => { + console.log('知识库配置更新完成', response); + resolve(response); + }) + .catch((error: any) => { + console.error('知识库配置更新失败:', error); + reject(error.error || error); + }); + }); +} + +// 检查Ollama服务状态 +export function checkOllamaStatus(): Promise<{ available: boolean; version?: string; error?: string; baseUrl?: string }> { + return new Promise((resolve, reject) => { + get('/api/v1/initialization/ollama/status') + .then((response: any) => { + resolve(response.data || { available: false }); + }) + .catch((error: any) => { + console.error('检查Ollama状态失败:', error); + resolve({ available: false, error: error.message || '检查失败' }); + }); + }); +} + +// 列出已安装的 Ollama 模型 +export function listOllamaModels(): Promise { + return new Promise((resolve, reject) => { + get('/api/v1/initialization/ollama/models') + .then((response: any) => { + resolve((response.data && response.data.models) || []); + }) + .catch((error: any) => { + console.error('获取 Ollama 模型列表失败:', error); + resolve([]); + }); + }); +} + +// 检查Ollama模型状态 +export function checkOllamaModels(models: string[]): Promise<{ models: Record }> { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/ollama/models/check', { models }) + .then((response: any) => { + resolve(response.data || { models: {} }); + }) + .catch((error: any) => { + console.error('检查Ollama模型状态失败:', error); + reject(error); + }); + }); +} + +// 启动Ollama模型下载（异步） +export function downloadOllamaModel(modelName: string): Promise<{ taskId: string; modelName: string; status: string; progress: number }> { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/ollama/models/download', { modelName }) + .then((response: any) => { + resolve(response.data || { taskId: '', modelName, status: 'failed', progress: 0 }); + }) + .catch((error: any) => { + console.error('启动Ollama模型下载失败:', error); + reject(error); + }); + }); +} + +// 查询下载进度 +export function getDownloadProgress(taskId: string): Promise { + return new Promise((resolve, reject) => { + get(`/api/v1/initialization/ollama/download/progress/${taskId}`) + .then((response: any) => { + resolve(response.data); + }) + .catch((error: any) => { + console.error('查询下载进度失败:', error); + reject(error); + }); + }); +} + +// 获取所有下载任务 +export function listDownloadTasks(): Promise { + return new Promise((resolve, reject) => { + get('/api/v1/initialization/ollama/download/tasks') + .then((response: any) => { + resolve(response.data || []); + }) + .catch((error: any) => { + console.error('获取下载任务列表失败:', error); + reject(error); + }); + }); +} + + +export function getCurrentConfigByKB(kbId: string): Promise { + return new Promise((resolve, reject) => { + get(`/api/v1/initialization/config/${kbId}`) + .then((response: any) => { + resolve(response.data || {}); + }) + .catch((error: any) => { + console.error('获取知识库配置失败:', error); + reject(error); + }); + }); +} + +// 检查远程API模型 +export function checkRemoteModel(modelConfig: { + modelName: string; + baseUrl: string; + apiKey?: string; +}): Promise<{ + available: boolean; + message?: string; +}> { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/remote/check', modelConfig) + .then((response: any) => { + resolve(response.data || {}); + }) + .catch((error: any) => { + console.error('检查远程模型失败:', error); + reject(error); + }); + }); +} + +// 测试 Embedding 模型（本地/远程）是否可用 +export function testEmbeddingModel(modelConfig: { + source: 'local' | 'remote'; + modelName: string; + baseUrl?: string; + apiKey?: string; + dimension?: number; +}): Promise<{ available: boolean; message?: string; dimension?: number }> { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/embedding/test', modelConfig) + .then((response: any) => { + resolve(response.data || {}); + }) + .catch((error: any) => { + console.error('测试Embedding模型失败:', error); + reject(error); + }); + }); +} + + +export function checkRerankModel(modelConfig: { + modelName: string; + baseUrl: string; + apiKey?: string; +}): Promise<{ + available: boolean; + message?: string; +}> { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/rerank/check', modelConfig) + .then((response: any) => { + resolve(response.data || {}); + }) + .catch((error: any) => { + console.error('检查Rerank模型失败:', error); + reject(error); + }); + }); +} + +export function testMultimodalFunction(testData: { + image: File; + vlm_model: string; + vlm_base_url: string; + vlm_api_key?: string; + vlm_interface_type?: string; + storage_type?: 'cos'|'minio'; + // COS optional fields (required only when storage_type === 'cos') + cos_secret_id?: string; + cos_secret_key?: string; + cos_region?: string; + cos_bucket_name?: string; + cos_app_id?: string; + cos_path_prefix?: string; + // MinIO optional fields + minio_bucket_name?: string; + minio_path_prefix?: string; + chunk_size: number; + chunk_overlap: number; + separators: string[]; +}): Promise<{ + success: boolean; + caption?: string; + ocr?: string; + processing_time?: number; + message?: string; +}> { + return new Promise((resolve, reject) => { + const formData = new FormData(); + formData.append('image', testData.image); + formData.append('vlm_model', testData.vlm_model); + formData.append('vlm_base_url', testData.vlm_base_url); + if (testData.vlm_api_key) { + formData.append('vlm_api_key', testData.vlm_api_key); + } + if (testData.vlm_interface_type) { + formData.append('vlm_interface_type', testData.vlm_interface_type); + } + if (testData.storage_type) { + formData.append('storage_type', testData.storage_type); + } + // Append COS fields only when storage_type is COS + if (testData.storage_type === 'cos') { + if (testData.cos_secret_id) formData.append('cos_secret_id', testData.cos_secret_id); + if (testData.cos_secret_key) formData.append('cos_secret_key', testData.cos_secret_key); + if (testData.cos_region) formData.append('cos_region', testData.cos_region); + if (testData.cos_bucket_name) formData.append('cos_bucket_name', testData.cos_bucket_name); + if (testData.cos_app_id) formData.append('cos_app_id', testData.cos_app_id); + if (testData.cos_path_prefix) formData.append('cos_path_prefix', testData.cos_path_prefix); + } + // MinIO fields + if (testData.minio_bucket_name) formData.append('minio_bucket_name', testData.minio_bucket_name); + if (testData.minio_path_prefix) formData.append('minio_path_prefix', testData.minio_path_prefix); + formData.append('chunk_size', testData.chunk_size.toString()); + formData.append('chunk_overlap', testData.chunk_overlap.toString()); + formData.append('separators', JSON.stringify(testData.separators)); + + // 获取鉴权Token + const token = localStorage.getItem('weknora_token'); + const headers: Record = {}; + if (token) { + headers['Authorization'] = `Bearer ${token}`; + } + + // 使用原生fetch因为需要发送FormData + fetch('/api/v1/initialization/multimodal/test', { + method: 'POST', + headers, + body: formData + }) + .then(response => response.json()) + .then((data: any) => { + if (data.success) { + resolve(data.data || {}); + } else { + resolve({ success: false, message: data.message || '测试失败' }); + } + }) + .catch((error: any) => { + console.error('多模态测试失败:', error); + reject(error); + }); + }); +} + +// 文本内容关系提取接口 +export interface TextRelationExtractionRequest { + text: string; + tags: string[]; + llmConfig: LLMConfig; +} + +export interface Node { + name: string; + attributes: string[]; +} + +export interface Relation { + node1: string; + node2: string; + type: string; +} + +export interface LLMConfig { + source: 'local' | 'remote'; + modelName: string; + baseUrl: string; + apiKey: string; +} + +export interface TextRelationExtractionResponse { + nodes: Node[]; + relations: Relation[]; +} + +// 文本内容关系提取 +export function extractTextRelations(request: TextRelationExtractionRequest): Promise { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/extract/text-relation', request) + .then((response: any) => { + resolve(response.data || { nodes: [], relations: [] }); + }) + .catch((error: any) => { + console.error('文本内容关系提取失败:', error); + reject(error); + }); + }); +} + +export interface FabriTextRequest { + tags: string[]; + llmConfig: LLMConfig; +} + +export interface FabriTextResponse { + text: string; +} + +// 文本内容生成 +export function fabriText(request: FabriTextRequest): Promise { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/extract/fabri-text', request) + .then((response: any) => { + resolve(response.data || { text: '' }); + }) + .catch((error: any) => { + console.error('文本内容生成失败:', error); + reject(error); + }); + }); +} + +export interface FabriTagRequest { + llmConfig: LLMConfig; +} + +export interface FabriTagResponse { + tags: string[]; +} + +// 文本内容生成 +export function fabriTag(request: FabriTagRequest): Promise { + return new Promise((resolve, reject) => { + post('/api/v1/initialization/extract/fabri-tag', request) + .then((response: any) => { + resolve(response.data || { tags: [] as string[] }); + }) + .catch((error: any) => { + console.error('标签生成失败:', error); + reject(error); + }); + }); +} \ No newline at end of file diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts new file mode 100644 index 0000000..e507d33 --- /dev/null +++ b/frontend/src/api/knowledge-base/index.ts @@ -0,0 +1,55 @@ +import { get, post, put, del, postUpload, getDown } from "../../utils/request"; + +// 知识库管理 API（列表、创建、获取、更新、删除、复制） +export function listKnowledgeBases() { + return get(`/api/v1/knowledge-bases`); +} + +export function createKnowledgeBase(data: { name: string; description?: string; chunking_config?: any }) { + return post(`/api/v1/knowledge-bases`, data); +} + +export function getKnowledgeBaseById(id: string) { + return get(`/api/v1/knowledge-bases/${id}`); +} + +export function updateKnowledgeBase(id: string, data: { name: string; description?: string; config: any }) { + return put(`/api/v1/knowledge-bases/${id}` , data); +} + +export function deleteKnowledgeBase(id: string) { + return del(`/api/v1/knowledge-bases/${id}`); +} + +export function copyKnowledgeBase(data: { source_id: string; target_id?: string }) { + return post(`/api/v1/knowledge-bases/copy`, data); +} + +// 知识文件 API（基于具体知识库） +export function uploadKnowledgeFile(kbId: string, data = {}) { + return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, data); +} + +export function listKnowledgeFiles(kbId: string, { page, page_size }: { page: number; page_size: number }) { + return get(`/api/v1/knowledge-bases/${kbId}/knowledge?page=${page}&page_size=${page_size}`); +} + +export function getKnowledgeDetails(id: string) { + return get(`/api/v1/knowledge/${id}`); +} + +export function delKnowledgeDetails(id: string) { + return del(`/api/v1/knowledge/${id}`); +} + +export function downKnowledgeDetails(id: string) { + return getDown(`/api/v1/knowledge/${id}/download`); +} + +export function batchQueryKnowledge(idsQueryString: string) { + return get(`/api/v1/knowledge/batch?${idsQueryString}`); +} + +export function getKnowledgeDetailsCon(id: string, page: number) { + return get(`/api/v1/chunks/${id}?page=${page}&page_size=25`); +} \ No newline at end of file diff --git a/frontend/src/api/system/index.ts b/frontend/src/api/system/index.ts new file mode 100644 index 0000000..4101101 --- /dev/null +++ b/frontend/src/api/system/index.ts @@ -0,0 +1,12 @@ +import { get } from '@/utils/request' + +export interface SystemInfo { + version: string + commit_id?: string + build_time?: string + go_version?: string +} + +export function getSystemInfo(): Promise<{ data: SystemInfo }> { + return get('/api/v1/system/info') +} diff --git a/frontend/src/assets/fonts.css b/frontend/src/assets/fonts.css new file mode 100644 index 0000000..3a52489 --- /dev/null +++ b/frontend/src/assets/fonts.css @@ -0,0 +1,6 @@ +@font-face { + font-family: 'TencentSans'; + src: url('fonts/TencentSans.ttf') format('truetype'); + font-weight: normal; + font-style: normal; +} \ No newline at end of file diff --git a/frontend/src/assets/fonts/TencentSans.ttf b/frontend/src/assets/fonts/TencentSans.ttf new file mode 100644 index 0000000..386dd20 Binary files /dev/null and b/frontend/src/assets/fonts/TencentSans.ttf differ diff --git a/frontend/src/assets/img/Frame3718.svg b/frontend/src/assets/img/Frame3718.svg new file mode 100644 index 0000000..16de2d9 --- /dev/null +++ b/frontend/src/assets/img/Frame3718.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/botanswer_loading.gif b/frontend/src/assets/img/botanswer_loading.gif new file mode 100644 index 0000000..3d1b9cb Binary files /dev/null and b/frontend/src/assets/img/botanswer_loading.gif differ diff --git a/frontend/src/assets/img/circle.png b/frontend/src/assets/img/circle.png new file mode 100644 index 0000000..042603f Binary files /dev/null and b/frontend/src/assets/img/circle.png differ diff --git a/frontend/src/assets/img/delete.svg b/frontend/src/assets/img/delete.svg new file mode 100644 index 0000000..fe00ae5 --- /dev/null +++ b/frontend/src/assets/img/delete.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/download.svg b/frontend/src/assets/img/download.svg new file mode 100644 index 0000000..20ce796 --- /dev/null +++ b/frontend/src/assets/img/download.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/file-add-green.svg b/frontend/src/assets/img/file-add-green.svg new file mode 100644 index 0000000..6fe63f7 --- /dev/null +++ b/frontend/src/assets/img/file-add-green.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/file-add-icon.svg b/frontend/src/assets/img/file-add-icon.svg new file mode 100644 index 0000000..2326060 --- /dev/null +++ b/frontend/src/assets/img/file-add-icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/src/assets/img/file-add.svg b/frontend/src/assets/img/file-add.svg new file mode 100644 index 0000000..c8b3456 --- /dev/null +++ b/frontend/src/assets/img/file-add.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/logout.svg b/frontend/src/assets/img/logout.svg new file mode 100644 index 0000000..710d939 --- /dev/null +++ b/frontend/src/assets/img/logout.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/frontend/src/assets/img/more.png b/frontend/src/assets/img/more.png new file mode 100644 index 0000000..d6f3654 Binary files /dev/null and b/frontend/src/assets/img/more.png differ diff --git a/frontend/src/assets/img/prefixIcon-green.svg b/frontend/src/assets/img/prefixIcon-green.svg new file mode 100644 index 0000000..b83c674 --- /dev/null +++ b/frontend/src/assets/img/prefixIcon-green.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/prefixIcon-grey.svg b/frontend/src/assets/img/prefixIcon-grey.svg new file mode 100644 index 0000000..af434f5 --- /dev/null +++ b/frontend/src/assets/img/prefixIcon-grey.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/prefixIcon.svg b/frontend/src/assets/img/prefixIcon.svg new file mode 100644 index 0000000..e38907f --- /dev/null +++ b/frontend/src/assets/img/prefixIcon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/sending-aircraft.svg b/frontend/src/assets/img/sending-aircraft.svg new file mode 100644 index 0000000..077247c --- /dev/null +++ b/frontend/src/assets/img/sending-aircraft.svg @@ -0,0 +1,3 @@ + + + diff --git a/frontend/src/assets/img/setting-green.svg b/frontend/src/assets/img/setting-green.svg new file mode 100644 index 0000000..7148887 --- /dev/null +++ b/frontend/src/assets/img/setting-green.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/setting.svg b/frontend/src/assets/img/setting.svg new file mode 100644 index 0000000..e0f5b33 --- /dev/null +++ b/frontend/src/assets/img/setting.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/assets/img/think.gif b/frontend/src/assets/img/think.gif new file mode 100644 index 0000000..02cadce Binary files /dev/null and b/frontend/src/assets/img/think.gif differ diff --git a/frontend/src/assets/img/upload-mask.svg b/frontend/src/assets/img/upload-mask.svg new file mode 100644 index 0000000..15c8bc1 --- /dev/null +++ b/frontend/src/assets/img/upload-mask.svg @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/assets/img/upload.svg b/frontend/src/assets/img/upload.svg new file mode 100644 index 0000000..2bb5f61 --- /dev/null +++ b/frontend/src/assets/img/upload.svg @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/assets/img/user-green.svg b/frontend/src/assets/img/user-green.svg new file mode 100644 index 0000000..6685b1b --- /dev/null +++ b/frontend/src/assets/img/user-green.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/src/assets/img/user.svg b/frontend/src/assets/img/user.svg new file mode 100644 index 0000000..1e26d1e --- /dev/null +++ b/frontend/src/assets/img/user.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/src/assets/img/weknora.png b/frontend/src/assets/img/weknora.png new file mode 100644 index 0000000..23a2cc4 Binary files /dev/null and b/frontend/src/assets/img/weknora.png differ diff --git a/frontend/src/assets/img/zhishiku-green.svg b/frontend/src/assets/img/zhishiku-green.svg new file mode 100644 index 0000000..c61f850 --- /dev/null +++ b/frontend/src/assets/img/zhishiku-green.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/src/assets/img/zhishiku.svg b/frontend/src/assets/img/zhishiku.svg new file mode 100644 index 0000000..a590d03 --- /dev/null +++ b/frontend/src/assets/img/zhishiku.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/src/assets/img/ziliao.svg b/frontend/src/assets/img/ziliao.svg new file mode 100644 index 0000000..b1338f3 --- /dev/null +++ b/frontend/src/assets/img/ziliao.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/src/assets/theme/theme.css b/frontend/src/assets/theme/theme.css new file mode 100644 index 0000000..c032b32 --- /dev/null +++ b/frontend/src/assets/theme/theme.css @@ -0,0 +1,95 @@ +:root,:root[theme-mode="light"]{ --brand-main: var(--td-brand-color-4); --td-brand-color-light: var(--td-brand-color-1); --td-brand-color-focus: var(--td-brand-color-2); --td-brand-color-disabled: var(--td-brand-color-3); --td-brand-color-hover: var(--td-brand-color-3); --td-brand-color: var(--td-brand-color-4); --td-brand-color-active:var(--td-brand-color-5); --td-brand-color-1: #e9f8ec; --td-brand-color-2: #09f479; --td-brand-color-3: #08dd6e; --td-brand-color-4: #07c05f; --td-brand-color-5: #06b04d; --td-brand-color-6: #049b38; --td-brand-color-7: #038626; --td-brand-color-8: #027218; --td-brand-color-9: #015e0d; --td-brand-color-10: #004b05; --td-warning-color-1: #fef3e6;--td-warning-color-2: #f9e0c7;--td-warning-color-3: #f7c797;--td-warning-color-4: #f2995f;--td-warning-color-5: #ed7b2f;--td-warning-color-6: #d35a21;--td-warning-color-7: #ba431b;--td-warning-color-8: #9e3610;--td-warning-color-9: #842b0b;--td-warning-color-10: #5a1907;--td-warning-color: var(--td-warning-color-5);--td-warning-color-hover: var(--td-warning-color-4);--td-warning-color-focus: var(--td-warning-color-2);--td-warning-color-active: var(--td-warning-color-6);--td-warning-color-disabled: var(--td-warning-color-3);--td-warning-color-light: var(--td-warning-color-1); --td-error-color-1: #fdecee;--td-error-color-2: #f9d7d9;--td-error-color-3: #f8b9be;--td-error-color-4: #f78d94;--td-error-color-5: #f36d78;--td-error-color-6: #e34d59;--td-error-color-7: #c9353f;--td-error-color-8: #b11f26;--td-error-color-9: #951114;--td-error-color-10: #680506;--td-error-color: var(--td-error-color-6);--td-error-color-hover: var(--td-error-color-5);--td-error-color-focus: var(--td-error-color-2);--td-error-color-active: var(--td-error-color-7);--td-error-color-disabled: var(--td-error-color-3);--td-error-color-light: var(--td-error-color-1); --td-success-color-1: #e8f8f2;--td-success-color-2: #bcebdc;--td-success-color-3: #85dbbe;--td-success-color-4: #48c79c;--td-success-color-5: #00a870;--td-success-color-6: #078d5c;--td-success-color-7: #067945;--td-success-color-8: #056334;--td-success-color-9: #044f2a;--td-success-color-10: #033017;--td-success-color: var(--td-success-color-5);--td-success-color-hover: var(--td-success-color-4);--td-success-color-focus: var(--td-success-color-2);--td-success-color-active: var(--td-success-color-6);--td-success-color-disabled: var(--td-success-color-3);--td-success-color-light: var(--td-success-color-1); --td-gray-color-1: #f3f3f3;--td-gray-color-2: #eee;--td-gray-color-3: #e7e7e7;--td-gray-color-4: #dcdcdc;--td-gray-color-5: #c5c5c5;--td-gray-color-6: #a6a6a6;--td-gray-color-7: #8b8b8b;--td-gray-color-8: #777;--td-gray-color-9: #5e5e5e;--td-gray-color-10: #4b4b4b;--td-gray-color-11: #383838;--td-gray-color-12: #2c2c2c;--td-gray-color-13: #242424;--td-gray-color-14: #181818;--td-bg-color-container: #fff;--td-bg-color-container-select: #fff;--td-bg-color-page: var(--td-gray-color-2);--td-bg-color-container-hover: var(--td-gray-color-1);--td-bg-color-container-active: var(--td-gray-color-3);--td-bg-color-secondarycontainer: var(--td-gray-color-1);--td-bg-color-secondarycontainer-hover: var(--td-gray-color-2);--td-bg-color-secondarycontainer-active: var(--td-gray-color-4);--td-bg-color-component: var(--td-gray-color-3);--td-bg-color-component-hover: var(--td-gray-color-4);--td-bg-color-component-active: var(--td-gray-color-6);--td-bg-color-component-disabled: var(--td-gray-color-2);--td-component-stroke: var(--td-gray-color-3);--td-component-border: var(--td-gray-color-4); --td-font-white-1: #ffffff;--td-font-white-2: rgba(255, 255, 255, 0.55);--td-font-white-3: rgba(255, 255, 255, 0.35);--td-font-white-4: rgba(255, 255, 255, 0.22);--td-font-gray-1: rgba(0, 0, 0, 0.9);--td-font-gray-2: rgba(0, 0, 0, 0.6);--td-font-gray-3: rgba(0, 0, 0, 0.4);--td-font-gray-4: rgba(0, 0, 0, 0.26);--td-text-color-primary: var(--td-font-gray-1);--td-text-color-secondary: var(--td-font-gray-2);--td-text-color-placeholder: var(--td-font-gray-3);--td-text-color-disabled: var(--td-font-gray-4);--td-text-color-anti: #fff;--td-text-color-brand: var(--td-brand-color);--td-text-color-link: var(--td-brand-color); /* 字体配置 */ --td-font-family: PingFang SC, Microsoft YaHei, Arial Regular; --td-font-family-medium: PingFang SC, Microsoft YaHei, Arial Medium; --td-font-size-link-small: 12px; --td-font-size-link-medium: 14px; --td-font-size-link-large: 16px; --td-font-size-mark-small: 12px; --td-font-size-mark-medium: 14px; --td-font-size-body-small: 12px; --td-font-size-body-medium: 14px; --td-font-size-body-large: 16px; --td-font-size-title-small: 14px; --td-font-size-title-medium: 16px; --td-font-size-title-large: 20px; --td-font-size-headline-small: 24px; --td-font-size-headline-medium: 28px; --td-font-size-headline-large: 36px; --td-font-size-display-medium: 48px; --td-font-size-display-large: 64px; --td-line-height-common: 8px; --td-line-height-link-small: calc( var(--td-font-size-link-small) + var(--td-line-height-common) ); --td-line-height-link-medium: calc( var(--td-font-size-link-medium) + var(--td-line-height-common) ); --td-line-height-link-large: calc( var(--td-font-size-link-large) + var(--td-line-height-common) ); --td-line-height-mark-small: calc( var(--td-font-size-mark-small) + var(--td-line-height-common) ); --td-line-height-mark-medium: calc( var(--td-font-size-mark-medium) + var(--td-line-height-common) ); --td-line-height-body-small: calc( var(--td-font-size-body-small) + var(--td-line-height-common) ); --td-line-height-body-medium: calc( var(--td-font-size-body-medium) + var(--td-line-height-common) ); --td-line-height-body-large: calc( var(--td-font-size-body-large) + var(--td-line-height-common) ); --td-line-height-title-small: calc( var(--td-font-size-title-small) + var(--td-line-height-common) ); --td-line-height-title-medium: calc( var(--td-font-size-title-medium) + var(--td-line-height-common) ); --td-line-height-title-large: calc( var(--td-font-size-title-medium) + var(--td-line-height-common) ); --td-line-height-headline-small: calc( var(--td-font-size-headline-small) + var(--td-line-height-common) ); --td-line-height-headline-medium: calc( var(--td-font-size-headline-medium) + var(--td-line-height-common) ); --td-line-height-headline-large: calc( var(--td-font-size-headline-large) + var(--td-line-height-common) ); --td-line-height-display-medium: calc( var(--td-font-size-display-medium) + var(--td-line-height-common) ); --td-line-height-display-large: calc( var(--td-font-size-display-large) + var(--td-line-height-common) ); --td-font-link-small: var(--td-font-size-link-small) / var(--td-line-height-link-small) var(--td-font-family); --td-font-link-medium: var(--td-font-size-link-medium) / var(--td-line-height-link-medium) var(--td-font-family); --td-font-link-large: var(--td-font-size-link-large) / var(--td-line-height-link-large) var(--td-font-family); --td-font-mark-small: 600 var(--td-font-size-mark-small) / var(--td-line-height-mark-small) var(--td-font-family); --td-font-mark-medium: 600 var(--td-font-size-mark-medium) / var(--td-line-height-mark-medium) var(--td-font-family); --td-font-body-small: var(--td-font-size-body-small) / var(--td-line-height-body-small) var(--td-font-family); --td-font-body-medium: var(--td-font-size-body-medium) / var(--td-line-height-body-medium) var(--td-font-family); --td-font-body-large: var(--td-font-size-body-large) / var(--td-line-height-body-large) var(--td-font-family); --td-font-title-small: var(--td-font-size-title-small) / var(--td-line-height-title-small) var(--td-font-family); --td-font-title-medium: var(--td-font-size-title-medium) / var(--td-line-height-title-medium) var(--td-font-family); --td-font-title-large: var(--td-font-size-title-large) / var(--td-line-height-title-large) var(--td-font-family); --td-font-headline-small: var(--td-font-size-headline-small) / var(--td-line-height-headline-small) var(--td-font-family); --td-font-headline-medium: var(--td-font-size-headline-medium) / var(--td-line-height-headline-medium) var(--td-font-family); --td-font-headline-large: var(--td-font-size-headline-large) / var(--td-line-height-headline-large) var(--td-font-family); --td-font-display-medium: var(--td-font-size-display-medium) / var(--td-line-height-display-medium) var(--td-font-family); --td-font-display-large: var(--td-font-size-display-large) / var(--td-line-height-display-large) var(--td-font-family); /* 字体颜色 */ --td-text-color-primary: var(--td-font-gray-1); --td-text-color-secondary: var(--td-font-gray-2); --td-text-color-placeholder: var(--td-font-gray-3); --td-text-color-disabled: var(--td-font-gray-4); --td-text-color-anti: #fff; --td-text-color-brand: var(--td-brand-color); --td-text-color-link: var(--td-brand-color); /* end 字体配置 */ /* 圆角配置 */ --td-radius-small: 2px; --td-radius-default: 3px; --td-radius-medium: 6px; --td-radius-large: 9px; --td-radius-extraLarge: 12px; --td-radius-round: 999px; --td-radius-circle: 50%; /* end 圆角配置 */ /* 阴影配置 */ --td-shadow-1: 0px 1px 10px rgba(0, 0, 0, 0.05), 0px 4px 5px rgba(0, 0, 0, 0.08), 0px 2px 4px -1px rgba(0, 0, 0, 0.12); --td-shadow-2: 0px 3px 14px 2px rgba(0, 0, 0, 0.05), 0px 8px 10px 1px rgba(0, 0, 0, 0.06), 0px 5px 5px -3px rgba(0, 0, 0, 0.1); --td-shadow-3: 0px 6px 30px 5px rgba(0, 0, 0, 0.05), 0px 16px 24px 2px rgba(0, 0, 0, 0.04), 0px 8px 10px -5px rgba(0, 0, 0, 0.08); /* end 阴影配置 */ } +:root[theme-mode="dark"]{ + --brand-main: var(--td-brand-color-6); + --td-brand-color-light: var(--td-brand-color-1); + --td-brand-color-focus: var(--td-brand-color-2); + --td-brand-color-disabled: var(--td-brand-color-3); + --td-brand-color-hover: var(--td-brand-color-5); + --td-brand-color: var(--td-brand-color-6); + --td-brand-color-active:var(--td-brand-color-7); + --td-brand-color-1: #06b04d20; + --td-brand-color-2: #015e0d; + --td-brand-color-3: #027218; + --td-brand-color-4: #038626; + --td-brand-color-5: #049b38; + --td-brand-color-6: #06b04d; + --td-brand-color-7: #07c05f; + --td-brand-color-8: #08dd6e; + --td-brand-color-9: #09f479; + --td-brand-color-10: #a6fccf; + --td-warning-color-1: #4f2a1d; +--td-warning-color-2: #582f21; +--td-warning-color-3: #733c23; +--td-warning-color-4: #a75d2b; +--td-warning-color-5: #cf6e2d; +--td-warning-color-6: #dc7633; +--td-warning-color-7: #e8935c; +--td-warning-color-8: #ecbf91; +--td-warning-color-9: #eed7bf; +--td-warning-color-10: #f3e9dc; --td-error-color-1: #472324; +--td-error-color-2: #5e2a2d; +--td-error-color-3: #703439; +--td-error-color-4: #83383e; +--td-error-color-5: #a03f46; +--td-error-color-6: #c64751; +--td-error-color-7: #de6670; +--td-error-color-8: #ec888e; +--td-error-color-9: #edb1b6; +--td-error-color-10: #eeced0; --td-success-color-1: #193a2a; +--td-success-color-2: #1a4230; +--td-success-color-3: #17533d; +--td-success-color-4: #0d7a55; +--td-success-color-5: #059465; +--td-success-color-6: #43af8a; +--td-success-color-7: #46bf96; +--td-success-color-8: #80d2b6; +--td-success-color-9: #b4e1d3; +--td-success-color-10: #deede8; --td-gray-color-1: #f3f3f3; +--td-gray-color-2: #eee; +--td-gray-color-3: #e7e7e7; +--td-gray-color-4: #dcdcdc; +--td-gray-color-5: #c5c5c5; +--td-gray-color-6: #a6a6a6; +--td-gray-color-7: #8b8b8b; +--td-gray-color-8: #777; +--td-gray-color-9: #5e5e5e; +--td-gray-color-10: #4b4b4b; +--td-gray-color-11: #383838; +--td-gray-color-12: #2c2c2c; +--td-gray-color-13: #242424; +--td-gray-color-14: #181818; +--td-bg-color-page: var(--td-gray-color-14); +--td-bg-color-container: var(--td-gray-color-13); +--td-bg-color-container-hover: var(--td-gray-color-12); +--td-bg-color-container-active: var(--td-gray-color-10); +--td-bg-color-container-select: var(--td-gray-color-9); +--td-bg-color-secondarycontainer: var(--td-gray-color-12); +--td-bg-color-secondarycontainer-hover: var(--td-gray-color-11); +--td-bg-color-secondarycontainer-active: var(--td-gray-color-9); +--td-bg-color-component: var(--td-gray-color-11); +--td-bg-color-component-hover: var(--td-gray-color-10); +--td-bg-color-component-active: var(--td-gray-color-9); +--td-bg-color-component-disabled: var(--td-gray-color-12); +--td-component-stroke: var(--td-gray-color-11); +--td-component-border: var(--td-gray-color-9); --td-font-white-1: rgba(255, 255, 255, 0.9); +--td-font-white-2: rgba(255, 255, 255, 0.55); +--td-font-white-3: rgba(255, 255, 255, 0.35); +--td-font-white-4: rgba(255, 255, 255, 0.22); +--td-font-gray-1: rgba(0, 0, 0, 0.9); +--td-font-gray-2: rgba(0, 0, 0, 0.6); +--td-font-gray-3: rgba(0, 0, 0, 0.4); +--td-font-gray-4: rgba(0, 0, 0, 0.26); +--td-text-color-primary: var(--td-font-white-1); +--td-text-color-secondary: var(--td-font-white-2); +--td-text-color-placeholder: var(--td-font-white-3); +--td-text-color-disabled: var(--td-font-white-4); +--td-text-color-anti: #fff; +--td-text-color-brand: var(--td-brand-color); +--td-text-color-link: var(--td-brand-color); + + /* 字体配置 */ --td-font-family: PingFang SC, Microsoft YaHei, Arial Regular; --td-font-family-medium: PingFang SC, Microsoft YaHei, Arial Medium; --td-font-size-link-small: 12px; --td-font-size-link-medium: 14px; --td-font-size-link-large: 16px; --td-font-size-mark-small: 12px; --td-font-size-mark-medium: 14px; --td-font-size-body-small: 12px; --td-font-size-body-medium: 14px; --td-font-size-body-large: 16px; --td-font-size-title-small: 14px; --td-font-size-title-medium: 16px; --td-font-size-title-large: 20px; --td-font-size-headline-small: 24px; --td-font-size-headline-medium: 28px; --td-font-size-headline-large: 36px; --td-font-size-display-medium: 48px; --td-font-size-display-large: 64px; --td-line-height-common: 8px; --td-line-height-link-small: calc( var(--td-font-size-link-small) + var(--td-line-height-common) ); --td-line-height-link-medium: calc( var(--td-font-size-link-medium) + var(--td-line-height-common) ); --td-line-height-link-large: calc( var(--td-font-size-link-large) + var(--td-line-height-common) ); --td-line-height-mark-small: calc( var(--td-font-size-mark-small) + var(--td-line-height-common) ); --td-line-height-mark-medium: calc( var(--td-font-size-mark-medium) + var(--td-line-height-common) ); --td-line-height-body-small: calc( var(--td-font-size-body-small) + var(--td-line-height-common) ); --td-line-height-body-medium: calc( var(--td-font-size-body-medium) + var(--td-line-height-common) ); --td-line-height-body-large: calc( var(--td-font-size-body-large) + var(--td-line-height-common) ); --td-line-height-title-small: calc( var(--td-font-size-title-small) + var(--td-line-height-common) ); --td-line-height-title-medium: calc( var(--td-font-size-title-medium) + var(--td-line-height-common) ); --td-line-height-title-large: calc( var(--td-font-size-title-medium) + var(--td-line-height-common) ); --td-line-height-headline-small: calc( var(--td-font-size-headline-small) + var(--td-line-height-common) ); --td-line-height-headline-medium: calc( var(--td-font-size-headline-medium) + var(--td-line-height-common) ); --td-line-height-headline-large: calc( var(--td-font-size-headline-large) + var(--td-line-height-common) ); --td-line-height-display-medium: calc( var(--td-font-size-display-medium) + var(--td-line-height-common) ); --td-line-height-display-large: calc( var(--td-font-size-display-large) + var(--td-line-height-common) ); --td-font-link-small: var(--td-font-size-link-small) / var(--td-line-height-link-small) var(--td-font-family); --td-font-link-medium: var(--td-font-size-link-medium) / var(--td-line-height-link-medium) var(--td-font-family); --td-font-link-large: var(--td-font-size-link-large) / var(--td-line-height-link-large) var(--td-font-family); --td-font-mark-small: 600 var(--td-font-size-mark-small) / var(--td-line-height-mark-small) var(--td-font-family); --td-font-mark-medium: 600 var(--td-font-size-mark-medium) / var(--td-line-height-mark-medium) var(--td-font-family); --td-font-body-small: var(--td-font-size-body-small) / var(--td-line-height-body-small) var(--td-font-family); --td-font-body-medium: var(--td-font-size-body-medium) / var(--td-line-height-body-medium) var(--td-font-family); --td-font-body-large: var(--td-font-size-body-large) / var(--td-line-height-body-large) var(--td-font-family); --td-font-title-small: var(--td-font-size-title-small) / var(--td-line-height-title-small) var(--td-font-family); --td-font-title-medium: var(--td-font-size-title-medium) / var(--td-line-height-title-medium) var(--td-font-family); --td-font-title-large: var(--td-font-size-title-large) / var(--td-line-height-title-large) var(--td-font-family); --td-font-headline-small: var(--td-font-size-headline-small) / var(--td-line-height-headline-small) var(--td-font-family); --td-font-headline-medium: var(--td-font-size-headline-medium) / var(--td-line-height-headline-medium) var(--td-font-family); --td-font-headline-large: var(--td-font-size-headline-large) / var(--td-line-height-headline-large) var(--td-font-family); --td-font-display-medium: var(--td-font-size-display-medium) / var(--td-line-height-display-medium) var(--td-font-family); --td-font-display-large: var(--td-font-size-display-large) / var(--td-line-height-display-large) var(--td-font-family); /* 字体颜色 */ --td-text-color-primary: var(--td-font-gray-1); --td-text-color-secondary: var(--td-font-gray-2); --td-text-color-placeholder: var(--td-font-gray-3); --td-text-color-disabled: var(--td-font-gray-4); --td-text-color-anti: #fff; --td-text-color-brand: var(--td-brand-color); --td-text-color-link: var(--td-brand-color); /* end 字体配置 */ + + /* 圆角配置 */ --td-radius-small: 2px; --td-radius-default: 3px; --td-radius-medium: 6px; --td-radius-large: 9px; --td-radius-extraLarge: 12px; --td-radius-round: 999px; --td-radius-circle: 50%; /* end 圆角配置 */ + + /* 阴影配置 */ --td-shadow-1: 0px 1px 10px rgba(0, 0, 0, 0.05), 0px 4px 5px rgba(0, 0, 0, 0.08), 0px 2px 4px -1px rgba(0, 0, 0, 0.12); --td-shadow-2: 0px 3px 14px 2px rgba(0, 0, 0, 0.05), 0px 8px 10px 1px rgba(0, 0, 0, 0.06), 0px 5px 5px -3px rgba(0, 0, 0, 0.1); --td-shadow-3: 0px 6px 30px 5px rgba(0, 0, 0, 0.05), 0px 16px 24px 2px rgba(0, 0, 0, 0.04), 0px 8px 10px -5px rgba(0, 0, 0, 0.08); /* end 阴影配置 */ + } \ No newline at end of file diff --git a/frontend/src/components/Input-field.vue b/frontend/src/components/Input-field.vue new file mode 100644 index 0000000..5d45e14 --- /dev/null +++ b/frontend/src/components/Input-field.vue @@ -0,0 +1,135 @@ + + + diff --git a/frontend/src/components/css/markdown.less b/frontend/src/components/css/markdown.less new file mode 100644 index 0000000..744b851 --- /dev/null +++ b/frontend/src/components/css/markdown.less @@ -0,0 +1,426 @@ +:deep(.md-content) { + box-sizing: border-box !important; + + img { + max-width: 444px; + cursor: pointer; + } + + h1, + h2, + h3, + h4, + h5, + h6 { + margin-top: 5px; + font-weight: bold; + color: #00000099; + font-family: "PingFang SC", "Cascadia Code"; + transition: all 0.2s ease-out; + font-size: 20px; + } + + .hljs-title, + .hljs-title.class_, + .hljs-title.class_.inherited__, + .hljs-title.function_ { + white-space: pre-wrap; + word-break: break-all; + } + + .proto { + word-break: break-all; + white-space: pre-wrap; + } + + h1 tt, + h1 code { + font-size: inherit !important; + } + + h2 tt, + h2 code { + font-size: inherit !important; + } + + h3 tt, + h3 code { + font-size: inherit !important; + } + + h4 tt, + h4 code { + font-size: inherit !important; + } + + h5 tt, + h5 code { + font-size: inherit !important; + } + + h6 tt, + h6 code { + font-size: inherit !important; + } + + h2 a, + h3 a { + color: #34495e; + } + + p, + blockquote, + ul, + ol, + dl, + table { + font-size: 14px; + margin: 10px 0; + font-family: "PingFang SC", "Cascadia Code"; + } + + h2 { + font-size: 18px; + } + + h3 { + font-size: 16px; + font-weight: 500; + } + + summary { + font-size: 14px; + cursor: pointer; + } + + li>ol, + li>ul { + margin: 0 0; + } + + hr { + padding: 0; + margin: 32px 0; + border-top: 0.5rem dotted #0590ff57; + overflow: hidden; + box-sizing: content-box; + } + + body>h2:first-child { + margin-top: 0; + padding-top: 0; + } + + body>h1:first-child { + margin-top: 0; + padding-top: 0; + } + + body>h1:first-child+h2 { + margin-top: 0; + padding-top: 0; + } + + body>h3:first-child, + body>h4:first-child, + body>h5:first-child, + body>h6:first-child { + margin-top: 0; + padding-top: 0; + } + + a:first-child h1, + a:first-child h2, + a:first-child h3, + a:first-child h4, + a:first-child h5, + a:first-child h6 { + margin-top: 0; + padding-top: 0; + } + + p { + margin: 0; + } + + code { + white-space: pre-wrap; + word-break: break-all; + } + + h1 p, + h2 p, + h3 p, + h4 p, + h5 p, + h6 p { + margin-top: 0; + } + + li p.first { + display: inline-block; + } + + ul, + ol { + padding-left: 30px; + } + + ul:first-child, + ol:first-child { + margin-top: 0; + } + + ul:last-child, + ol:last-child { + margin-bottom: 0; + } + + blockquote { + padding: 0.8em 1.4rem; + margin: 1em 0; + font-weight: 400; + border-left: 4px solid #2196f3; + background-color: #2196f321; + border-radius: 0px 8px 8px 0px; + box-shadow: rgb(149 149 149 / 13%) 0px 5px 10px; + } + + table { + padding: 0; + word-break: initial; + /* border-radius: 4px; */ + border-collapse: collapse; + border-spacing: 0; + width: 100%; + } + + table tr { + border-top: 1px solid #2196f31f; + margin: 0; + padding: 0; + } + + table tr:nth-child(2n), + thead { + background-color: #fafafa; + } + + table tr th { + font-weight: bold; + border: 1px solid #9b9b9b3b; + border-bottom: 0; + text-align: left; + margin: 0; + padding: 6px 13px; + } + + table tr td { + border: 1px solid #9b9b9b3b; + text-align: left; + margin: 0; + padding: 6px 13px; + } + + table tr th:first-child, + table tr td:first-child { + margin-top: 0; + } + + table tr th:last-child, + table tr td:last-child { + margin-bottom: 0; + } + + + tt { + margin: 0 2px; + } + + figure { + border-radius: 8px; + margin-left: 0; + margin-right: 0; + background: #fff; + } + + + + .md-task-list-item>input { + margin-left: -1.3em; + } + + @media print { + html { + font-size: 13px; + } + + table, + pre { + page-break-inside: avoid; + } + + pre { + word-wrap: break-word; + } + } + + .md-fences { + background-color: #f8f8f8; + } + + .md-diagram-panel { + position: static !important; + } + + + .mathjax-block>.code-tooltip { + bottom: 0.375rem; + } + + h3.md-focus:before, + h4.md-focus:before, + h5.md-focus:before, + h6.md-focus:before { + border: 0px; + position: unset; + padding: 0px; + font-size: unset; + line-height: unset; + float: unset; + } + + .md-image>.md-meta { + border-radius: 3px; + font-family: var(--font-monospace); + padding: 2px 0 0 4px; + font-size: 0.9em; + color: inherit; + } + + .md-tag { + color: inherit; + } + + .md-toc { + margin-top: 20px; + padding-bottom: 20px; + } + + .sidebar-tabs { + border-bottom: none; + } + + + /** focus mode */ + + .on-focus-mode blockquote { + border-left-color: rgba(85, 85, 85, 0.12); + } + + header, + .context-menu, + .megamenu-content, + footer { + font-family: var(--font-sans-serif); + } + + .file-node-content:hover .file-node-icon, + .file-node-content:hover .file-node-open-state { + visibility: visible; + } + + .mac-seamless-mode #typora-sidebar { + background-color: var(--side-bar-bg-color); + } + + .md-lang { + color: #b4654d; + } + + .html-for-mac .context-menu { + --item-hover-bg-color: #e6f0fe; + } + + .pin-outline #outline-content .outline-active strong, + .pin-outline .outline-active { + color: #2196f3; + } + + .code-tooltip { + border-radius: 4px; + border: 1px solid #ededed; + background-color: #f8f8f8; + } + + .cm-s-inner .cm-comment, + .cm-s-inner.cm-comment { + color: #57a64a; + font-style: italic; + /* font-family: 'PingFang'; */ + } + + h1.md-end-block.md-heading:after, + h2.md-end-block.md-heading:after, + h3.md-end-block.md-heading:after, + h4.md-end-block.md-heading:after, + h5.md-end-block.md-heading:after, + h6.md-end-block.md-heading:after { + color: #bfbfbf !important; + border: 1px solid; + border-radius: 4px; + position: absolute; + left: -2.5rem; + float: left; + font-size: 14px; + padding-left: 4px; + padding-right: 5px; + vertical-align: bottom; + font-weight: 400; + line-height: normal; + opacity: 0; + } + + h1.md-end-block.md-heading:hover:after, + h2.md-end-block.md-heading:hover:after, + h3.md-end-block.md-heading:hover:after, + h4.md-end-block.md-heading:hover:after, + h5.md-end-block.md-heading:hover:after, + h6.md-end-block.md-heading:hover:after { + opacity: 1; + } + + h1.md-end-block.md-heading:hover:after { + content: "h1"; + top: 1.1rem; + } + + h2.md-end-block.md-heading:hover:after { + content: "h2"; + top: 0.63rem; + } + + h3.md-end-block.md-heading:hover:after { + content: "h3"; + top: 0.55rem; + } + + h4.md-end-block.md-heading:hover:after { + content: "h4"; + top: 0.3rem; + } + + h5.md-end-block.md-heading:hover:after { + content: "h5"; + top: 0.18rem; + } + + h6.md-end-block.md-heading:hover:after { + content: "h6"; + top: 0.16rem; + } + + .outline-label { + font-family: "Cascadia Code", "PingFang SC"; + } +} \ No newline at end of file diff --git a/frontend/src/components/doc-content.vue b/frontend/src/components/doc-content.vue new file mode 100644 index 0000000..6a96c34 --- /dev/null +++ b/frontend/src/components/doc-content.vue @@ -0,0 +1,263 @@ + + + diff --git a/frontend/src/components/empty-knowledge.vue b/frontend/src/components/empty-knowledge.vue new file mode 100644 index 0000000..1310c88 --- /dev/null +++ b/frontend/src/components/empty-knowledge.vue @@ -0,0 +1,42 @@ + + + diff --git a/frontend/src/components/menu.vue b/frontend/src/components/menu.vue new file mode 100644 index 0000000..b8f6f00 --- /dev/null +++ b/frontend/src/components/menu.vue @@ -0,0 +1,987 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/components/picture-preview.vue b/frontend/src/components/picture-preview.vue new file mode 100644 index 0000000..0f69c98 --- /dev/null +++ b/frontend/src/components/picture-preview.vue @@ -0,0 +1,18 @@ + + + diff --git a/frontend/src/components/upload-mask.vue b/frontend/src/components/upload-mask.vue new file mode 100644 index 0000000..a718120 --- /dev/null +++ b/frontend/src/components/upload-mask.vue @@ -0,0 +1,40 @@ + + + diff --git a/frontend/src/hooks/useKnowledgeBase.ts b/frontend/src/hooks/useKnowledgeBase.ts new file mode 100644 index 0000000..ac30701 --- /dev/null +++ b/frontend/src/hooks/useKnowledgeBase.ts @@ -0,0 +1,166 @@ +import { ref, reactive } from "vue"; +import { storeToRefs } from "pinia"; +import { formatStringDate, kbFileTypeVerification } from "../utils/index"; +import { MessagePlugin } from "tdesign-vue-next"; +import { + uploadKnowledgeFile, + listKnowledgeFiles, + getKnowledgeDetails, + delKnowledgeDetails, + getKnowledgeDetailsCon, +} from "@/api/knowledge-base/index"; +import { knowledgeStore } from "@/stores/knowledge"; +import { useRoute } from 'vue-router'; + +const usemenuStore = knowledgeStore(); +export default function (knowledgeBaseId?: string) { + const route = useRoute(); + const { cardList, total } = storeToRefs(usemenuStore); + let moreIndex = ref(-1); + const details = reactive({ + title: "", + time: "", + md: [] as any[], + id: "", + total: 0 + }); + const getKnowled = (query = { page: 1, page_size: 35 }, kbId?: string) => { + const targetKbId = kbId || knowledgeBaseId; + if (!targetKbId) return; + + listKnowledgeFiles(targetKbId, query) + .then((result: any) => { + const { data, total: totalResult } = result; + const cardList_ = data.map((item: any) => ({ + ...item, + file_name: item.file_name.substring(0, item.file_name.lastIndexOf(".")), + updated_at: formatStringDate(new Date(item.updated_at)), + isMore: false, + file_type: item.file_type.toLocaleUpperCase(), + })); + + if (query.page === 1) { + cardList.value = cardList_; + } else { + cardList.value.push(...cardList_); + } + total.value = totalResult; + }) + .catch(() => {}); + }; + const delKnowledge = (index: number, item: any) => { + cardList.value[index].isMore = false; + moreIndex.value = -1; + delKnowledgeDetails(item.id) + .then((result: any) => { + if (result.success) { + MessagePlugin.info("知识删除成功！"); + getKnowled(); + } else { + MessagePlugin.error("知识删除失败！"); + } + }) + .catch(() => { + MessagePlugin.error("知识删除失败！"); + }); + }; + const openMore = (index: number) => { + moreIndex.value = index; + }; + const onVisibleChange = (visible: boolean) => { + if (!visible) { + moreIndex.value = -1; + } + }; + const requestMethod = (file: any, uploadInput: any) => { + if (!(file instanceof File) || !uploadInput) { + MessagePlugin.error("文件类型错误！"); + return; + } + + if (kbFileTypeVerification(file)) { + return; + } + + // 获取当前知识库ID + let currentKbId: string | undefined = (route.params as any)?.kbId as string; + if (!currentKbId && typeof window !== 'undefined') { + const match = window.location.pathname.match(/knowledge-bases\/([^/]+)/); + if (match?.[1]) currentKbId = match[1]; + } + if (!currentKbId) { + currentKbId = knowledgeBaseId; + } + if (!currentKbId) { + MessagePlugin.error("缺少知识库ID"); + return; + } + + uploadKnowledgeFile(currentKbId, { file }) + .then((result: any) => { + if (result.success) { + MessagePlugin.info("上传成功！"); + getKnowled({ page: 1, page_size: 35 }, currentKbId); + } else { + const errorMessage = result.error?.message || result.message || "上传失败！"; + MessagePlugin.error(result.code === 'duplicate_file' ? "文件已存在" : errorMessage); + } + uploadInput.value.value = ""; + }) + .catch((err: any) => { + const errorMessage = err.error?.message || err.message || "上传失败！"; + MessagePlugin.error(err.code === 'duplicate_file' ? "文件已存在" : errorMessage); + uploadInput.value.value = ""; + }); + }; + const getCardDetails = (item: any) => { + Object.assign(details, { + title: "", + time: "", + md: [], + id: "", + }); + getKnowledgeDetails(item.id) + .then((result: any) => { + if (result.success && result.data) { + const { data } = result; + Object.assign(details, { + title: data.file_name, + time: formatStringDate(new Date(data.updated_at)), + id: data.id, + }); + } + }) + .catch(() => {}); + getfDetails(item.id, 1); + }; + + const getfDetails = (id: string, page: number) => { + getKnowledgeDetailsCon(id, page) + .then((result: any) => { + if (result.success && result.data) { + const { data, total: totalResult } = result; + if (page === 1) { + details.md = data; + } else { + details.md.push(...data); + } + details.total = totalResult; + } + }) + .catch(() => {}); + }; + return { + cardList, + moreIndex, + getKnowled, + details, + delKnowledge, + openMore, + onVisibleChange, + requestMethod, + getCardDetails, + total, + getfDetails, + }; +} diff --git a/frontend/src/main.ts b/frontend/src/main.ts new file mode 100644 index 0000000..0628e01 --- /dev/null +++ b/frontend/src/main.ts @@ -0,0 +1,16 @@ +import { createApp } from "vue"; +import { createPinia } from "pinia"; +import App from "./App.vue"; +import router from "./router"; +import "./assets/fonts.css"; +import TDesign from "tdesign-vue-next"; +// 引入组件库的少量全局样式变量 +import "tdesign-vue-next/es/style/index.css"; +import "@/assets/theme/theme.css"; +const app = createApp(App); + +app.use(TDesign); +app.use(createPinia()); +app.use(router); + +app.mount("#app"); diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts new file mode 100644 index 0000000..8432b0a --- /dev/null +++ b/frontend/src/router/index.ts @@ -0,0 +1,117 @@ +import { createRouter, createWebHistory } from 'vue-router' +import { listKnowledgeBases } from '@/api/knowledge-base' +import { useAuthStore } from '@/stores/auth' +import { validateToken } from '@/api/auth' + +const router = createRouter({ + history: createWebHistory(import.meta.env.BASE_URL), + routes: [ + { + path: "/", + redirect: "/platform/knowledge-bases", + }, + { + path: "/login", + name: "login", + component: () => import("../views/auth/Login.vue"), + meta: { requiresAuth: false, requiresInit: false } + }, + { + path: "/knowledgeBase", + name: "home", + component: () => import("../views/knowledge/KnowledgeBase.vue"), + meta: { requiresInit: true, requiresAuth: true } + }, + { + path: "/platform", + name: "Platform", + redirect: "/platform/knowledge-bases", + component: () => import("../views/platform/index.vue"), + meta: { requiresInit: true, requiresAuth: true }, + children: [ + { + path: "tenant", + name: "tenant", + component: () => import("../views/tenant/TenantInfo.vue"), + meta: { requiresInit: true, requiresAuth: true } + }, + { + path: "knowledge-bases", + name: "knowledgeBaseList", + component: () => import("../views/knowledge/KnowledgeBaseList.vue"), + meta: { requiresInit: true, requiresAuth: true } + }, + { + path: "knowledge-bases/:kbId", + name: "knowledgeBaseDetail", + component: () => import("../views/knowledge/KnowledgeBase.vue"), + meta: { requiresInit: true, requiresAuth: true } + }, + { + path: "knowledge-bases/:kbId/creatChat", + name: "kbCreatChat", + component: () => import("../views/creatChat/creatChat.vue"), + meta: { requiresInit: true, requiresAuth: true } + }, + { + path: "knowledge-bases/:kbId/settings", + name: "knowledgeBaseSettings", + component: () => import("../views/initialization/InitializationContent.vue"), + props: { isKbSettings: true }, + meta: { requiresInit: true, requiresAuth: true } + }, + { + path: "chat/:kbId/:chatid", + name: "chat", + component: () => import("../views/chat/index.vue"), + meta: { requiresInit: true, requiresAuth: true } + }, + ], + }, + ], +}); + +// 路由守卫：检查认证状态和系统初始化状态 +router.beforeEach(async (to, from, next) => { + const authStore = useAuthStore() + + // 如果访问的是登录页面或初始化页面，直接放行 + if (to.meta.requiresAuth === false || to.meta.requiresInit === false) { + // 如果已登录用户访问登录页面，重定向到知识库列表页面 + if (to.path === '/login' && authStore.isLoggedIn) { + next('/platform/knowledge-bases') + return + } + next() + return + } + + // 检查用户认证状态 + if (to.meta.requiresAuth !== false) { + if (!authStore.isLoggedIn) { + // 未登录，跳转到登录页面 + next('/login') + return + } + + // 验证Token有效性 + // try { + // const { valid } = await validateToken() + // if (!valid) { + // // Token无效，清空认证信息并跳转到登录页面 + // authStore.logout() + // next('/login') + // return + // } + // } catch (error) { + // console.error('Token验证失败:', error) + // authStore.logout() + // next('/login') + // return + // } + } + + next() +}); + +export default router diff --git a/frontend/src/stores/auth.ts b/frontend/src/stores/auth.ts new file mode 100644 index 0000000..ab8a03e --- /dev/null +++ b/frontend/src/stores/auth.ts @@ -0,0 +1,169 @@ +import { defineStore } from 'pinia' +import { ref, computed } from 'vue' +import type { UserInfo, TenantInfo, KnowledgeBaseInfo } from '@/api/auth' + +export const useAuthStore = defineStore('auth', () => { + // 状态 + const user = ref(null) + const tenant = ref(null) + const token = ref('') + const refreshToken = ref('') + const knowledgeBases = ref([]) + const currentKnowledgeBase = ref(null) + + // 计算属性 + const isLoggedIn = computed(() => { + return !!token.value && !!user.value + }) + + const hasValidTenant = computed(() => { + return !!tenant.value && !!tenant.value.api_key + }) + + const currentTenantId = computed(() => { + return tenant.value?.id || '' + }) + + const currentUserId = computed(() => { + return user.value?.id || '' + }) + + // 操作方法 + const setUser = (userData: UserInfo) => { + user.value = userData + // 保存到localStorage + localStorage.setItem('weknora_user', JSON.stringify(userData)) + } + + const setTenant = (tenantData: TenantInfo) => { + tenant.value = tenantData + // 保存到localStorage + localStorage.setItem('weknora_tenant', JSON.stringify(tenantData)) + } + + const setToken = (tokenValue: string) => { + token.value = tokenValue + localStorage.setItem('weknora_token', tokenValue) + } + + const setRefreshToken = (refreshTokenValue: string) => { + refreshToken.value = refreshTokenValue + localStorage.setItem('weknora_refresh_token', refreshTokenValue) + } + + const setKnowledgeBases = (kbList: KnowledgeBaseInfo[]) => { + // 确保输入是数组 + knowledgeBases.value = Array.isArray(kbList) ? kbList : [] + localStorage.setItem('weknora_knowledge_bases', JSON.stringify(knowledgeBases.value)) + } + + const setCurrentKnowledgeBase = (kb: KnowledgeBaseInfo | null) => { + currentKnowledgeBase.value = kb + if (kb) { + localStorage.setItem('weknora_current_kb', JSON.stringify(kb)) + } else { + localStorage.removeItem('weknora_current_kb') + } + } + + + const logout = () => { + // 清空状态 + user.value = null + tenant.value = null + token.value = '' + refreshToken.value = '' + knowledgeBases.value = [] + currentKnowledgeBase.value = null + + // 清空localStorage + localStorage.removeItem('weknora_user') + localStorage.removeItem('weknora_tenant') + localStorage.removeItem('weknora_token') + localStorage.removeItem('weknora_refresh_token') + localStorage.removeItem('weknora_knowledge_bases') + localStorage.removeItem('weknora_current_kb') + + } + + const initFromStorage = () => { + // 从localStorage恢复状态 + const storedUser = localStorage.getItem('weknora_user') + const storedTenant = localStorage.getItem('weknora_tenant') + const storedToken = localStorage.getItem('weknora_token') + const storedRefreshToken = localStorage.getItem('weknora_refresh_token') + const storedKnowledgeBases = localStorage.getItem('weknora_knowledge_bases') + const storedCurrentKb = localStorage.getItem('weknora_current_kb') + + if (storedUser) { + try { + user.value = JSON.parse(storedUser) + } catch (e) { + console.error('解析用户信息失败:', e) + } + } + + if (storedTenant) { + try { + tenant.value = JSON.parse(storedTenant) + } catch (e) { + console.error('解析租户信息失败:', e) + } + } + + if (storedToken) { + token.value = storedToken + } + + if (storedRefreshToken) { + refreshToken.value = storedRefreshToken + } + + if (storedKnowledgeBases) { + try { + const parsed = JSON.parse(storedKnowledgeBases) + knowledgeBases.value = Array.isArray(parsed) ? parsed : [] + } catch (e) { + console.error('解析知识库列表失败:', e) + knowledgeBases.value = [] + } + } + + if (storedCurrentKb) { + try { + currentKnowledgeBase.value = JSON.parse(storedCurrentKb) + } catch (e) { + console.error('解析当前知识库失败:', e) + } + } + } + + // 初始化时从localStorage恢复状态 + initFromStorage() + + return { + // 状态 + user, + tenant, + token, + refreshToken, + knowledgeBases, + currentKnowledgeBase, + + // 计算属性 + isLoggedIn, + hasValidTenant, + currentTenantId, + currentUserId, + + // 方法 + setUser, + setTenant, + setToken, + setRefreshToken, + setKnowledgeBases, + setCurrentKnowledgeBase, + logout, + initFromStorage + } +}) \ No newline at end of file diff --git a/frontend/src/stores/knowledge.ts b/frontend/src/stores/knowledge.ts new file mode 100644 index 0000000..29d3a40 --- /dev/null +++ b/frontend/src/stores/knowledge.ts @@ -0,0 +1,11 @@ +import { ref, computed, reactive } from "vue"; + +import { defineStore } from "pinia"; + +export const knowledgeStore = defineStore("knowledge", { + state: () => ({ + cardList: ref([]), + total: ref(0), + }), + actions: {}, +}); diff --git a/frontend/src/stores/menu.ts b/frontend/src/stores/menu.ts new file mode 100644 index 0000000..90ebb5b --- /dev/null +++ b/frontend/src/stores/menu.ts @@ -0,0 +1,50 @@ +import { ref, computed, reactive } from 'vue' + +import { defineStore } from 'pinia'; + +export const useMenuStore = defineStore('menuStore', { + state: () => ({ + menuArr: reactive([ + { title: '知识库', icon: 'zhishiku', path: 'knowledge-bases' }, + { + title: '对话', + icon: 'prefixIcon', + path: 'creatChat', + childrenPath: 'chat', + children: reactive([]), + }, + { title: '系统信息', icon: 'tenant', path: 'tenant' }, + { title: '退出登录', icon: 'logout', path: 'logout' } + ]), + isFirstSession: false, + firstQuery: '' + } + ), + actions: { + clearMenuArr() { + this.menuArr[1].children = reactive([]); + }, + updatemenuArr(obj: any) { + this.menuArr[1].children?.push(obj); + }, + updataMenuChildren(item: object) { + this.menuArr[1].children?.unshift(item) + }, + updatasessionTitle(session_id: string, title: string) { + this.menuArr[1].children?.forEach((item: any) => { + if (item.id == session_id) { + item.title = title; + item.isNoTitle = false; + } + }); + }, + changeIsFirstSession(payload: boolean) { + this.isFirstSession = payload; + }, + changeFirstQuery(payload: string) { + this.firstQuery = payload; + } + } +}); + + diff --git a/frontend/src/stores/settings.ts b/frontend/src/stores/settings.ts new file mode 100644 index 0000000..b22a8b4 --- /dev/null +++ b/frontend/src/stores/settings.ts @@ -0,0 +1,51 @@ +import { defineStore } from "pinia"; + +// 定义设置接口 +interface Settings { + endpoint: string; + apiKey: string; + knowledgeBaseId: string; +} + +// 默认设置 +const defaultSettings: Settings = { + endpoint: import.meta.env.VITE_IS_DOCKER ? "" : "http://localhost:8080", + apiKey: "", + knowledgeBaseId: "", +}; + +export const useSettingsStore = defineStore("settings", { + state: () => ({ + // 从本地存储加载设置，如果没有则使用默认设置 + settings: JSON.parse(localStorage.getItem("WeKnora_settings") || JSON.stringify(defaultSettings)), + }), + + actions: { + // 保存设置 + saveSettings(settings: Settings) { + this.settings = { ...settings }; + // 保存到localStorage + localStorage.setItem("WeKnora_settings", JSON.stringify(this.settings)); + }, + + // 获取设置 + getSettings(): Settings { + return this.settings; + }, + + // 获取API端点 + getEndpoint(): string { + return this.settings.endpoint || defaultSettings.endpoint; + }, + + // 获取API Key + getApiKey(): string { + return this.settings.apiKey; + }, + + // 获取知识库ID + getKnowledgeBaseId(): string { + return this.settings.knowledgeBaseId; + }, + }, +}); \ No newline at end of file diff --git a/frontend/src/utils/index.ts b/frontend/src/utils/index.ts new file mode 100644 index 0000000..04f8c71 --- /dev/null +++ b/frontend/src/utils/index.ts @@ -0,0 +1,44 @@ +import { MessagePlugin } from "tdesign-vue-next"; +export function generateRandomString(length: number) { + let result = ""; + const characters = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + const charactersLength = characters.length; + for (let i = 0; i < length; i++) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + } + return result; +} + +export function formatStringDate(date: any) { + let data = new Date(date); + let year = data.getFullYear(); + let month = String(data.getMonth() + 1).padStart(2, '0'); + let day = String(data.getDate()).padStart(2, '0'); + let hour = String(data.getHours()).padStart(2, '0'); + let minute = String(data.getMinutes()).padStart(2, '0'); + let second = String(data.getSeconds()).padStart(2, '0'); + return ( + year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second + ); +} +export function kbFileTypeVerification(file: any) { + let validTypes = ["pdf", "txt", "md", "docx", "doc", "jpg", "jpeg", "png"]; + let type = file.name.substring(file.name.lastIndexOf(".") + 1); + if (!validTypes.includes(type)) { + MessagePlugin.error("文件类型错误！"); + return true; + } + if ( + (type == "pdf" || type == "docx" || type == "doc") && + file.size > 31457280 + ) { + MessagePlugin.error("pdf/doc文件不能超过30M！"); + return true; + } + if ((type == "txt" || type == "md") && file.size > 31457280) { + MessagePlugin.error("txt/md文件不能超过30M！"); + return true; + } + return false +} diff --git a/frontend/src/utils/request.ts b/frontend/src/utils/request.ts new file mode 100644 index 0000000..0d644fb --- /dev/null +++ b/frontend/src/utils/request.ts @@ -0,0 +1,198 @@ +// src/utils/request.js +import axios from "axios"; +import { generateRandomString } from "./index"; + +// API基础URL +const BASE_URL = import.meta.env.VITE_IS_DOCKER ? "" : "http://localhost:8080"; + + +// 创建Axios实例 +const instance = axios.create({ + baseURL: BASE_URL, // 使用配置的API基础URL + timeout: 30000, // 请求超时时间 + headers: { + "Content-Type": "application/json", + "X-Request-ID": `${generateRandomString(12)}`, + }, +}); + + +instance.interceptors.request.use( + (config) => { + // 添加JWT token认证 + const token = localStorage.getItem('weknora_token'); + if (token) { + config.headers["Authorization"] = `Bearer ${token}`; + } + + config.headers["X-Request-ID"] = `${generateRandomString(12)}`; + return config; + }, + (error) => { + return Promise.reject(error); + } +); + +// Token刷新标志，防止多个请求同时刷新token +let isRefreshing = false; +let failedQueue: Array<{ resolve: Function; reject: Function }> = []; +let hasRedirectedOn401 = false; + +// 处理队列中的请求 +const processQueue = (error: any, token: string | null = null) => { + failedQueue.forEach(({ resolve, reject }) => { + if (error) { + reject(error); + } else { + resolve(token); + } + }); + + failedQueue = []; +}; + +instance.interceptors.response.use( + (response) => { + // 根据业务状态码处理逻辑 + const { status, data } = response; + if (status === 200 || status === 201) { + return data; + } else { + return Promise.reject(data); + } + }, + async (error: any) => { + const originalRequest = error.config; + + if (!error.response) { + return Promise.reject({ message: "网络错误，请检查您的网络连接" }); + } + + // 如果是登录接口的401，直接返回错误以便页面展示toast，不做跳转 + if (error.response.status === 401 && originalRequest?.url?.includes('/auth/login')) { + const { status, data } = error.response; + return Promise.reject({ status, message: (typeof data === 'object' ? data?.message : data) || '用户名或密码错误' }); + } + + // 如果是401错误且不是刷新token的请求，尝试刷新token + if (error.response.status === 401 && !originalRequest._retry && !originalRequest.url?.includes('/auth/refresh')) { + if (isRefreshing) { + // 如果正在刷新token，将请求加入队列 + return new Promise((resolve, reject) => { + failedQueue.push({ resolve, reject }); + }).then(token => { + originalRequest.headers['Authorization'] = 'Bearer ' + token; + return instance(originalRequest); + }).catch(err => { + return Promise.reject(err); + }); + } + + originalRequest._retry = true; + isRefreshing = true; + + const refreshToken = localStorage.getItem('weknora_refresh_token'); + + if (refreshToken) { + try { + // 动态导入refresh token API + const { refreshToken: refreshTokenAPI } = await import('../api/auth/index'); + const response = await refreshTokenAPI(refreshToken); + + if (response.success && response.data) { + const { token, refreshToken: newRefreshToken } = response.data; + + // 更新localStorage中的token + localStorage.setItem('weknora_token', token); + localStorage.setItem('weknora_refresh_token', newRefreshToken); + + // 更新请求头 + originalRequest.headers['Authorization'] = 'Bearer ' + token; + + // 处理队列中的请求 + processQueue(null, token); + + return instance(originalRequest); + } else { + throw new Error(response.message || 'Token刷新失败'); + } + } catch (refreshError) { + // 刷新失败，清除所有token并跳转到登录页 + localStorage.removeItem('weknora_token'); + localStorage.removeItem('weknora_refresh_token'); + localStorage.removeItem('weknora_user'); + localStorage.removeItem('weknora_tenant'); + + processQueue(refreshError, null); + + // 跳转到登录页 + if (!hasRedirectedOn401 && typeof window !== 'undefined') { + hasRedirectedOn401 = true; + window.location.href = '/login'; + } + + return Promise.reject(refreshError); + } finally { + isRefreshing = false; + } + } else { + // 没有refresh token，直接跳转到登录页 + localStorage.removeItem('weknora_token'); + localStorage.removeItem('weknora_user'); + localStorage.removeItem('weknora_tenant'); + + if (!hasRedirectedOn401 && typeof window !== 'undefined') { + hasRedirectedOn401 = true; + window.location.href = '/login'; + } + + return Promise.reject({ message: '请重新登录' }); + } + } + + const { status, data } = error.response; + // 将HTTP状态码一并抛出，方便上层判断401等场景 + return Promise.reject({ status, ...(typeof data === 'object' ? data : { message: data }) }); + } +); + +export function get(url: string) { + return instance.get(url); +} + +export async function getDown(url: string) { + let res = await instance.get(url, { + responseType: "blob", + }); + return res +} + +export function postUpload(url: string, data = {}) { + return instance.post(url, data, { + headers: { + "Content-Type": "multipart/form-data", + "X-Request-ID": `${generateRandomString(12)}`, + }, + }); +} + +export function postChat(url: string, data = {}) { + return instance.post(url, data, { + headers: { + "Content-Type": "text/event-stream;charset=utf-8", + "X-Request-ID": `${generateRandomString(12)}`, + }, + }); +} + +export function post(url: string, data = {}) { + return instance.post(url, data); +} + +export function put(url: string, data = {}) { + return instance.put(url, data); +} + +export function del(url: string) { + return instance.delete(url); +} diff --git a/frontend/src/utils/security.ts b/frontend/src/utils/security.ts new file mode 100644 index 0000000..b1eab2f --- /dev/null +++ b/frontend/src/utils/security.ts @@ -0,0 +1,207 @@ +/** + * 安全工具类 - 防止 XSS 攻击 + */ + +import DOMPurify from 'dompurify'; + +// 配置 DOMPurify 的安全策略 +const DOMPurifyConfig = { + // 允许的标签 + ALLOWED_TAGS: [ + 'p', 'br', 'strong', 'em', 'u', 's', 'del', 'ins', + 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', + 'ul', 'ol', 'li', 'blockquote', 'pre', 'code', + 'a', 'img', 'table', 'thead', 'tbody', 'tr', 'th', 'td', + 'div', 'span', 'figure', 'figcaption', 'think' + ], + // 允许的属性 + ALLOWED_ATTR: [ + 'href', 'title', 'alt', 'src', 'class', 'id', 'style', + 'target', 'rel', 'width', 'height' + ], + // 允许的协议 + ALLOWED_URI_REGEXP: /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i, + // 禁止的标签和属性 + FORBID_TAGS: ['script', 'object', 'embed', 'form', 'input', 'button'], + FORBID_ATTR: ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'], + // 其他安全配置 + KEEP_CONTENT: true, + RETURN_DOM: false, + RETURN_DOM_FRAGMENT: false, + RETURN_DOM_IMPORT: false, + SANITIZE_DOM: true, + SANITIZE_NAMED_PROPS: true, + WHOLE_DOCUMENT: false, + // 自定义钩子函数 + HOOKS: { + // 在清理前处理 + beforeSanitizeElements: (currentNode: Element) => { + // 移除所有 script 标签 + if (currentNode.tagName === 'SCRIPT') { + currentNode.remove(); + return null; + } + // 移除所有事件处理器 + const eventAttrs = ['onclick', 'onload', 'onerror', 'onmouseover', 'onfocus', 'onblur']; + eventAttrs.forEach(attr => { + if (currentNode.hasAttribute(attr)) { + currentNode.removeAttribute(attr); + } + }); + }, + // 在清理后处理 + afterSanitizeElements: (currentNode: Element) => { + // 确保所有链接都有 rel="noopener noreferrer" + if (currentNode.tagName === 'A') { + const href = currentNode.getAttribute('href'); + if (href && href.startsWith('http')) { + currentNode.setAttribute('rel', 'noopener noreferrer'); + currentNode.setAttribute('target', '_blank'); + } + } + // 确保所有图片都有 alt 属性 + if (currentNode.tagName === 'IMG') { + if (!currentNode.getAttribute('alt')) { + currentNode.setAttribute('alt', ''); + } + } + } + } +}; + +/** + * 安全地清理 HTML 内容 + * @param html 需要清理的 HTML 字符串 + * @returns 清理后的安全 HTML 字符串 + */ +export function sanitizeHTML(html: string): string { + if (!html || typeof html !== 'string') { + return ''; + } + + try { + return DOMPurify.sanitize(html, DOMPurifyConfig); + } catch (error) { + console.error('HTML sanitization failed:', error); + // 如果清理失败，返回转义的纯文本 + return escapeHTML(html); + } +} + +/** + * 转义 HTML 特殊字符 + * @param text 需要转义的文本 + * @returns 转义后的文本 + */ +export function escapeHTML(text: string): string { + if (!text || typeof text !== 'string') { + return ''; + } + + const map: { [key: string]: string } = { + '&': '&', + '<': '<', + '>': '>', + '"': '"', + "'": ''', + '/': '/', + '`': '`', + '=': '=' + }; + + return text.replace(/[&<>"'`=\/]/g, (s) => map[s]); +} + +/** + * 验证 URL 是否安全 + * @param url 需要验证的 URL + * @returns 是否为安全 URL + */ +export function isValidURL(url: string): boolean { + if (!url || typeof url !== 'string') { + return false; + } + + try { + const urlObj = new URL(url); + // 只允许 http 和 https 协议 + return ['http:', 'https:'].includes(urlObj.protocol); + } catch { + return false; + } +} + +/** + * 安全地处理 Markdown 内容 + * @param markdown Markdown 文本 + * @returns 安全的 HTML 字符串 + */ +export function safeMarkdownToHTML(markdown: string): string { + if (!markdown || typeof markdown !== 'string') { + return ''; + } + + // 首先转义可能的 HTML 标签 + const escapedMarkdown = markdown + .replace(/)<[^<]*)*<\/script>/gi, '') + .replace(/)<[^<]*)*<\/iframe>/gi, '') + .replace(/)<[^<]*)*<\/object>/gi, '') + .replace(/)<[^<]*)*<\/embed>/gi, ''); + + return escapedMarkdown; +} + +/** + * 清理用户输入 + * @param input 用户输入 + * @returns 清理后的安全输入 + */ +export function sanitizeUserInput(input: string): string { + if (!input || typeof input !== 'string') { + return ''; + } + + // 移除控制字符 + let cleaned = input.replace(/[\x00-\x1F\x7F-\x9F]/g, ''); + + // 限制长度 + if (cleaned.length > 10000) { + cleaned = cleaned.substring(0, 10000); + } + + return cleaned.trim(); +} + +/** + * 验证图片 URL 是否安全 + * @param url 图片 URL + * @returns 是否为安全的图片 URL + */ +export function isValidImageURL(url: string): boolean { + if (!isValidURL(url)) { + return false; + } + + // 检查是否为图片文件 + const imageExtensions = /\.(jpg|jpeg|png|gif|webp|svg|bmp|ico)(\?.*)?$/i; + return imageExtensions.test(url); +} + +/** + * 创建安全的图片元素 + * @param src 图片源 + * @param alt 替代文本 + * @param title 标题 + * @returns 安全的图片 HTML + */ +export function createSafeImage(src: string, alt: string = '', title: string = ''): string { + if (!isValidImageURL(src)) { + return ''; + } + + const safeSrc = escapeHTML(src); + const safeAlt = escapeHTML(alt); + const safeTitle = escapeHTML(title); + + return `${safeAlt}`; +} diff --git a/frontend/src/views/auth/Login.vue b/frontend/src/views/auth/Login.vue new file mode 100644 index 0000000..efd2e92 --- /dev/null +++ b/frontend/src/views/auth/Login.vue @@ -0,0 +1,553 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/views/chat/components/botmsg.vue b/frontend/src/views/chat/components/botmsg.vue new file mode 100644 index 0000000..d476065 --- /dev/null +++ b/frontend/src/views/chat/components/botmsg.vue @@ -0,0 +1,248 @@ + + + \ No newline at end of file diff --git a/frontend/src/views/chat/components/deepThink.vue b/frontend/src/views/chat/components/deepThink.vue new file mode 100644 index 0000000..36bd149 --- /dev/null +++ b/frontend/src/views/chat/components/deepThink.vue @@ -0,0 +1,151 @@ + + + \ No newline at end of file diff --git a/frontend/src/views/chat/components/docInfo.vue b/frontend/src/views/chat/components/docInfo.vue new file mode 100644 index 0000000..e8259aa --- /dev/null +++ b/frontend/src/views/chat/components/docInfo.vue @@ -0,0 +1,138 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/views/chat/components/sendMsg.vue b/frontend/src/views/chat/components/sendMsg.vue new file mode 100644 index 0000000..ab9a8a0 --- /dev/null +++ b/frontend/src/views/chat/components/sendMsg.vue @@ -0,0 +1,20 @@ + + + \ No newline at end of file diff --git a/frontend/src/views/chat/components/usermsg.vue b/frontend/src/views/chat/components/usermsg.vue new file mode 100644 index 0000000..fed5d6b --- /dev/null +++ b/frontend/src/views/chat/components/usermsg.vue @@ -0,0 +1,38 @@ + + + \ No newline at end of file diff --git a/frontend/src/views/chat/index.vue b/frontend/src/views/chat/index.vue new file mode 100644 index 0000000..c770ddc --- /dev/null +++ b/frontend/src/views/chat/index.vue @@ -0,0 +1,312 @@ + + + \ No newline at end of file diff --git a/frontend/src/views/creatChat/creatChat.vue b/frontend/src/views/creatChat/creatChat.vue new file mode 100644 index 0000000..10783ba --- /dev/null +++ b/frontend/src/views/creatChat/creatChat.vue @@ -0,0 +1,216 @@ + + + + \ No newline at end of file diff --git a/frontend/src/views/initialization/InitializationContent.vue b/frontend/src/views/initialization/InitializationContent.vue new file mode 100644 index 0000000..e26716f --- /dev/null +++ b/frontend/src/views/initialization/InitializationContent.vue @@ -0,0 +1,3567 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue new file mode 100644 index 0000000..ae68163 --- /dev/null +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -0,0 +1,627 @@ + + + + + diff --git a/frontend/src/views/knowledge/KnowledgeBaseList.vue b/frontend/src/views/knowledge/KnowledgeBaseList.vue new file mode 100644 index 0000000..cb8ecc9 --- /dev/null +++ b/frontend/src/views/knowledge/KnowledgeBaseList.vue @@ -0,0 +1,234 @@ + + + + + diff --git a/frontend/src/views/platform/index.vue b/frontend/src/views/platform/index.vue new file mode 100644 index 0000000..8ce426e --- /dev/null +++ b/frontend/src/views/platform/index.vue @@ -0,0 +1,138 @@ + + + \ No newline at end of file diff --git a/frontend/src/views/settings/Settings.vue b/frontend/src/views/settings/Settings.vue new file mode 100644 index 0000000..616e43a --- /dev/null +++ b/frontend/src/views/settings/Settings.vue @@ -0,0 +1,100 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/views/settings/SystemSettings.vue b/frontend/src/views/settings/SystemSettings.vue new file mode 100644 index 0000000..17ba9e1 --- /dev/null +++ b/frontend/src/views/settings/SystemSettings.vue @@ -0,0 +1,211 @@ + + + + + diff --git a/frontend/src/views/tenant/TenantInfo.vue b/frontend/src/views/tenant/TenantInfo.vue new file mode 100644 index 0000000..c15f379 --- /dev/null +++ b/frontend/src/views/tenant/TenantInfo.vue @@ -0,0 +1,535 @@ + + + + + diff --git a/frontend/tsconfig.app.json b/frontend/tsconfig.app.json new file mode 100644 index 0000000..913b8f2 --- /dev/null +++ b/frontend/tsconfig.app.json @@ -0,0 +1,12 @@ +{ + "extends": "@vue/tsconfig/tsconfig.dom.json", + "include": ["env.d.ts", "src/**/*", "src/**/*.vue"], + "exclude": ["src/**/__tests__/*"], + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", + + "paths": { + "@/*": ["./src/*"] + } + } +} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json new file mode 100644 index 0000000..66b5e57 --- /dev/null +++ b/frontend/tsconfig.json @@ -0,0 +1,11 @@ +{ + "files": [], + "references": [ + { + "path": "./tsconfig.node.json" + }, + { + "path": "./tsconfig.app.json" + } + ] +} diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json new file mode 100644 index 0000000..a83dfc9 --- /dev/null +++ b/frontend/tsconfig.node.json @@ -0,0 +1,19 @@ +{ + "extends": "@tsconfig/node22/tsconfig.json", + "include": [ + "vite.config.*", + "vitest.config.*", + "cypress.config.*", + "nightwatch.conf.*", + "playwright.config.*", + "eslint.config.*" + ], + "compilerOptions": { + "noEmit": true, + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", + + "module": "ESNext", + "moduleResolution": "Bundler", + "types": ["node"] + } +} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts new file mode 100644 index 0000000..24bc464 --- /dev/null +++ b/frontend/vite.config.ts @@ -0,0 +1,20 @@ +import { fileURLToPath, URL } from 'node:url' +import { defineConfig } from 'vite' +import vue from '@vitejs/plugin-vue' +import vueJsx from '@vitejs/plugin-vue-jsx' + +export default defineConfig({ + server: { + port: 5000, // 将端口改为 5000 + strictPort: true, // 端口被占用时直接退出，不尝试其他端口:cite[1] + }, + plugins: [ + vue(), + vueJsx(), + ], + resolve: { + alias: { + '@': fileURLToPath(new URL('./src', import.meta.url)) + }, + } +}) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1675d54 --- /dev/null +++ b/go.mod @@ -0,0 +1,113 @@ +module github.com/Tencent/WeKnora + +go 1.24.0 + +toolchain go1.24.2 + +require ( + github.com/elastic/go-elasticsearch/v7 v7.17.10 + github.com/elastic/go-elasticsearch/v8 v8.18.0 + github.com/gin-contrib/cors v1.7.5 + github.com/gin-gonic/gin v1.10.0 + github.com/go-viper/mapstructure/v2 v2.2.1 + github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/google/uuid v1.6.0 + github.com/hibiken/asynq v0.25.1 + github.com/minio/minio-go/v7 v7.0.90 + github.com/neo4j/neo4j-go-driver/v6 v6.0.0-alpha.1 + github.com/ollama/ollama v0.11.4 + github.com/panjf2000/ants/v2 v2.11.2 + github.com/parquet-go/parquet-go v0.25.0 + github.com/pgvector/pgvector-go v0.3.0 + github.com/redis/go-redis/v9 v9.14.0 + github.com/sashabaranov/go-openai v1.40.5 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/viper v1.20.1 + github.com/stretchr/testify v1.11.1 + github.com/tencentyun/cos-go-sdk-v5 v0.7.65 + github.com/yanyiwu/gojieba v1.4.5 + go.opentelemetry.io/otel v1.37.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 + go.opentelemetry.io/otel/sdk v1.37.0 + go.opentelemetry.io/otel/trace v1.37.0 + go.uber.org/dig v1.18.1 + golang.org/x/crypto v0.42.0 + golang.org/x/sync v0.17.0 + google.golang.org/grpc v1.73.0 + google.golang.org/protobuf v1.36.9 + gorm.io/driver/postgres v1.5.11 + gorm.io/gorm v1.25.12 +) + +require ( + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/bytedance/sonic v1.13.2 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cenkalti/backoff/v5 v5.0.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clbanning/mxj v1.8.4 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gin-contrib/sse v1.0.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.2 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/minio/crc64nvme v1.0.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mozillazg/go-httpheader v0.2.1 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.15.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/time v0.13.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e8a3ae1 --- /dev/null +++ b/go.sum @@ -0,0 +1,297 @@ +entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= +entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= +github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elastic/elastic-transport-go/v8 v8.7.0 h1:OgTneVuXP2uip4BA658Xi6Hfw+PeIOod2rY3GVMGoVE= +github.com/elastic/elastic-transport-go/v8 v8.7.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= +github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/elastic/go-elasticsearch/v8 v8.18.0 h1:ANNq1h7DEiPUaALb8+5w3baQzaS08WfHV0DNzp0VG4M= +github.com/elastic/go-elasticsearch/v8 v8.18.0/go.mod h1:WLqwXsJmQoYkoA9JBFeEwPkQhCfAZuUvfpdU/NvSSf0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/gin-contrib/cors v1.7.5 h1:cXC9SmofOrRg0w9PigwGlHG3ztswH6bqq4vJVXnvYMk= +github.com/gin-contrib/cors v1.7.5/go.mod h1:4q3yi7xBEDDWKapjT2o1V7mScKDDr8k+jZ0fSquGoy0= +github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= +github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hibiken/asynq v0.25.1 h1:phj028N0nm15n8O2ims+IvJ2gz4k2auvermngh9JhTw= +github.com/hibiken/asynq v0.25.1/go.mod h1:pazWNOLBu0FEynQRBvHA26qdIKRSmfdIfUm4HdsLmXg= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.90 h1:TmSj1083wtAD0kEYTx7a5pFsv3iRYMsOJ6A4crjA1lE= +github.com/minio/minio-go/v7 v7.0.90/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw94j2Nxku/Go= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/neo4j/neo4j-go-driver/v6 v6.0.0-alpha.1 h1:nV3ZdYJTi73jel0mm3dpWumNY3i3nwyo25y69SPGwyg= +github.com/neo4j/neo4j-go-driver/v6 v6.0.0-alpha.1/go.mod h1:hzSTfNfM31p1uRSzL1F/BAYOgaiTarE6OAQBajfsm+I= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/ollama/ollama v0.11.4 h1:6xLYLEPTKtw6N20qQecyEL/rrBktPO4o5U05cnvkSmI= +github.com/ollama/ollama v0.11.4/go.mod h1:9+1//yWPsDE2u+l1a5mpaKrYw4VdnSsRU3ioq5BvMms= +github.com/panjf2000/ants/v2 v2.11.2 h1:AVGpMSePxUNpcLaBO34xuIgM1ZdKOiGnpxLXixLi5Jo= +github.com/panjf2000/ants/v2 v2.11.2/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/parquet-go/parquet-go v0.25.0 h1:GwKy11MuF+al/lV6nUsFw8w8HCiPOSAx1/y8yFxjH5c= +github.com/parquet-go/parquet-go v0.25.0/go.mod h1:OqBBRGBl7+llplCvDMql8dEKaDqjaFA/VAPw+OJiNiw= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc= +github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE= +github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sashabaranov/go-openai v1.40.5 h1:SwIlNdWflzR1Rxd1gv3pUg6pwPc6cQ2uMoHs8ai+/NY= +github.com/sashabaranov/go-openai v1.40.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= +github.com/tencentyun/cos-go-sdk-v5 v0.7.65 h1:+WBbfwThfZSbxpf1Dw6fyMwyzVtWBBExqfDJ5giiR2s= +github.com/tencentyun/cos-go-sdk-v5 v0.7.65/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yanyiwu/gojieba v1.4.5 h1:VyZogGtdFSnJbACHvDRvDreXPPVPCg8axKFUdblU/JI= +github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 h1:T0Ec2E+3YZf5bgTNQVet8iTDW7oIk03tXHq+wkwIDnE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0/go.mod h1:30v2gqH+vYGJsesLWFov8u47EpYTcIQcBjKpI6pJThg= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= +go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= +go.uber.org/dig v1.18.1 h1:rLww6NuajVjeQn+49u5NcezUJEGwd5uXmyoCKW2g5Es= +go.uber.org/dig v1.18.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= +golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= +golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 h1:oWVWY3NzT7KJppx2UKhKmzPq4SRe0LdCijVRwvGeikY= +google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822/go.mod h1:h3c4v36UTKzUiuaOKQ6gr3S+0hovBtUrXzTG/i3+XEc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 h1:fc6jSaCT0vBduLYZHYrBBNY4dsWuvgyff9noRNDdBeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= +google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314= +gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go new file mode 100644 index 0000000..2b01251 --- /dev/null +++ b/internal/application/repository/chunk.go @@ -0,0 +1,131 @@ +package repository + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/common" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +// chunkRepository implements the ChunkRepository interface +type chunkRepository struct { + db *gorm.DB +} + +// NewChunkRepository creates a new chunk repository +func NewChunkRepository(db *gorm.DB) interfaces.ChunkRepository { + return &chunkRepository{db: db} +} + +// CreateChunks creates multiple chunks in batches +func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { + for _, chunk := range chunks { + chunk.Content = common.CleanInvalidUTF8(chunk.Content) + } + return r.db.WithContext(ctx).CreateInBatches(chunks, 100).Error +} + +// GetChunkByID retrieves a chunk by its ID and tenant ID +func (r *chunkRepository) GetChunkByID(ctx context.Context, tenantID uint, id string) (*types.Chunk, error) { + var chunk types.Chunk + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&chunk).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, errors.New("chunk not found") + } + return nil, err + } + return &chunk, nil +} + +// ListChunksByID retrieves multiple chunks by their IDs +func (r *chunkRepository) ListChunksByID( + ctx context.Context, tenantID uint, ids []string, +) ([]*types.Chunk, error) { + var chunks []*types.Chunk + if err := r.db.WithContext(ctx). + Where("tenant_id = ? AND id IN ?", tenantID, ids). + Find(&chunks).Error; err != nil { + return nil, err + } + return chunks, nil +} + +// ListChunksByKnowledgeID lists all chunks for a knowledge ID +func (r *chunkRepository) ListChunksByKnowledgeID( + ctx context.Context, tenantID uint, knowledgeID string, +) ([]*types.Chunk, error) { + var chunks []*types.Chunk + if err := r.db.WithContext(ctx). + Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info"). + Where("tenant_id = ? AND knowledge_id = ? and chunk_type = ?", tenantID, knowledgeID, "text"). + Order("chunk_index ASC"). + Find(&chunks).Error; err != nil { + return nil, err + } + return chunks, nil +} + +// ListPagedChunksByKnowledgeID lists chunks for a knowledge ID with pagination +func (r *chunkRepository) ListPagedChunksByKnowledgeID( + ctx context.Context, tenantID uint, knowledgeID string, page *types.Pagination, chunk_type []types.ChunkType, +) ([]*types.Chunk, int64, error) { + var chunks []*types.Chunk + var total int64 + + // First query the total count + if err := r.db.WithContext(ctx).Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID). + Count(&total).Error; err != nil { + return nil, 0, err + } + + // Then query the paginated data + if err := r.db.WithContext(ctx). + Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info"). + Where("tenant_id = ? AND knowledge_id = ? and chunk_type in (?)", tenantID, knowledgeID, chunk_type). + Order("chunk_index ASC"). + Offset(page.Offset()). + Limit(page.Limit()). + Find(&chunks).Error; err != nil { + return nil, 0, err + } + + return chunks, total, nil +} + +func (r *chunkRepository) ListChunkByParentID(ctx context.Context, tenantID uint, parentID string) ([]*types.Chunk, error) { + var chunks []*types.Chunk + if err := r.db.WithContext(ctx). + Where("tenant_id = ? AND parent_chunk_id = ?", tenantID, parentID). + Find(&chunks).Error; err != nil { + return nil, err + } + return chunks, nil +} + +// UpdateChunk updates a chunk +func (r *chunkRepository) UpdateChunk(ctx context.Context, chunk *types.Chunk) error { + return r.db.WithContext(ctx).Save(chunk).Error +} + +// DeleteChunk deletes a chunk by its ID +func (r *chunkRepository) DeleteChunk(ctx context.Context, tenantID uint, id string) error { + return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Chunk{}).Error +} + +// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID +func (r *chunkRepository) DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) error { + return r.db.WithContext(ctx).Where( + "tenant_id = ? AND knowledge_id = ?", tenantID, knowledgeID, + ).Delete(&types.Chunk{}).Error +} + +// DeleteByKnowledgeList deletes all chunks for a knowledge list +func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID uint, knowledgeIDs []string) error { + return r.db.WithContext(ctx).Where( + "tenant_id = ? AND knowledge_id in ?", tenantID, knowledgeIDs, + ).Delete(&types.Chunk{}).Error +} diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go new file mode 100644 index 0000000..946b5d5 --- /dev/null +++ b/internal/application/repository/knowledge.go @@ -0,0 +1,204 @@ +package repository + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +var ErrKnowledgeNotFound = errors.New("knowledge not found") + +// knowledgeRepository implements knowledge base and knowledge repository interface +type knowledgeRepository struct { + db *gorm.DB +} + +// NewKnowledgeRepository creates a new knowledge repository +func NewKnowledgeRepository(db *gorm.DB) interfaces.KnowledgeRepository { + return &knowledgeRepository{db: db} +} + +// CreateKnowledge creates knowledge +func (r *knowledgeRepository) CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { + err := r.db.WithContext(ctx).Create(knowledge).Error + return err +} + +// GetKnowledgeByID gets knowledge +func (r *knowledgeRepository) GetKnowledgeByID(ctx context.Context, tenantID uint, id string) (*types.Knowledge, error) { + var knowledge types.Knowledge + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).First(&knowledge).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrKnowledgeNotFound + } + return nil, err + } + return &knowledge, nil +} + +// ListKnowledgeByKnowledgeBaseID lists all knowledge in a knowledge base +func (r *knowledgeRepository) ListKnowledgeByKnowledgeBaseID( + ctx context.Context, tenantID uint, kbID string, +) ([]*types.Knowledge, error) { + var knowledges []*types.Knowledge + if err := r.db.WithContext(ctx).Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID). + Order("created_at DESC").Find(&knowledges).Error; err != nil { + return nil, err + } + return knowledges, nil +} + +// ListPagedKnowledgeByKnowledgeBaseID lists all knowledge in a knowledge base with pagination +func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID( + ctx context.Context, + tenantID uint, + kbID string, + page *types.Pagination, +) ([]*types.Knowledge, int64, error) { + var knowledges []*types.Knowledge + var total int64 + + // Query total count first + if err := r.db.WithContext(ctx).Model(&types.Knowledge{}). + Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID). + Count(&total).Error; err != nil { + return nil, 0, err + } + + // Then query paginated data + if err := r.db.WithContext(ctx). + Where("tenant_id = ? AND knowledge_base_id = ?", tenantID, kbID). + Order("created_at DESC"). + Offset(page.Offset()). + Limit(page.Limit()). + Find(&knowledges).Error; err != nil { + return nil, 0, err + } + + return knowledges, total, nil +} + +// UpdateKnowledge updates knowledge +func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { + err := r.db.WithContext(ctx).Save(knowledge).Error + return err +} + +// DeleteKnowledge deletes knowledge +func (r *knowledgeRepository) DeleteKnowledge(ctx context.Context, tenantID uint, id string) error { + return r.db.WithContext(ctx).Where("tenant_id = ? AND id = ?", tenantID, id).Delete(&types.Knowledge{}).Error +} + +// DeleteKnowledge deletes knowledge +func (r *knowledgeRepository) DeleteKnowledgeList(ctx context.Context, tenantID uint, ids []string) error { + return r.db.WithContext(ctx).Where("tenant_id = ? AND id in ?", tenantID, ids).Delete(&types.Knowledge{}).Error +} + +// GetKnowledgeBatch gets knowledge in batch +func (r *knowledgeRepository) GetKnowledgeBatch( + ctx context.Context, tenantID uint, ids []string, +) ([]*types.Knowledge, error) { + var knowledge []*types.Knowledge + if err := r.db.WithContext(ctx).Debug(). + Where("tenant_id = ? AND id IN ?", tenantID, ids). + Find(&knowledge).Error; err != nil { + return nil, err + } + return knowledge, nil +} + +// CheckKnowledgeExists checks if knowledge already exists +func (r *knowledgeRepository) CheckKnowledgeExists( + ctx context.Context, + tenantID uint, + kbID string, + params *types.KnowledgeCheckParams, +) (bool, *types.Knowledge, error) { + query := r.db.WithContext(ctx).Model(&types.Knowledge{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND parse_status <> ?", tenantID, kbID, "failed") + + if params.Type == "file" { + // If file hash exists, prioritize exact match using hash + if params.FileHash != "" { + var knowledge types.Knowledge + err := query.Where("file_hash = ?", params.FileHash).First(&knowledge).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil, nil + } + return false, nil, err + } + return true, &knowledge, nil + } + + // If no hash or hash doesn't match, use filename and size + if params.FileName != "" && params.FileSize > 0 { + var knowledge types.Knowledge + err := query.Where( + "file_name = ? AND file_size = ?", + params.FileName, params.FileSize, + ).First(&knowledge).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil, nil + } + return false, nil, err + } + return true, &knowledge, nil + } + } else if params.Type == "url" { + // If file hash exists, prioritize exact match using hash + if params.FileHash != "" { + var knowledge types.Knowledge + err := query.Where("type = 'url' AND file_hash = ?", params.FileHash).First(&knowledge).Error + if err == nil && knowledge.ID != "" { + return true, &knowledge, nil + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil, err + } + } + + if params.URL != "" { + var knowledge types.Knowledge + err := query.Where("type = 'url' AND source = ?", params.URL).First(&knowledge).Error + if err == nil && knowledge.ID != "" { + return true, &knowledge, nil + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return false, nil, err + } + } + return false, nil, nil + } + + // No valid parameters, default to not existing + return false, nil, nil +} + +func (r *knowledgeRepository) AminusB( + ctx context.Context, + Atenant uint, A string, + Btenant uint, B string, +) ([]string, error) { + knowledgeIDs := []string{} + subQuery := r.db.Model(&types.Knowledge{}). + Where("tenant_id = ? AND knowledge_base_id = ?", Btenant, B).Select("file_hash") + err := r.db.Model(&types.Knowledge{}). + Where("tenant_id = ? AND knowledge_base_id = ?", Atenant, A). + Where("file_hash NOT IN (?)", subQuery). + Pluck("id", &knowledgeIDs). + Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return knowledgeIDs, nil + } + return knowledgeIDs, err +} + +func (r *knowledgeRepository) UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error { + err := r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Update(column, value).Error + return err +} diff --git a/internal/application/repository/knowledgebase.go b/internal/application/repository/knowledgebase.go new file mode 100644 index 0000000..efd7ca5 --- /dev/null +++ b/internal/application/repository/knowledgebase.go @@ -0,0 +1,70 @@ +package repository + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +var ErrKnowledgeBaseNotFound = errors.New("knowledge base not found") + +// knowledgeBaseRepository implements the KnowledgeBaseRepository interface +type knowledgeBaseRepository struct { + db *gorm.DB +} + +// NewKnowledgeBaseRepository creates a new knowledge base repository +func NewKnowledgeBaseRepository(db *gorm.DB) interfaces.KnowledgeBaseRepository { + return &knowledgeBaseRepository{db: db} +} + +// CreateKnowledgeBase creates a new knowledge base +func (r *knowledgeBaseRepository) CreateKnowledgeBase(ctx context.Context, kb *types.KnowledgeBase) error { + return r.db.WithContext(ctx).Create(kb).Error +} + +// GetKnowledgeBaseByID gets a knowledge base by id +func (r *knowledgeBaseRepository) GetKnowledgeBaseByID(ctx context.Context, id string) (*types.KnowledgeBase, error) { + var kb types.KnowledgeBase + if err := r.db.WithContext(ctx).Where("id = ?", id).First(&kb).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrKnowledgeBaseNotFound + } + return nil, err + } + return &kb, nil +} + +// ListKnowledgeBases lists all knowledge bases +func (r *knowledgeBaseRepository) ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) { + var kbs []*types.KnowledgeBase + if err := r.db.WithContext(ctx).Find(&kbs).Error; err != nil { + return nil, err + } + return kbs, nil +} + +// ListKnowledgeBasesByTenantID lists all knowledge bases by tenant id +func (r *knowledgeBaseRepository) ListKnowledgeBasesByTenantID( + ctx context.Context, tenantID uint, +) ([]*types.KnowledgeBase, error) { + var kbs []*types.KnowledgeBase + if err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID). + Order("created_at DESC").Find(&kbs).Error; err != nil { + return nil, err + } + return kbs, nil +} + +// UpdateKnowledgeBase updates a knowledge base +func (r *knowledgeBaseRepository) UpdateKnowledgeBase(ctx context.Context, kb *types.KnowledgeBase) error { + return r.db.WithContext(ctx).Save(kb).Error +} + +// DeleteKnowledgeBase deletes a knowledge base +func (r *knowledgeBaseRepository) DeleteKnowledgeBase(ctx context.Context, id string) error { + return r.db.WithContext(ctx).Where("id = ?", id).Delete(&types.KnowledgeBase{}).Error +} diff --git a/internal/application/repository/message.go b/internal/application/repository/message.go new file mode 100644 index 0000000..0808e78 --- /dev/null +++ b/internal/application/repository/message.go @@ -0,0 +1,153 @@ +package repository + +import ( + "context" + "slices" + "time" + + "gorm.io/gorm" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// messageRepository implements the message repository interface +type messageRepository struct { + db *gorm.DB +} + +// NewMessageRepository creates a new message repository +func NewMessageRepository(db *gorm.DB) interfaces.MessageRepository { + return &messageRepository{ + db: db, + } +} + +// CreateMessage creates a new message +func (r *messageRepository) CreateMessage( + ctx context.Context, message *types.Message, +) (*types.Message, error) { + if err := r.db.WithContext(ctx).Create(message).Error; err != nil { + return nil, err + } + return message, nil +} + +// GetMessage retrieves a message +func (r *messageRepository) GetMessage( + ctx context.Context, sessionID string, messageID string, +) (*types.Message, error) { + var message types.Message + if err := r.db.WithContext(ctx).Where( + "id = ? AND session_id = ?", messageID, sessionID, + ).First(&message).Error; err != nil { + return nil, err + } + return &message, nil +} + +// GetMessagesBySession retrieves all messages for a session with pagination +func (r *messageRepository) GetMessagesBySession( + ctx context.Context, sessionID string, page int, pageSize int, +) ([]*types.Message, error) { + var messages []*types.Message + if err := r.db.WithContext(ctx).Where("session_id = ?", sessionID).Order("created_at ASC"). + Offset((page - 1) * pageSize).Limit(pageSize).Find(&messages).Error; err != nil { + return nil, err + } + return messages, nil +} + +// GetRecentMessagesBySession retrieves recent messages for a session +func (r *messageRepository) GetRecentMessagesBySession( + ctx context.Context, sessionID string, limit int, +) ([]*types.Message, error) { + var messages []*types.Message + if err := r.db.WithContext(ctx).Where( + "session_id = ?", sessionID, + ).Order("created_at DESC").Limit(limit).Find(&messages).Error; err != nil { + if err == gorm.ErrRecordNotFound { + return nil, nil + } + return nil, err + } + slices.SortFunc(messages, func(a, b *types.Message) int { + cmp := a.CreatedAt.Compare(b.CreatedAt) + if cmp == 0 { + if a.Role == "user" { // User messages come first + return -1 + } + return 1 // Assistant messages come last + } + return cmp + }) + return messages, nil +} + +// GetMessagesBySessionBeforeTime retrieves messages from a session created before a specific time +func (r *messageRepository) GetMessagesBySessionBeforeTime( + ctx context.Context, sessionID string, beforeTime time.Time, limit int, +) ([]*types.Message, error) { + var messages []*types.Message + if err := r.db.WithContext(ctx).Where( + "session_id = ? AND created_at < ?", sessionID, beforeTime, + ).Order("created_at DESC").Limit(limit).Find(&messages).Error; err != nil { + return nil, err + } + slices.SortFunc(messages, func(a, b *types.Message) int { + cmp := a.CreatedAt.Compare(b.CreatedAt) + if cmp == 0 { + if a.Role == "user" { // User messages come first + return -1 + } + return 1 // Assistant messages come last + } + return cmp + }) + return messages, nil +} + +// UpdateMessage updates an existing message +func (r *messageRepository) UpdateMessage(ctx context.Context, message *types.Message) error { + return r.db.WithContext(ctx).Model(&types.Message{}).Where( + "id = ? AND session_id = ?", message.ID, message.SessionID, + ).Updates(message).Error +} + +// DeleteMessage deletes a message +func (r *messageRepository) DeleteMessage(ctx context.Context, sessionID string, messageID string) error { + return r.db.WithContext(ctx).Where( + "id = ? AND session_id = ?", messageID, sessionID, + ).Delete(&types.Message{}).Error +} + +// GetFirstMessageOfUser retrieves the first message from a user in a session +func (r *messageRepository) GetFirstMessageOfUser(ctx context.Context, sessionID string) (*types.Message, error) { + var message types.Message + if err := r.db.WithContext(ctx).Where( + "session_id = ? and role = ?", sessionID, "user", + ).Order("created_at ASC").First(&message).Error; err != nil { + return nil, err + } + return &message, nil +} + +// GetMessageByRequestID retrieves a message by request ID +func (r *messageRepository) GetMessageByRequestID( + ctx context.Context, sessionID string, requestID string, +) (*types.Message, error) { + var message types.Message + + result := r.db.WithContext(ctx). + Where("session_id = ? AND request_id = ?", sessionID, requestID). + First(&message) + + if result.Error != nil { + if result.Error == gorm.ErrRecordNotFound { + return nil, nil + } + return nil, result.Error + } + + return &message, nil +} diff --git a/internal/application/repository/model.go b/internal/application/repository/model.go new file mode 100644 index 0000000..568f76c --- /dev/null +++ b/internal/application/repository/model.go @@ -0,0 +1,77 @@ +package repository + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +// modelRepository implements the model repository interface +type modelRepository struct { + db *gorm.DB +} + +// NewModelRepository creates a new model repository +func NewModelRepository(db *gorm.DB) interfaces.ModelRepository { + return &modelRepository{db: db} +} + +// Create creates a new model +func (r *modelRepository) Create(ctx context.Context, m *types.Model) error { + return r.db.WithContext(ctx).Create(m).Error +} + +// GetByID retrieves a model by ID +func (r *modelRepository) GetByID(ctx context.Context, tenantID uint, id string) (*types.Model, error) { + var m types.Model + if err := r.db.WithContext(ctx).Where("id = ?", id).Where( + "tenant_id = ? or tenant_id = 0", tenantID, + ).First(&m).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + return nil, err + } + return &m, nil +} + +// List lists models with optional filtering +func (r *modelRepository) List( + ctx context.Context, tenantID uint, modelType types.ModelType, source types.ModelSource, +) ([]*types.Model, error) { + var models []*types.Model + query := r.db.WithContext(ctx).Where( + "tenant_id = ? or tenant_id = 0", tenantID, + ) + + if modelType != "" { + query = query.Where("type = ?", modelType) + } + + if source != "" { + query = query.Where("source = ?", source) + } + + if err := query.Find(&models).Error; err != nil { + return nil, err + } + + return models, nil +} + +// Update updates a model +func (r *modelRepository) Update(ctx context.Context, m *types.Model) error { + return r.db.WithContext(ctx).Debug().Model(&types.Model{}).Where( + "id = ? AND tenant_id = ?", m.ID, m.TenantID, + ).Updates(m).Error +} + +// Delete deletes a model +func (r *modelRepository) Delete(ctx context.Context, tenantID uint, id string) error { + return r.db.WithContext(ctx).Where( + "id = ? AND tenant_id = ?", id, tenantID, + ).Delete(&types.Model{}).Error +} diff --git a/internal/application/repository/retriever/elasticsearch/structs.go b/internal/application/repository/retriever/elasticsearch/structs.go new file mode 100644 index 0000000..b30b8bb --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/structs.go @@ -0,0 +1,62 @@ +package elasticsearch + +import ( + "maps" + "slices" + + "github.com/Tencent/WeKnora/internal/types" +) + +// VectorEmbedding defines the Elasticsearch document structure for vector embeddings +type VectorEmbedding struct { + Content string `json:"content" gorm:"column:content;not null"` // Text content of the chunk + SourceID string `json:"source_id" gorm:"column:source_id;not null"` // ID of the source document + SourceType int `json:"source_type" gorm:"column:source_type;not null"` // Type of the source document + ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` // Unique ID of the text chunk + KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` // ID of the knowledge item + KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` // ID of the knowledge base + Embedding []float32 `json:"embedding" gorm:"column:embedding;not null"` // Vector embedding of the content +} + +// VectorEmbeddingWithScore extends VectorEmbedding with similarity score +type VectorEmbeddingWithScore struct { + VectorEmbedding + Score float64 // Similarity score from vector search +} + +// ToDBVectorEmbedding converts IndexInfo to Elasticsearch document format +func ToDBVectorEmbedding(embedding *types.IndexInfo, additionalParams map[string]interface{}) *VectorEmbedding { + vector := &VectorEmbedding{ + Content: embedding.Content, + SourceID: embedding.SourceID, + SourceType: int(embedding.SourceType), + ChunkID: embedding.ChunkID, + KnowledgeID: embedding.KnowledgeID, + KnowledgeBaseID: embedding.KnowledgeBaseID, + } + // Add embedding data if available in additionalParams + if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), "embedding") { + if embeddingMap, ok := additionalParams["embedding"].(map[string][]float32); ok { + vector.Embedding = embeddingMap[embedding.SourceID] + } + } + return vector +} + +// FromDBVectorEmbeddingWithScore converts Elasticsearch document to IndexWithScore domain model +func FromDBVectorEmbeddingWithScore(id string, + embedding *VectorEmbeddingWithScore, + matchType types.MatchType, +) *types.IndexWithScore { + return &types.IndexWithScore{ + ID: id, + SourceID: embedding.SourceID, + SourceType: types.SourceType(embedding.SourceType), + ChunkID: embedding.ChunkID, + KnowledgeID: embedding.KnowledgeID, + KnowledgeBaseID: embedding.KnowledgeBaseID, + Content: embedding.Content, + Score: embedding.Score, + MatchType: matchType, + } +} diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go new file mode 100644 index 0000000..cfa2bfe --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -0,0 +1,1030 @@ +// Package v7 implements the Elasticsearch v7 retriever engine repository +// It provides functionality for storing and retrieving embeddings using Elasticsearch v7 +// The repository supports both single and batch operations for saving and deleting embeddings +// It also supports retrieving embeddings using different retrieval methods +// The repository is used to store and retrieve embeddings for different tasks +// It is used to store and retrieve embeddings for different tasks +package v7 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + elasticsearchRetriever "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + typesLocal "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/elastic/go-elasticsearch/v7" + "github.com/elastic/go-elasticsearch/v7/esapi" + "github.com/google/uuid" +) + +type elasticsearchRepository struct { + client *elasticsearch.Client + index string +} + +func NewElasticsearchEngineRepository(client *elasticsearch.Client, + config *config.Config, +) interfaces.RetrieveEngineRepository { + log := logger.GetLogger(context.Background()) + log.Info("[ElasticsearchV7] Initializing Elasticsearch v7 retriever engine repository") + + indexName := os.Getenv("ELASTICSEARCH_INDEX") + if indexName == "" { + log.Warn("[ElasticsearchV7] ELASTICSEARCH_INDEX environment variable not set, using default index name") + indexName = "xwrag_default" + } + + log.Infof("[ElasticsearchV7] Using index: %s", indexName) + res := &elasticsearchRepository{client: client, index: indexName} + return res +} + +func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType { + return typesLocal.ElasticsearchRetrieverEngineType +} + +func (e *elasticsearchRepository) Support() []typesLocal.RetrieverType { + return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType} +} + +// EstimateStorageSize 估算存储空间大小 +func (e *elasticsearchRepository) EstimateStorageSize(ctx context.Context, + indexInfoList []*typesLocal.IndexInfo, params map[string]any, +) int64 { + log := logger.GetLogger(ctx) + log.Infof("[ElasticsearchV7] Estimating storage size for %d indices", len(indexInfoList)) + + // 计算总存储大小 + var totalStorageSize int64 = 0 + for _, indexInfo := range indexInfoList { + embeddingDB := elasticsearchRetriever.ToDBVectorEmbedding(indexInfo, params) + // 计算单个文档的存储大小并累加 + totalStorageSize += e.calculateStorageSize(embeddingDB) + } + + // 记录存储大小 + log.Infof("[ElasticsearchV7] Estimated storage size: %d bytes (%d MB) for %d indices", + totalStorageSize, totalStorageSize/(1024*1024), len(indexInfoList)) + return totalStorageSize +} + +// 计算单个索引的存储占用大小(Bytes) +func (e *elasticsearchRepository) calculateStorageSize(embedding *elasticsearchRetriever.VectorEmbedding) int64 { + // 1. 文本内容大小 + contentSizeBytes := int64(len(embedding.Content)) + + // 2. 向量存储大小 + var vectorSizeBytes int64 = 0 + if embedding.Embedding != nil { + // 4字节/维度 (全精度浮点数) + vectorSizeBytes = int64(len(embedding.Embedding) * 4) + } + + // 3. 元数据大小 (ID、时间戳等固定开销) + metadataSizeBytes := int64(250) // 约250字节的元数据 + + // 4. 索引开销 (ES索引的放大系数约为1.5) + indexOverheadBytes := (contentSizeBytes + vectorSizeBytes) * 5 / 10 + + // 总大小 (字节) + totalSizeBytes := contentSizeBytes + vectorSizeBytes + metadataSizeBytes + indexOverheadBytes + + return totalSizeBytes +} + +// Save 保存索引 +func (e *elasticsearchRepository) Save(ctx context.Context, + embedding *typesLocal.IndexInfo, additionalParams map[string]any, +) error { + log := logger.GetLogger(ctx) + log.Debugf("[ElasticsearchV7] Saving index for chunk ID: %s", embedding.ChunkID) + + embeddingDB := elasticsearchRetriever.ToDBVectorEmbedding(embedding, additionalParams) + if len(embeddingDB.Embedding) == 0 { + err := fmt.Errorf("empty embedding vector for chunk ID: %s", embedding.ChunkID) + log.Errorf("[ElasticsearchV7] %v", err) + return err + } + + docBytes, err := json.Marshal(embeddingDB) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to marshal embedding: %v", err) + return err + } + + docID := uuid.New().String() + log.Debugf("[ElasticsearchV7] Creating document with ID: %s for chunk ID: %s", docID, embedding.ChunkID) + + resp, err := e.client.Create( + e.index, + docID, + bytes.NewReader(docBytes), + e.client.Create.WithContext(ctx), + ) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to create document: %v", err) + return err + } + defer resp.Body.Close() + + if resp.IsError() { + log.Errorf("[ElasticsearchV7] Failed to index document: %s", resp.String()) + return fmt.Errorf("failed to index document: %s", resp.String()) + } + + log.Infof( + "[ElasticsearchV7] Successfully saved index for chunk ID: %s with document ID: %s", + embedding.ChunkID, docID, + ) + return nil +} + +// BatchSave performs bulk indexing of multiple embeddings in Elasticsearch +// It constructs a bulk request with index operations for each embedding +// Returns error if any operation fails during the bulk indexing process +func (e *elasticsearchRepository) BatchSave(ctx context.Context, + embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any, +) error { + log := logger.GetLogger(ctx) + if len(embeddingList) == 0 { + log.Warn("[ElasticsearchV7] Empty list provided to BatchSave, skipping") + return nil + } + + log.Infof("[ElasticsearchV7] Batch saving %d indices", len(embeddingList)) + + // Prepare bulk request body + body, processedCount, err := e.prepareBulkRequestBody(ctx, embeddingList, additionalParams) + if err != nil { + return err + } + + if processedCount == 0 { + log.Warn("[ElasticsearchV7] No valid documents to index after filtering, skipping bulk request") + return nil + } + + // Execute bulk request + log.Debugf("[ElasticsearchV7] Executing bulk request with %d documents", processedCount) + resp, err := e.client.Bulk( + body, + e.client.Bulk.WithIndex(e.index), + e.client.Bulk.WithContext(ctx), + ) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to execute bulk index operation: %v", err) + return err + } + defer resp.Body.Close() + + // Process bulk response + err = e.processBulkResponse(ctx, resp, len(embeddingList)) + if err != nil { + return err + } + + log.Infof("[ElasticsearchV7] Successfully batch saved %d indices", processedCount) + return nil +} + +// prepareBulkRequestBody prepares the bulk request body for batch indexing +func (e *elasticsearchRepository) prepareBulkRequestBody(ctx context.Context, + embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any, +) (*bytes.Buffer, int, error) { + log := logger.GetLogger(ctx) + body := &bytes.Buffer{} + processedCount := 0 + + // Prepare bulk request body + for _, embedding := range embeddingList { + // Convert to Elasticsearch document format + embeddingDB := elasticsearchRetriever.ToDBVectorEmbedding(embedding, additionalParams) + + // Generate document ID and metadata line + docID := uuid.New().String() + meta := []byte(fmt.Sprintf(`{ "index" : { "_id" : "%s" } }%s`, docID, "\n")) + + // Marshal document to JSON + data, err := json.Marshal(embeddingDB) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to marshal embedding for chunk ID %s: %v", embedding.ChunkID, err) + return nil, 0, err + } + + // Append newline and add to bulk request body + data = append(data, "\n"...) + body.Grow(len(meta) + len(data)) + body.Write(meta) + body.Write(data) + processedCount++ + + log.Debugf("[ElasticsearchV7] Added document for chunk ID: %s to bulk request", embedding.ChunkID) + } + + return body, processedCount, nil +} + +// processBulkResponse processes the response from a bulk indexing operation +func (e *elasticsearchRepository) processBulkResponse(ctx context.Context, + resp *esapi.Response, totalDocuments int, +) error { + log := logger.GetLogger(ctx) + + // Check for bulk operation errors + if resp.IsError() { + log.Errorf("[ElasticsearchV7] Bulk operation failed: %s", resp.String()) + return fmt.Errorf("failed to index documents: %s", resp.String()) + } + + // Parse bulk response to check for individual document errors + var bulkResponse map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&bulkResponse); err != nil { + log.Warnf("[ElasticsearchV7] Could not parse bulk response: %v", err) + return nil + } + + // Check for errors in individual operations + if hasErrors, ok := bulkResponse["errors"].(bool); ok && hasErrors { + errorCount := e.countBulkErrors(ctx, bulkResponse, totalDocuments) + if errorCount > 0 { + log.Warnf("[ElasticsearchV7] %d/%d documents failed to index", errorCount, totalDocuments) + } + } + + return nil +} + +// countBulkErrors counts the number of errors in a bulk response +func (e *elasticsearchRepository) countBulkErrors(ctx context.Context, + bulkResponse map[string]interface{}, totalDocuments int, +) int { + log := logger.GetLogger(ctx) + log.Warn("[ElasticsearchV7] Bulk operation completed with some errors") + + errorCount := 0 + if items, ok := bulkResponse["items"].([]interface{}); ok { + for _, item := range items { + if itemMap, ok := item.(map[string]interface{}); ok { + if indexResp, ok := itemMap["index"].(map[string]interface{}); ok { + if indexResp["error"] != nil { + errorCount++ + log.Errorf("[ElasticsearchV7] Item error: %v", indexResp["error"]) + } + } + } + } + } + + return errorCount +} + +// DeleteByChunkIDList Delete indices by chunk ID list +func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { + return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList) +} + +// DeleteByKnowledgeIDList Delete indices by knowledge ID list +func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, + knowledgeIDList []string, dimension int, +) error { + return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList) +} + +// deleteByFieldList Delete documents by field value list +func (e *elasticsearchRepository) deleteByFieldList(ctx context.Context, field string, valueList []string) error { + log := logger.GetLogger(ctx) + if len(valueList) == 0 { + log.Warnf("[ElasticsearchV7] Empty %s list provided for deletion, skipping", field) + return nil + } + + log.Infof("[ElasticsearchV7] Deleting indices by %s, count: %d", field, len(valueList)) + ids, err := json.Marshal(valueList) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to marshal %s list: %v", field, err) + return err + } + + query := fmt.Sprintf(`{"query": {"terms": {"%s": %s}}}`, field, ids) + log.Debugf("[ElasticsearchV7] Executing delete by query: %s", query) + + resp, err := e.client.DeleteByQuery( + []string{e.index}, + bytes.NewReader([]byte(query)), + e.client.DeleteByQuery.WithContext(ctx), + ) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to execute delete by query: %v", err) + return err + } + defer resp.Body.Close() + + if resp.IsError() { + errMsg := fmt.Sprintf("failed to delete by query: %s", resp.String()) + log.Errorf("[ElasticsearchV7] %s", errMsg) + return fmt.Errorf(errMsg) + } + + // Try to extract deletion count from response + var deleteResponse map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&deleteResponse); err != nil { + log.Warnf("[ElasticsearchV7] Could not parse delete response: %v", err) + } else { + if deleted, ok := deleteResponse["deleted"].(float64); ok { + log.Infof("[ElasticsearchV7] Successfully deleted %d documents by %s", int(deleted), field) + } else { + log.Infof("[ElasticsearchV7] Successfully deleted documents by %s", field) + } + } + + return nil +} + +// getBaseConds Construct base Elasticsearch query conditions based on retrieval parameters +// It creates MUST conditions for required fields and MUST_NOT conditions for excluded fields +// Returns a JSON string representing the query conditions +func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) string { + // Build MUST conditions (positive filters) + must := make([]string, 0) + if len(params.KnowledgeBaseIDs) > 0 { + ids, _ := json.Marshal(params.KnowledgeBaseIDs) + must = append(must, fmt.Sprintf(`{"terms": {"knowledge_base_id.keyword": %s}}`, ids)) + } + + // Build MUST_NOT conditions (negative filters) + mustNot := make([]string, 0) + if len(params.ExcludeKnowledgeIDs) > 0 { + ids, _ := json.Marshal(params.ExcludeKnowledgeIDs) + mustNot = append(mustNot, fmt.Sprintf(`{"terms": {"knowledge_id.keyword": %s}}`, ids)) + } + if len(params.ExcludeChunkIDs) > 0 { + ids, _ := json.Marshal(params.ExcludeChunkIDs) + mustNot = append(mustNot, fmt.Sprintf(`{"terms": {"chunk_id.keyword": %s}}`, ids)) + } + + // Combine conditions based on presence + switch { + case len(must) == 0 && len(mustNot) == 0: + return "{}" // Empty query if no conditions + case len(must) == 0: + return fmt.Sprintf(`{"bool": {"must_not": [%s]}}`, strings.Join(mustNot, ",")) + case len(mustNot) == 0: + return fmt.Sprintf(`{"bool": {"must": [%s]}}`, strings.Join(must, ",")) + default: + return fmt.Sprintf(`{"bool": {"must": [%s], "must_not": [%s]}}`, + strings.Join(must, ","), strings.Join(mustNot, ",")) + } +} + +func (e *elasticsearchRepository) Retrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Debugf("[ElasticsearchV7] Processing retrieval request of type: %s", params.RetrieverType) + + switch params.RetrieverType { + case typesLocal.KeywordsRetrieverType: + return e.KeywordsRetrieve(ctx, params) + } + + err := fmt.Errorf("invalid retriever type: %v", params.RetrieverType) + log.Errorf("[ElasticsearchV7] %v", err) + return nil, err +} + +// VectorRetrieve Implement vector similarity retrieval +func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Infof("[ElasticsearchV7] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", + len(params.Embedding), params.TopK, params.Threshold) + + // Build search query + query, err := e.buildVectorSearchQuery(ctx, params) + if err != nil { + return nil, err + } + + // Execute search + results, err := e.executeVectorSearch(ctx, query) + if err != nil { + return nil, err + } + + return []*typesLocal.RetrieveResult{ + { + Results: results, + RetrieverEngineType: typesLocal.ElasticsearchRetrieverEngineType, + RetrieverType: typesLocal.VectorRetrieverType, + Error: nil, + }, + }, nil +} + +// buildVectorSearchQuery builds the vector search query JSON +func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context, + params typesLocal.RetrieveParams, +) (string, error) { + log := logger.GetLogger(ctx) + + filter := e.getBaseConds(params) + + // Serialize the query vector + queryVectorJSON, err := json.Marshal(params.Embedding) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to marshal query vector: %v", err) + return "", fmt.Errorf("failed to marshal query embedding: %w", err) + } + + // Construct the script_score query + query := fmt.Sprintf( + `{"query":{"script_score":{"query":{"bool":{"filter":[%s]}}, + "script":{"source":"cosineSimilarity(params.query_vector,'embedding')", + "params":{"query_vector":%s}},"min_score":%f}},"size":%d}`, + filter, + string(queryVectorJSON), + params.Threshold, + params.TopK, + ) + + log.Debugf("[ElasticsearchV7] Executing vector search with query: %s", query) + return query, nil +} + +// executeVectorSearch executes the vector search query +func (e *elasticsearchRepository) executeVectorSearch( + ctx context.Context, + query string, +) ([]*typesLocal.IndexWithScore, error) { + log := logger.GetLogger(ctx) + + response, err := e.client.Search( + e.client.Search.WithIndex(e.index), + e.client.Search.WithBody(strings.NewReader(query)), + e.client.Search.WithContext(ctx), + ) + if err != nil { + log.Errorf("[ElasticsearchV7] Vector search failed: %v", err) + return nil, err + } + defer response.Body.Close() + + results, err := e.processSearchResponse(ctx, response, typesLocal.VectorRetrieverType) + if err != nil { + return nil, err + } + + return results, nil +} + +// KeywordsRetrieve Implement keyword retrieval +func (e *elasticsearchRepository) KeywordsRetrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Infof("[ElasticsearchV7] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK) + + // Build search query + query, err := e.buildKeywordSearchQuery(ctx, params) + if err != nil { + return nil, err + } + + // Execute search + results, err := e.executeKeywordSearch(ctx, query) + if err != nil { + return nil, err + } + + return []*typesLocal.RetrieveResult{ + { + Results: results, + RetrieverEngineType: typesLocal.ElasticsearchRetrieverEngineType, + RetrieverType: typesLocal.KeywordsRetrieverType, + Error: nil, + }, + }, nil +} + +// buildKeywordSearchQuery builds the keyword search query JSON +func (e *elasticsearchRepository) buildKeywordSearchQuery(ctx context.Context, + params typesLocal.RetrieveParams, +) (string, error) { + log := logger.GetLogger(ctx) + content, err := json.Marshal(params.Query) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to marshal query: %v", err) + return "", err + } + + filter := e.getBaseConds(params) + query := fmt.Sprintf( + `{"query": {"bool": {"must": [{"match": {"content": %s}}], "filter": [%s]}}}`, + string(content), filter, + ) + + log.Debugf("[ElasticsearchV7] Executing keyword search with query: %s", query) + return query, nil +} + +// executeKeywordSearch executes the keyword search query +func (e *elasticsearchRepository) executeKeywordSearch( + ctx context.Context, query string, +) ([]*typesLocal.IndexWithScore, error) { + log := logger.GetLogger(ctx) + + response, err := e.client.Search( + e.client.Search.WithIndex(e.index), + e.client.Search.WithBody( + strings.NewReader(query), + ), + e.client.Search.WithContext(ctx), + ) + if err != nil { + log.Errorf("[ElasticsearchV7] Keywords search failed: %v", err) + return nil, err + } + defer response.Body.Close() + + results, err := e.processSearchResponse(ctx, response, typesLocal.KeywordsRetrieverType) + if err != nil { + return nil, err + } + + return results, nil +} + +// processSearchResponse Process search response +func (e *elasticsearchRepository) processSearchResponse(ctx context.Context, + response *esapi.Response, retrieverType typesLocal.RetrieverType, +) ([]*typesLocal.IndexWithScore, error) { + log := logger.GetLogger(ctx) + + if response.IsError() { + errMsg := fmt.Sprintf("failed to retrieve: %s", response.String()) + log.Errorf("[ElasticsearchV7] %s", errMsg) + return nil, errors.New(errMsg) + } + + // Decode response body + rJson, err := e.decodeSearchResponse(ctx, response) + if err != nil { + return nil, err + } + + // Extract hits from response + hitsList, err := e.extractHitsFromResponse(ctx, rJson) + if err != nil { + return nil, err + } + + // Process hits into results + results, err := e.processHits(ctx, hitsList, retrieverType) + if err != nil { + return nil, err + } + + // Log results summary + e.logResultsSummary(ctx, results, retrieverType) + + return results, nil +} + +// decodeSearchResponse decodes the search response body +func (e *elasticsearchRepository) decodeSearchResponse(ctx context.Context, + response *esapi.Response, +) (map[string]any, error) { + log := logger.GetLogger(ctx) + var rJson map[string]any + + if err := json.NewDecoder(response.Body).Decode(&rJson); err != nil { + log.Errorf("[ElasticsearchV7] Failed to decode search response: %v", err) + return nil, err + } + + return rJson, nil +} + +// extractHitsFromResponse extracts the hits list from the response JSON +func (e *elasticsearchRepository) extractHitsFromResponse(ctx context.Context, + rJson map[string]any, +) ([]interface{}, error) { + log := logger.GetLogger(ctx) + + // Extract hits from response + hitsObj, ok := rJson["hits"].(map[string]interface{}) + if !ok { + log.Errorf("[ElasticsearchV7] Invalid search response format: 'hits' object missing") + return nil, fmt.Errorf("invalid search response format") + } + + hitsList, ok := hitsObj["hits"].([]interface{}) + if !ok { + log.Warnf("[ElasticsearchV7] No hits found in search response") + return []interface{}{}, nil + } + + return hitsList, nil +} + +// processHits processes the hits into IndexWithScore results +func (e *elasticsearchRepository) processHits(ctx context.Context, + hitsList []interface{}, retrieverType typesLocal.RetrieverType, +) ([]*typesLocal.IndexWithScore, error) { + log := logger.GetLogger(ctx) + results := make([]*typesLocal.IndexWithScore, 0, len(hitsList)) + + for _, hit := range hitsList { + indexWithScore, err := e.processHit(ctx, hit, retrieverType) + if err != nil { + // Log error but continue processing other hits + log.Warnf("[ElasticsearchV7] Error processing hit: %v", err) + continue + } + + if indexWithScore != nil { + results = append(results, indexWithScore) + } + } + + return results, nil +} + +// processHit processes a single hit into an IndexWithScore +func (e *elasticsearchRepository) processHit(ctx context.Context, + hit interface{}, retrieverType typesLocal.RetrieverType, +) (*typesLocal.IndexWithScore, error) { + log := logger.GetLogger(ctx) + + hitMap, ok := hit.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid hit object format") + } + + // Get document ID + docID, ok := hitMap["_id"].(string) + if !ok { + return nil, fmt.Errorf("hit missing document ID") + } + + // Get document source + sourceObj, ok := hitMap["_source"] + if !ok { + return nil, fmt.Errorf("hit %s missing _source", docID) + } + + // Get score + score, ok := hitMap["_score"].(float64) + if !ok { + return nil, fmt.Errorf("hit %s missing score", docID) + } + + // Convert source to embedding + embedding, err := e.convertSourceToEmbedding(ctx, sourceObj, docID, score) + if err != nil { + return nil, err + } + + result := elasticsearchRetriever.FromDBVectorEmbeddingWithScore( + docID, embedding, typesLocal.MatchTypeKeywords, + ) + + matchType := "keyword" + if retrieverType == typesLocal.VectorRetrieverType { + matchType = "vector" + } + log.Debugf("[ElasticsearchV7] %s search result: id=%s, score=%.4f", matchType, docID, score) + + return result, nil +} + +// convertSourceToEmbedding converts the source object to an embedding +func (e *elasticsearchRepository) convertSourceToEmbedding(ctx context.Context, + sourceObj interface{}, docID string, score float64, +) (*elasticsearchRetriever.VectorEmbeddingWithScore, error) { + log := logger.GetLogger(ctx) + + // Convert source to embedding + var embedding *elasticsearchRetriever.VectorEmbeddingWithScore + sourceBytes, err := json.Marshal(sourceObj) + if err != nil { + log.Warnf("[ElasticsearchV7] Failed to marshal source for hit %s: %v", docID, err) + return nil, fmt.Errorf("failed to marshal source for hit %s: %v", docID, err) + } + + if err := json.Unmarshal(sourceBytes, &embedding); err != nil { + log.Warnf("[ElasticsearchV7] Failed to unmarshal source for hit %s: %v", docID, err) + return nil, fmt.Errorf("failed to unmarshal source for hit %s: %v", docID, err) + } + + embedding.Score = score + return embedding, nil +} + +// logResultsSummary logs a summary of the results +func (e *elasticsearchRepository) logResultsSummary(ctx context.Context, + results []*typesLocal.IndexWithScore, retrieverType typesLocal.RetrieverType, +) { + log := logger.GetLogger(ctx) + + if len(results) == 0 { + if retrieverType == typesLocal.KeywordsRetrieverType { + log.Warnf("[ElasticsearchV7] No keyword matches found") + } else { + log.Warnf("[ElasticsearchV7] No vector matches found that meet threshold") + } + } else { + retrievalType := "Keywords" + if retrieverType == typesLocal.VectorRetrieverType { + retrievalType = "Vector" + } + log.Infof("[ElasticsearchV7] %s retrieval found %d results", retrievalType, len(results)) + log.Debugf("[ElasticsearchV7] Top result score: %.4f", results[0].Score) + } +} + +// CopyIndices Copy index data +func (e *elasticsearchRepository) CopyIndices(ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, +) error { + log := logger.GetLogger(ctx) + log.Infof( + "[ElasticsearchV7] Copying indices from source knowledge base %s to target knowledge base %s, count: %d", + sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), + ) + + if len(sourceToTargetChunkIDMap) == 0 { + log.Warnf("[ElasticsearchV7] Empty mapping, skipping copy") + return nil + } + + // Build query parameters + retrieveParams := typesLocal.RetrieveParams{ + KnowledgeBaseIDs: []string{sourceKnowledgeBaseID}, + } + + // Set batch processing parameters + batchSize := 500 + from := 0 + totalCopied := 0 + + for { + // Query source data batch + hitsList, err := e.querySourceBatch(ctx, retrieveParams, from, batchSize) + if err != nil { + return err + } + + // If no more data, break the loop + if len(hitsList) == 0 { + break + } + + log.Infof("[ElasticsearchV7] Found %d source index data, batch start position: %d", len(hitsList), from) + + // Process the batch and create index information + indexInfoList, err := e.processSourceBatch(ctx, hitsList, sourceToTargetKBIDMap, + sourceToTargetChunkIDMap, targetKnowledgeBaseID) + if err != nil { + return err + } + + // Save processed indices + if len(indexInfoList) > 0 { + err := e.saveCopiedIndices(ctx, indexInfoList) + if err != nil { + return err + } + + totalCopied += len(indexInfoList) + log.Infof("[ElasticsearchV7] Successfully copied batch data, batch size: %d, total copied: %d", + len(indexInfoList), totalCopied) + } + + // Move to next batch + from += len(hitsList) + + // If the number of returned records is less than the request size, it means the last page has been reached + if len(hitsList) < batchSize { + break + } + } + + log.Infof("[ElasticsearchV7] Index copy completed, total copied: %d", totalCopied) + return nil +} + +// querySourceBatch queries a batch of source data +func (e *elasticsearchRepository) querySourceBatch(ctx context.Context, + retrieveParams typesLocal.RetrieveParams, from int, batchSize int, +) ([]interface{}, error) { + log := logger.GetLogger(ctx) + + // Build query request + filter := e.getBaseConds(retrieveParams) + query := fmt.Sprintf(`{ + "query": %s, + "from": %d, + "size": %d + }`, filter, from, batchSize) + + // Execute query + response, err := e.client.Search( + e.client.Search.WithIndex(e.index), + e.client.Search.WithBody(strings.NewReader(query)), + e.client.Search.WithContext(ctx), + ) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to query source index data: %v", err) + return nil, err + } + defer response.Body.Close() + + if response.IsError() { + log.Errorf("[ElasticsearchV7] Failed to query source index data: %s", response.String()) + return nil, fmt.Errorf("failed to query source index data: %s", response.String()) + } + + // 解析搜索结果 + var searchResult map[string]interface{} + if err := json.NewDecoder(response.Body).Decode(&searchResult); err != nil { + log.Errorf("[ElasticsearchV7] Failed to parse query result: %v", err) + return nil, err + } + + // 提取结果列表 + hitsObj, ok := searchResult["hits"].(map[string]interface{}) + if !ok { + log.Errorf("[ElasticsearchV7] Invalid search result format: 'hits' object missing") + return nil, fmt.Errorf("invalid search result format") + } + + hitsList, ok := hitsObj["hits"].([]interface{}) + if !ok || len(hitsList) == 0 { + if from == 0 { + log.Warnf("[ElasticsearchV7] No source index data found") + } + return []interface{}{}, nil + } + + return hitsList, nil +} + +// processSourceBatch processes a batch of source data and creates index information +func (e *elasticsearchRepository) processSourceBatch(ctx context.Context, + hitsList []interface{}, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, +) ([]*typesLocal.IndexInfo, error) { + log := logger.GetLogger(ctx) + + // Prepare index information for batch save + indexInfoList := make([]*typesLocal.IndexInfo, 0, len(hitsList)) + embeddingMap := make(map[string][]float32) + + // Process each hit result + for _, hit := range hitsList { + indexInfo, embeddingVector, err := e.processSingleHit(ctx, hit, + sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID) + if err != nil { + log.Warnf("[ElasticsearchV7] Error processing hit: %v", err) + continue + } + + if indexInfo != nil { + indexInfoList = append(indexInfoList, indexInfo) + if embeddingVector != nil { + embeddingMap[indexInfo.ChunkID] = embeddingVector + } + } + } + + return indexInfoList, nil +} + +// processSingleHit processes a single hit and creates index information +func (e *elasticsearchRepository) processSingleHit(ctx context.Context, + hit interface{}, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, +) (*typesLocal.IndexInfo, []float32, error) { + log := logger.GetLogger(ctx) + + hitMap, ok := hit.(map[string]interface{}) + if !ok { + log.Warnf("[ElasticsearchV7] Invalid hit object format") + return nil, nil, fmt.Errorf("invalid hit object format") + } + + // Get document source + sourceObj, ok := hitMap["_source"].(map[string]interface{}) + if !ok { + log.Warnf("[ElasticsearchV7] Hit missing _source field") + return nil, nil, fmt.Errorf("hit missing _source field") + } + + // Get source ChunkID and corresponding target ChunkID + sourceChunkID, ok := sourceObj["chunk_id"].(string) + if !ok { + log.Warnf("[ElasticsearchV7] Source index data missing chunk_id field") + return nil, nil, fmt.Errorf("source index data missing chunk_id field") + } + + targetChunkID, ok := sourceToTargetChunkIDMap[sourceChunkID] + if !ok { + log.Warnf("[ElasticsearchV7] Source chunk ID %s not found in mapping", sourceChunkID) + return nil, nil, fmt.Errorf("source chunk ID %s not found in mapping", sourceChunkID) + } + + // Get mapped target knowledge ID + sourceKnowledgeID, ok := sourceObj["knowledge_id"].(string) + if !ok { + log.Warnf("[ElasticsearchV7] Source index data missing knowledge_id field") + return nil, nil, fmt.Errorf("source index data missing knowledge_id field") + } + + targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceKnowledgeID] + if !ok { + log.Warnf("[ElasticsearchV7] Source knowledge ID %s not found in mapping", sourceKnowledgeID) + return nil, nil, fmt.Errorf("source knowledge ID %s not found in mapping", sourceKnowledgeID) + } + + // Extract basic content + content, _ := sourceObj["content"].(string) + sourceType := 0 + if st, ok := sourceObj["source_type"].(float64); ok { + sourceType = int(st) + } + + // Extract embedding vector (if exists) + var embedding []float32 + if embeddingInterface, ok := sourceObj["embedding"].([]interface{}); ok { + embedding = make([]float32, len(embeddingInterface)) + for i, v := range embeddingInterface { + if f, ok := v.(float64); ok { + embedding[i] = float32(f) + } + } + log.Debugf("[ElasticsearchV7] Extracted embedding vector with %d dimensions for chunk %s", + len(embedding), targetChunkID) + } + + // Create IndexInfo object + indexInfo := &typesLocal.IndexInfo{ + ChunkID: targetChunkID, + SourceID: targetChunkID, + KnowledgeID: targetKnowledgeID, + KnowledgeBaseID: targetKnowledgeBaseID, + Content: content, + SourceType: typesLocal.SourceType(sourceType), + } + + return indexInfo, embedding, nil +} + +// saveCopiedIndices saves the copied indices +func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context, indexInfoList []*typesLocal.IndexInfo) error { + log := logger.GetLogger(ctx) + + if len(indexInfoList) == 0 { + log.Info("[ElasticsearchV7] No indices to save, skipping") + return nil + } + + // Prepare additional params with embedding map + additionalParams := make(map[string]any) + embeddingMap := make(map[string][]float32) + + // No need to extract embeddings from metadata as they're not stored there + // We'll use the embeddings directly from the embedding map created in processSourceBatch + + if len(embeddingMap) > 0 { + additionalParams["embedding"] = embeddingMap + log.Infof("[ElasticsearchV7] Found %d embeddings to save", len(embeddingMap)) + } + + // Perform batch save + err := e.BatchSave(ctx, indexInfoList, additionalParams) + if err != nil { + log.Errorf("[ElasticsearchV7] Failed to batch save copied indices: %v", err) + return err + } + + log.Infof("[ElasticsearchV7] Successfully saved %d indices", len(indexInfoList)) + return nil +} diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go new file mode 100644 index 0000000..17c9883 --- /dev/null +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -0,0 +1,541 @@ +package v8 + +import ( + "context" + "encoding/json" + "fmt" + "os" + + elasticsearchRetriever "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + typesLocal "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" +) + +// elasticsearchRepository implements the RetrieveEngineRepository interface for Elasticsearch v8 +type elasticsearchRepository struct { + client *elasticsearch.TypedClient // Elasticsearch client instance + index string // Name of the Elasticsearch index to use +} + +// NewElasticsearchEngineRepository creates and initializes a new Elasticsearch v8 repository +// It sets up the index and returns a repository instance ready for use +func NewElasticsearchEngineRepository(client *elasticsearch.TypedClient, + config *config.Config, +) interfaces.RetrieveEngineRepository { + log := logger.GetLogger(context.Background()) + log.Info("[Elasticsearch] Initializing Elasticsearch v8 retriever engine repository") + + // Get index name from environment variable or use default + indexName := os.Getenv("ELASTICSEARCH_INDEX") + if indexName == "" { + log.Warn("[Elasticsearch] ELASTICSEARCH_INDEX environment variable not set, using default index name") + indexName = "xwrag_default" + } + + // Create repository instance and ensure index exists + res := &elasticsearchRepository{client: client, index: indexName} + if err := res.createIndexIfNotExists(context.Background()); err != nil { + log.Errorf("[Elasticsearch] Failed to create index: %v", err) + } else { + log.Info("[Elasticsearch] Successfully initialized repository") + } + return res +} + +// EngineType returns the type of retriever engine (Elasticsearch) +func (e *elasticsearchRepository) EngineType() typesLocal.RetrieverEngineType { + return typesLocal.ElasticsearchRetrieverEngineType +} + +// Support returns the retrieval types supported by this repository (Keywords and Vector) +func (e *elasticsearchRepository) Support() []typesLocal.RetrieverType { + return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType, typesLocal.VectorRetrieverType} +} + +// calculateStorageSize estimates the storage size in bytes for a single index document +func (e *elasticsearchRepository) calculateStorageSize(embedding *elasticsearchRetriever.VectorEmbedding) int64 { + // 1. Content text size + contentSizeBytes := int64(len(embedding.Content)) + + // 2. Vector embedding size + var vectorSizeBytes int64 = 0 + if embedding.Embedding != nil { + // 4 bytes per dimension (full precision float) + vectorSizeBytes = int64(len(embedding.Embedding) * 4) + } + + // 3. Metadata size (IDs, timestamps, and other fixed overhead) + metadataSizeBytes := int64(250) // Approximately 250 bytes of metadata + + // 4. Index overhead (Elasticsearch index expansion factor ~1.5) + indexOverheadBytes := (contentSizeBytes + vectorSizeBytes) * 5 / 10 + + // Total size in bytes + totalSizeBytes := contentSizeBytes + vectorSizeBytes + metadataSizeBytes + indexOverheadBytes + return totalSizeBytes +} + +// EstimateStorageSize calculates the estimated storage size for a list of indices +// Returns the total size in bytes +func (e *elasticsearchRepository) EstimateStorageSize(ctx context.Context, + indexInfoList []*typesLocal.IndexInfo, params map[string]any, +) int64 { + var totalStorageSize int64 = 0 + for _, embedding := range indexInfoList { + embeddingDB := elasticsearchRetriever.ToDBVectorEmbedding(embedding, params) + totalStorageSize += e.calculateStorageSize(embeddingDB) + } + logger.GetLogger(ctx).Infof( + "[Elasticsearch] Storage size for %d indices: %d bytes", len(indexInfoList), totalStorageSize, + ) + return totalStorageSize +} + +// Save stores a single index document in Elasticsearch +// Returns an error if the operation fails +func (e *elasticsearchRepository) Save(ctx context.Context, + embedding *typesLocal.IndexInfo, + additionalParams map[string]any, +) error { + log := logger.GetLogger(ctx) + log.Debugf("[Elasticsearch] Saving index for chunk ID: %s", embedding.ChunkID) + + // Convert to database format + embeddingDB := elasticsearchRetriever.ToDBVectorEmbedding(embedding, additionalParams) + if len(embeddingDB.Embedding) == 0 { + err := fmt.Errorf("empty embedding vector for chunk ID: %s", embedding.ChunkID) + log.Errorf("[Elasticsearch] %v", err) + return err + } + + // Index the document + resp, err := e.client.Index(e.index).Request(embeddingDB).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to save index: %v", err) + return err + } + + log.Infof("[Elasticsearch] Successfully saved index for chunk ID: %s, document ID: %s", embedding.ChunkID, resp.Id_) + return nil +} + +// BatchSave stores multiple index documents in Elasticsearch using bulk API +// Returns an error if the operation fails +func (e *elasticsearchRepository) BatchSave(ctx context.Context, + embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any, +) error { + log := logger.GetLogger(ctx) + if len(embeddingList) == 0 { + log.Warn("[Elasticsearch] Empty list provided to BatchSave, skipping") + return nil + } + + log.Infof("[Elasticsearch] Batch saving %d indices", len(embeddingList)) + indexRequest := e.client.Bulk().Index(e.index) + + // Add each document to the bulk request + for _, embedding := range embeddingList { + embeddingDB := elasticsearchRetriever.ToDBVectorEmbedding(embedding, additionalParams) + err := indexRequest.CreateOp(types.CreateOperation{Index_: &e.index}, embeddingDB) + if err != nil { + log.Errorf("[Elasticsearch] Failed to create bulk operation: %v", err) + return fmt.Errorf("failed to create op: %w", err) + } + log.Debugf("[Elasticsearch] Added chunk ID %s to bulk request", embedding.ChunkID) + } + + // Execute the bulk request + _, err := indexRequest.Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to execute bulk operation: %v", err) + return fmt.Errorf("failed to do bulk: %w", err) + } + + log.Infof("[Elasticsearch] Successfully batch saved %d indices", len(embeddingList)) + return nil +} + +// DeleteByChunkIDList removes documents from the index based on chunk IDs +// Returns an error if the delete operation fails +func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { + log := logger.GetLogger(ctx) + if len(chunkIDList) == 0 { + log.Warn("[Elasticsearch] Empty chunk ID list provided for deletion, skipping") + return nil + } + + log.Infof("[Elasticsearch] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) + // Use DeleteByQuery to delete all documents matching the chunk IDs + _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": chunkIDList}}, + }).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to delete by chunk IDs: %v", err) + return fmt.Errorf("failed to delete by query: %w", err) + } + + log.Infof("[Elasticsearch] Successfully deleted documents by chunk IDs") + return nil +} + +// DeleteByKnowledgeIDList removes documents from the index based on knowledge IDs +// Returns an error if the delete operation fails +func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, + knowledgeIDList []string, dimension int, +) error { + log := logger.GetLogger(ctx) + if len(knowledgeIDList) == 0 { + log.Warn("[Elasticsearch] Empty knowledge ID list provided for deletion, skipping") + return nil + } + + log.Infof("[Elasticsearch] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) + // Use DeleteByQuery to delete all documents matching the knowledge IDs + _, err := e.client.DeleteByQuery(e.index).Query(&types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": knowledgeIDList}}, + }).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to delete by knowledge IDs: %v", err) + return fmt.Errorf("failed to delete by query: %w", err) + } + + log.Infof("[Elasticsearch] Successfully deleted documents by knowledge IDs") + return nil +} + +// getBaseConds creates the base query conditions for retrieval operations +// Returns a slice of Query objects with must and must_not conditions +func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) []types.Query { + must := []types.Query{} + if len(params.KnowledgeBaseIDs) > 0 { + must = append(must, types.Query{Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{ + "knowledge_base_id.keyword": params.KnowledgeBaseIDs, + }, + }}) + } + mustNot := make([]types.Query, 0) + if len(params.ExcludeKnowledgeIDs) > 0 { + mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{"knowledge_id.keyword": params.ExcludeKnowledgeIDs}, + }}) + } + if len(params.ExcludeChunkIDs) > 0 { + mustNot = append(mustNot, types.Query{Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{"chunk_id.keyword": params.ExcludeChunkIDs}, + }}) + } + return []types.Query{{Bool: &types.BoolQuery{Must: must, MustNot: mustNot}}} +} + +// createIndexIfNotExists checks if the specified index exists and creates it if not +// Returns an error if the operation fails +func (e *elasticsearchRepository) createIndexIfNotExists(ctx context.Context) error { + log := logger.GetLogger(ctx) + log.Debugf("[Elasticsearch] Checking if index exists: %s", e.index) + + // Check if index exists + exists, err := e.client.Indices.Exists(e.index).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to check if index exists: %v", err) + return err + } + + if exists { + log.Debugf("[Elasticsearch] Index already exists: %s", e.index) + return nil + } + + // Create index if it doesn't exist + log.Infof("[Elasticsearch] Creating index: %s", e.index) + _, err = e.client.Indices.Create(e.index).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to create index: %v", err) + return err + } + + log.Infof("[Elasticsearch] Index created successfully: %s", e.index) + return nil +} + +// Retrieve dispatches the retrieval operation to the appropriate method based on retriever type +// Returns a slice of RetrieveResult and an error if the operation fails +func (e *elasticsearchRepository) Retrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Debugf("[Elasticsearch] Processing retrieval request of type: %s", params.RetrieverType) + + // Route to appropriate retrieval method + switch params.RetrieverType { + case typesLocal.VectorRetrieverType: + return e.VectorRetrieve(ctx, params) + case typesLocal.KeywordsRetrieverType: + return e.KeywordsRetrieve(ctx, params) + } + + err := fmt.Errorf("invalid retriever type: %v", params.RetrieverType) + log.Errorf("[Elasticsearch] %v", err) + return nil, err +} + +// VectorRetrieve performs vector similarity search using cosine similarity +// Returns a slice of RetrieveResult containing matching documents +func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", + len(params.Embedding), params.TopK, params.Threshold) + + filter := e.getBaseConds(params) + + // Build script scoring query with cosine similarity + queryVectorJSON, err := json.Marshal(params.Embedding) + if err != nil { + log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err) + return nil, fmt.Errorf("failed to marshal query embedding: %w", err) + } + + scoreSource := "cosineSimilarity(params.query_vector, 'embedding')" + minScore := float32(params.Threshold) + scriptScore := &types.ScriptScoreQuery{ + Query: types.Query{Bool: &types.BoolQuery{Filter: filter}}, + Script: types.Script{ + Source: &scoreSource, + Params: map[string]json.RawMessage{ + "query_vector": json.RawMessage(queryVectorJSON), + }, + }, + MinScore: &minScore, + } + + log.Debugf("[Elasticsearch] Executing vector search in index: %s", e.index) + // Execute search with minimum score threshold + response, err := e.client.Search().Index(e.index).Request(&search.Request{ + Query: &types.Query{ScriptScore: scriptScore}, + Size: ¶ms.TopK, + }).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Vector search failed: %v", err) + return nil, err + } + + // Process search results + var results []*typesLocal.IndexWithScore + for _, hit := range response.Hits.Hits { + var embedding *elasticsearchRetriever.VectorEmbeddingWithScore + if err := json.Unmarshal(hit.Source_, &embedding); err != nil { + log.Errorf("[Elasticsearch] Failed to unmarshal search result: %v", err) + return nil, err + } + embedding.Score = float64(*hit.Score_) + results = append(results, + elasticsearchRetriever.FromDBVectorEmbeddingWithScore(*hit.Id_, embedding, typesLocal.MatchTypeEmbedding)) + } + + if len(results) == 0 { + log.Warnf("[Elasticsearch] No vector matches found that meet threshold %.4f", params.Threshold) + } else { + log.Infof("[Elasticsearch] Vector retrieval found %d results", len(results)) + log.Debugf("[Elasticsearch] Top result score: %.4f", results[0].Score) + } + + return []*typesLocal.RetrieveResult{ + { + Results: results, + RetrieverEngineType: typesLocal.ElasticsearchRetrieverEngineType, + RetrieverType: typesLocal.VectorRetrieverType, + Error: nil, + }, + }, nil +} + +// KeywordsRetrieve performs keyword-based search in document content +// Returns a slice of RetrieveResult containing matching documents +func (e *elasticsearchRepository) KeywordsRetrieve(ctx context.Context, + params typesLocal.RetrieveParams, +) ([]*typesLocal.RetrieveResult, error) { + log := logger.GetLogger(ctx) + log.Infof("[Elasticsearch] Performing keywords retrieval with query: %s, topK: %d", params.Query, params.TopK) + + filter := e.getBaseConds(params) + // Build must conditions for content matching + must := []types.Query{ + {Match: map[string]types.MatchQuery{"content": {Query: params.Query}}}, + } + + log.Debugf("[Elasticsearch] Executing keyword search in index: %s", e.index) + response, err := e.client.Search().Index(e.index).Request(&search.Request{ + Query: &types.Query{Bool: &types.BoolQuery{Filter: filter, Must: must}}, + Size: ¶ms.TopK, + }).Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Keywords search failed: %v", err) + return nil, err + } + + // Process search results + var results []*typesLocal.IndexWithScore + for _, hit := range response.Hits.Hits { + var embedding *elasticsearchRetriever.VectorEmbeddingWithScore + if err := json.Unmarshal(hit.Source_, &embedding); err != nil { + log.Errorf("[Elasticsearch] Failed to unmarshal search result: %v", err) + return nil, err + } + embedding.Score = float64(*hit.Score_) + results = append(results, + elasticsearchRetriever.FromDBVectorEmbeddingWithScore(*hit.Id_, embedding, typesLocal.MatchTypeKeywords), + ) + } + + if len(results) == 0 { + log.Warnf("[Elasticsearch] No keyword matches found for query: %s", params.Query) + } else { + log.Infof("[Elasticsearch] Keywords retrieval found %d results", len(results)) + log.Debugf("[Elasticsearch] Top result score: %.4f", results[0].Score) + } + + return []*typesLocal.RetrieveResult{ + { + Results: results, + RetrieverEngineType: typesLocal.ElasticsearchRetrieverEngineType, + RetrieverType: typesLocal.KeywordsRetrieverType, + Error: nil, + }, + }, nil +} + +// CopyIndices 复制索引数据 +func (e *elasticsearchRepository) CopyIndices(ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, +) error { + log := logger.GetLogger(ctx) + log.Infof( + "[Elasticsearch] Copying indices from source knowledge base %s to target knowledge base %s, count: %d", + sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), + ) + + if len(sourceToTargetChunkIDMap) == 0 { + log.Warn("[Elasticsearch] Empty mapping, skipping copy") + return nil + } + + // Build query parameters + params := typesLocal.RetrieveParams{ + KnowledgeBaseIDs: []string{sourceKnowledgeBaseID}, + } + + // Build base query conditions + filter := e.getBaseConds(params) + + // Set batch processing parameters + batchSize := 500 + from := 0 + totalCopied := 0 + + for { + // Execute pagination query + searchResponse, err := e.client.Search().Index(e.index). + Query(&types.Query{Bool: &types.BoolQuery{Filter: filter}}). + From(from). + Size(batchSize). + Do(ctx) + if err != nil { + log.Errorf("[Elasticsearch] Failed to query source index data: %v", err) + return err + } + + hitsCount := len(searchResponse.Hits.Hits) + if hitsCount == 0 { + break + } + + log.Infof("[Elasticsearch] Found %d source index data, batch start position: %d", hitsCount, from) + + // Prepare index list for BatchSave + var indexInfoList []*typesLocal.IndexInfo + + // Collect all embedding vector data for additionalParams + embeddingMap := make(map[string][]float32) + + for _, hit := range searchResponse.Hits.Hits { + // Parse source document + var sourceDoc elasticsearchRetriever.VectorEmbedding + if err := json.Unmarshal(hit.Source_, &sourceDoc); err != nil { + log.Errorf("[Elasticsearch] Failed to parse source index data: %v", err) + continue + } + + // Get mapped target chunk ID and knowledge ID + targetChunkID, ok := sourceToTargetChunkIDMap[sourceDoc.ChunkID] + if !ok { + log.Warnf("[Elasticsearch] Source chunk %s not found in target mapping, skipping", sourceDoc.ChunkID) + continue + } + + targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceDoc.KnowledgeID] + if !ok { + log.Warnf( + "[Elasticsearch] Source knowledge %s not found in target mapping, skipping", + sourceDoc.KnowledgeID, + ) + continue + } + + // Save embedding vector to embeddingMap + if len(sourceDoc.Embedding) > 0 { + embeddingMap[targetChunkID] = sourceDoc.Embedding + } + + // Create new index information + indexInfo := &typesLocal.IndexInfo{ + Content: sourceDoc.Content, + SourceID: targetChunkID, + SourceType: typesLocal.SourceType(sourceDoc.SourceType), + ChunkID: targetChunkID, + KnowledgeID: targetKnowledgeID, + KnowledgeBaseID: targetKnowledgeBaseID, + } + + indexInfoList = append(indexInfoList, indexInfo) + totalCopied++ + } + + // Use BatchSave function to save index + if len(indexInfoList) > 0 { + // Add embedding vector to additional parameters + additionalParams := map[string]any{ + "embedding": embeddingMap, + } + + if err := e.BatchSave(ctx, indexInfoList, additionalParams); err != nil { + log.Errorf("[Elasticsearch] Failed to batch save index: %v", err) + return err + } + + log.Infof("[Elasticsearch] Successfully copied batch data, batch size: %d, total copied: %d", + len(indexInfoList), totalCopied) + } + + // Move to next batch + from += hitsCount + + // If the number of returned records is less than the request size, it means the last page has been reached + if hitsCount < batchSize { + break + } + } + + log.Infof("[Elasticsearch] Index copy completed, total copied: %d", totalCopied) + return nil +} diff --git a/internal/application/repository/retriever/neo4j/repository.go b/internal/application/repository/retriever/neo4j/repository.go new file mode 100644 index 0000000..028086e --- /dev/null +++ b/internal/application/repository/retriever/neo4j/repository.go @@ -0,0 +1,231 @@ +package neo4j + +import ( + "context" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/neo4j/neo4j-go-driver/v6/neo4j" +) + +type Neo4jRepository struct { + driver neo4j.Driver + nodePrefix string +} + +func NewNeo4jRepository(driver neo4j.Driver) interfaces.RetrieveGraphRepository { + return &Neo4jRepository{driver: driver, nodePrefix: "ENTITY"} +} + +func _remove_hyphen(s string) string { + return strings.ReplaceAll(s, "-", "_") +} + +func (n *Neo4jRepository) Labels(namespace types.NameSpace) []string { + res := make([]string, 0) + for _, label := range namespace.Labels() { + res = append(res, n.nodePrefix+_remove_hyphen(label)) + } + return res +} + +func (n *Neo4jRepository) Label(namespace types.NameSpace) string { + labels := n.Labels(namespace) + return strings.Join(labels, ":") +} + +// AddGraph implements interfaces.RetrieveGraphRepository. +func (n *Neo4jRepository) AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error { + if n.driver == nil { + logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH") + return nil + } + for _, graph := range graphs { + if err := n.addGraph(ctx, namespace, graph); err != nil { + return err + } + } + return nil +} + +func (n *Neo4jRepository) addGraph(ctx context.Context, namespace types.NameSpace, graph *types.GraphData) error { + session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + _, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) { + node_import_query := ` + UNWIND $data AS row + CALL apoc.merge.node(row.labels, {name: row.name, kg: row.knowledge_id}, row.props, {}) YIELD node + SET node.chunks = apoc.coll.union(node.chunks, row.chunks) + RETURN distinct 'done' AS result + ` + nodeData := []map[string]interface{}{} + for _, node := range graph.Node { + nodeData = append(nodeData, map[string]interface{}{ + "name": node.Name, + "knowledge_id": namespace.Knowledge, + "props": map[string][]string{"attributes": node.Attributes}, + "chunks": node.Chunks, + "labels": n.Labels(namespace), + }) + } + if _, err := tx.Run(ctx, node_import_query, map[string]interface{}{"data": nodeData}); err != nil { + return nil, fmt.Errorf("failed to create nodes: %v", err) + } + + rel_import_query := ` + UNWIND $data AS row + CALL apoc.merge.node(row.source_labels, {name: row.source, kg: row.knowledge_id}, {}, {}) YIELD node as source + CALL apoc.merge.node(row.target_labels, {name: row.target, kg: row.knowledge_id}, {}, {}) YIELD node as target + CALL apoc.merge.relationship(source, row.type, {}, row.attributes, target) YIELD rel + RETURN distinct 'done' + ` + relData := []map[string]interface{}{} + for _, rel := range graph.Relation { + relData = append(relData, map[string]interface{}{ + "source": rel.Node1, + "target": rel.Node2, + "knowledge_id": namespace.Knowledge, + "type": rel.Type, + "source_labels": n.Labels(namespace), + "target_labels": n.Labels(namespace), + }) + } + if _, err := tx.Run(ctx, rel_import_query, map[string]interface{}{"data": relData}); err != nil { + return nil, fmt.Errorf("failed to create relationships: %v", err) + } + return nil, nil + }) + if err != nil { + logger.Errorf(ctx, "failed to add graph: %v", err) + return err + } + return nil +} + +// DelGraph implements interfaces.RetrieveGraphRepository. +func (n *Neo4jRepository) DelGraph(ctx context.Context, namespaces []types.NameSpace) error { + if n.driver == nil { + logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH") + return nil + } + session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + result, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) { + for _, namespace := range namespaces { + labelExpr := n.Label(namespace) + + deleteRelsQuery := ` + CALL apoc.periodic.iterate( + "MATCH (n:` + labelExpr + ` {kg: $knowledge_id})-[r]-(m:` + labelExpr + ` {kg: $knowledge_id}) RETURN r", + "DELETE r", + {batchSize: 1000, parallel: true, params: {knowledge_id: $knowledge_id}} + ) YIELD batches, total + RETURN total + ` + if _, err := tx.Run(ctx, deleteRelsQuery, map[string]interface{}{"knowledge_id": namespace.Knowledge}); err != nil { + return nil, fmt.Errorf("failed to delete relationships: %v", err) + } + + deleteNodesQuery := ` + CALL apoc.periodic.iterate( + "MATCH (n:` + labelExpr + ` {kg: $knowledge_id}) RETURN n", + "DELETE n", + {batchSize: 1000, parallel: true, params: {knowledge_id: $knowledge_id}} + ) YIELD batches, total + RETURN total + ` + if _, err := tx.Run(ctx, deleteNodesQuery, map[string]interface{}{"knowledge_id": namespace.Knowledge}); err != nil { + return nil, fmt.Errorf("failed to delete nodes: %v", err) + } + } + return nil, nil + }) + if err != nil { + return err + } + logger.Infof(ctx, "delete graph result: %v", result) + return nil +} + +func (n *Neo4jRepository) SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error) { + if n.driver == nil { + logger.Warnf(ctx, "NOT SUPPORT RETRIEVE GRAPH") + return nil, nil + } + session := n.driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeRead}) + defer session.Close(ctx) + + result, err := session.ExecuteRead(ctx, func(tx neo4j.ManagedTransaction) (interface{}, error) { + labelExpr := n.Label(namespace) + query := ` + MATCH (n:` + labelExpr + `)-[r]-(m:` + labelExpr + `) + WHERE ANY(nodeText IN $nodes WHERE n.name CONTAINS nodeText) + RETURN n, r, m + ` + params := map[string]interface{}{"nodes": nodes} + result, err := tx.Run(ctx, query, params) + if err != nil { + return nil, fmt.Errorf("failed to run query: %v", err) + } + + graphData := &types.GraphData{} + nodeSeen := make(map[string]bool) + for result.Next(ctx) { + record := result.Record() + node, _ := record.Get("n") + rel, _ := record.Get("r") + targetNode, _ := record.Get("m") + + nodeData := node.(neo4j.Node) + targetNodeData := targetNode.(neo4j.Node) + + // Convert node to types.Node + for _, n := range []neo4j.Node{nodeData, targetNodeData} { + nameStr := n.Props["name"].(string) + if _, ok := nodeSeen[nameStr]; !ok { + nodeSeen[nameStr] = true + graphData.Node = append(graphData.Node, &types.GraphNode{ + Name: nameStr, + Chunks: listI2listS(n.Props["chunks"].([]interface{})), + Attributes: listI2listS(n.Props["attributes"].([]interface{})), + }) + } + } + + // Convert relationship to types.Relation + relData := rel.(neo4j.Relationship) + graphData.Relation = append(graphData.Relation, &types.GraphRelation{ + Node1: nodeData.Props["name"].(string), + Node2: targetNodeData.Props["name"].(string), + Type: relData.Type, + }) + } + return graphData, nil + }) + if err != nil { + logger.Errorf(ctx, "search node failed: %v", err) + return nil, err + } + return result.(*types.GraphData), nil +} + +func listI2listS(list []any) []string { + result := make([]string, len(list)) + for i, v := range list { + result[i] = fmt.Sprintf("%v", v) + } + return result +} + +func mapI2mapS(prop map[string]any) map[string]string { + attributes := make(map[string]string) + for k, v := range prop { + attributes[k] = fmt.Sprintf("%v", v) + } + return attributes +} diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go new file mode 100644 index 0000000..9d29317 --- /dev/null +++ b/internal/application/repository/retriever/postgres/repository.go @@ -0,0 +1,390 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + + "github.com/Tencent/WeKnora/internal/common" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/pgvector/pgvector-go" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// pgRepository implements PostgreSQL-based retrieval operations +type pgRepository struct { + db *gorm.DB // Database connection +} + +// NewPostgresRetrieveEngineRepository creates a new PostgreSQL retriever repository +func NewPostgresRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRepository { + logger.GetLogger(context.Background()).Info("[Postgres] Initializing PostgreSQL retriever engine repository") + return &pgRepository{db: db} +} + +// EngineType returns the retriever engine type (PostgreSQL) +func (r *pgRepository) EngineType() types.RetrieverEngineType { + return types.PostgresRetrieverEngineType +} + +// Support returns supported retriever types (keywords and vector) +func (r *pgRepository) Support() []types.RetrieverType { + return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType} +} + +// calculateIndexStorageSize calculates storage size for a single index entry +func (g *pgRepository) calculateIndexStorageSize(embeddingDB *pgVector) int64 { + // 1. Text content size + contentSizeBytes := int64(len(embeddingDB.Content)) + + // 2. Vector storage size (2 bytes per dimension for half-precision float) + var vectorSizeBytes int64 = 0 + if embeddingDB.Dimension > 0 { + vectorSizeBytes = int64(embeddingDB.Dimension * 2) + } + + // 3. Metadata size (fixed overhead for IDs, timestamps etc.) + metadataSizeBytes := int64(200) + + // 4. Index overhead (HNSW index is ~2x vector size) + indexOverheadBytes := vectorSizeBytes * 2 + + // Total size in bytes + totalSizeBytes := contentSizeBytes + vectorSizeBytes + metadataSizeBytes + indexOverheadBytes + + return totalSizeBytes +} + +// EstimateStorageSize estimates total storage size for multiple indices +func (g *pgRepository) EstimateStorageSize( + ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any, +) int64 { + var totalStorageSize int64 = 0 + for _, indexInfo := range indexInfoList { + embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams) + totalStorageSize += g.calculateIndexStorageSize(embeddingDB) + } + logger.GetLogger(ctx).Infof( + "[Postgres] Estimated storage size for %d indices: %d bytes", + len(indexInfoList), totalStorageSize, + ) + return totalStorageSize +} + +// Save stores a single index entry +func (g *pgRepository) Save(ctx context.Context, indexInfo *types.IndexInfo, additionalParams map[string]any) error { + logger.GetLogger(ctx).Debugf("[Postgres] Saving index for source ID: %s", indexInfo.SourceID) + embeddingDB := toDBVectorEmbedding(indexInfo, additionalParams) + err := g.db.WithContext(ctx).Create(embeddingDB).Error + if err != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Failed to save index: %v", err) + return err + } + logger.GetLogger(ctx).Infof("[Postgres] Successfully saved index for source ID: %s", indexInfo.SourceID) + return nil +} + +// BatchSave stores multiple index entries in batch +func (g *pgRepository) BatchSave( + ctx context.Context, indexInfoList []*types.IndexInfo, additionalParams map[string]any, +) error { + logger.GetLogger(ctx).Infof("[Postgres] Batch saving %d indices", len(indexInfoList)) + indexInfoDBList := make([]*pgVector, len(indexInfoList)) + for i := range indexInfoList { + indexInfoDBList[i] = toDBVectorEmbedding(indexInfoList[i], additionalParams) + } + err := g.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(indexInfoDBList).Error + if err != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Batch save failed: %v", err) + return err + } + logger.GetLogger(ctx).Infof("[Postgres] Successfully batch saved %d indices", len(indexInfoList)) + return nil +} + +// DeleteByChunkIDList deletes indices by chunk IDs +func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { + logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) + result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{}) + if result.Error != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by chunk IDs: %v", result.Error) + return result.Error + } + logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by chunk IDs", result.RowsAffected) + return nil +} + +// DeleteByKnowledgeIDList deletes indices by knowledge IDs +func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error { + logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) + result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{}) + if result.Error != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Failed to delete indices by knowledge IDs: %v", result.Error) + return result.Error + } + logger.GetLogger(ctx).Infof("[Postgres] Successfully deleted %d indices by knowledge IDs", result.RowsAffected) + return nil +} + +// Retrieve handles retrieval requests and routes to appropriate method +func (g *pgRepository) Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) { + logger.GetLogger(ctx).Debugf("[Postgres] Processing retrieval request of type: %s", params.RetrieverType) + switch params.RetrieverType { + case types.KeywordsRetrieverType: + return g.KeywordsRetrieve(ctx, params) + case types.VectorRetrieverType: + return g.VectorRetrieve(ctx, params) + } + err := errors.New("invalid retriever type") + logger.GetLogger(ctx).Errorf("[Postgres] %v: %s", err, params.RetrieverType) + return nil, err +} + +// KeywordsRetrieve performs keyword-based search using PostgreSQL full-text search +func (g *pgRepository) KeywordsRetrieve(ctx context.Context, + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { + logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK) + conds := make([]clause.Expression, 0) + if len(params.KnowledgeBaseIDs) > 0 { + logger.GetLogger(ctx).Debugf("[Postgres] Filtering by knowledge base IDs: %v", params.KnowledgeBaseIDs) + conds = append(conds, clause.Expr{ + SQL: fmt.Sprintf("knowledge_base_id @@@ 'in (%s)'", common.StringSliceJoin(params.KnowledgeBaseIDs)), + }) + } + conds = append(conds, clause.Expr{ + SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)", + Vars: []interface{}{params.Query}, + }) + conds = append(conds, clause.OrderBy{Columns: []clause.OrderByColumn{ + {Column: clause.Column{Name: "score"}, Desc: true}, + }}) + + var embeddingDBList []pgVectorWithScore + err := g.db.WithContext(ctx).Clauses(conds...).Debug(). + Select([]string{ + "paradedb.score(id) as score", + "id", + "content", + "source_id", + "source_type", + "chunk_id", + "knowledge_id", + "knowledge_base_id", + }). + Limit(int(params.TopK)). + Find(&embeddingDBList).Error + + if err == gorm.ErrRecordNotFound { + logger.GetLogger(ctx).Warnf("[Postgres] No records found for keywords query: %s", params.Query) + return nil, nil + } + if err != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Keywords retrieval failed: %v", err) + return nil, err + } + + logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList)) + results := make([]*types.IndexWithScore, len(embeddingDBList)) + for i := range embeddingDBList { + results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords) + logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f", + i, results[i].ChunkID, results[i].Score) + } + return []*types.RetrieveResult{ + { + Results: results, + RetrieverEngineType: types.PostgresRetrieverEngineType, + RetrieverType: types.KeywordsRetrieverType, + Error: nil, + }, + }, nil +} + +// VectorRetrieve performs vector similarity search using pgvector +func (g *pgRepository) VectorRetrieve(ctx context.Context, + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { + logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", + len(params.Embedding), params.TopK, params.Threshold) + + conds := make([]clause.Expression, 0) + if len(params.KnowledgeBaseIDs) > 0 { + logger.GetLogger(ctx).Debugf( + "[Postgres] Filtering vector search by knowledge base IDs: %v", + params.KnowledgeBaseIDs, + ) + conds = append(conds, clause.IN{ + Column: "knowledge_base_id", + Values: common.ToInterfaceSlice(params.KnowledgeBaseIDs), + }) + } + // <=> Cosine similarity operator + // <-> L2 distance operator + // <#> Inner product operator + dimension := len(params.Embedding) + conds = append(conds, clause.Expr{SQL: "dimension = ?", Vars: []interface{}{dimension}}) + conds = append(conds, clause.Expr{ + SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec < ?", dimension), + Vars: []interface{}{pgvector.NewHalfVector(params.Embedding), 1 - params.Threshold}, + }) + conds = append(conds, clause.OrderBy{Expression: clause.Expr{ + SQL: fmt.Sprintf("embedding::halfvec(%d) <=> ?::halfvec", dimension), + Vars: []interface{}{pgvector.NewHalfVector(params.Embedding)}, + }}) + + var embeddingDBList []pgVectorWithScore + + err := g.db.WithContext(ctx).Clauses(conds...). + Select(fmt.Sprintf( + "id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, "+ + "(1 - (embedding::halfvec(%d) <=> ?::halfvec)) as score", + dimension, + ), pgvector.NewHalfVector(params.Embedding)). + Limit(int(params.TopK)). + Find(&embeddingDBList).Error + + if err == gorm.ErrRecordNotFound { + logger.GetLogger(ctx).Warnf("[Postgres] No vector matches found that meet threshold %.4f", params.Threshold) + return nil, nil + } + if err != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Vector retrieval failed: %v", err) + return nil, err + } + + logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList)) + results := make([]*types.IndexWithScore, len(embeddingDBList)) + for i := range embeddingDBList { + results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding) + logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f", + i, results[i].ChunkID, results[i].Score) + } + return []*types.RetrieveResult{ + { + Results: results, + RetrieverEngineType: types.PostgresRetrieverEngineType, + RetrieverType: types.VectorRetrieverType, + Error: nil, + }, + }, nil +} + +// CopyIndices copies index data +func (g *pgRepository) CopyIndices(ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, +) error { + logger.GetLogger(ctx).Infof( + "[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d", + sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), + ) + + if len(sourceToTargetChunkIDMap) == 0 { + logger.GetLogger(ctx).Warnf("[Postgres] Mapping is empty, no need to copy") + return nil + } + + // Batch processing parameters + batchSize := 500 // Number of records to process per batch + offset := 0 // Offset for pagination + totalCopied := 0 // Total number of copied records + + for { + // Paginated query for source data + var sourceVectors []*pgVector + if err := g.db.WithContext(ctx). + Where("knowledge_base_id = ?", sourceKnowledgeBaseID). + Limit(batchSize). + Offset(offset). + Find(&sourceVectors).Error; err != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Failed to query source index data: %v", err) + return err + } + + // If no more data, exit the loop + if len(sourceVectors) == 0 { + if offset == 0 { + logger.GetLogger(ctx).Warnf("[Postgres] No source index data found") + } + break + } + + batchCount := len(sourceVectors) + logger.GetLogger(ctx).Infof( + "[Postgres] Found %d source index data, batch start position: %d", + batchCount, offset, + ) + + // Create target vector index + targetVectors := make([]*pgVector, 0, batchCount) + for _, sourceVector := range sourceVectors { + // Get the mapped target chunk ID + targetChunkID, ok := sourceToTargetChunkIDMap[sourceVector.ChunkID] + if !ok { + logger.GetLogger(ctx).Warnf( + "[Postgres] Source chunk %s not found in target chunk mapping, skipping", + sourceVector.ChunkID, + ) + continue + } + + // Get the mapped target knowledge ID + targetKnowledgeID, ok := sourceToTargetKBIDMap[sourceVector.KnowledgeID] + if !ok { + logger.GetLogger(ctx).Warnf( + "[Postgres] Source knowledge %s not found in target knowledge mapping, skipping", + sourceVector.KnowledgeID, + ) + continue + } + + // Create new vector index, copy the content and vector of the source index + targetVector := &pgVector{ + Content: sourceVector.Content, + SourceID: targetChunkID, // Update to target chunk ID + SourceType: sourceVector.SourceType, + ChunkID: targetChunkID, // Update to target chunk ID + KnowledgeID: targetKnowledgeID, // Update to target knowledge ID + KnowledgeBaseID: targetKnowledgeBaseID, // Update to target knowledge base ID + Dimension: sourceVector.Dimension, + Embedding: sourceVector.Embedding, // Copy the vector embedding directly, avoid recalculation + } + + targetVectors = append(targetVectors, targetVector) + } + + // Batch insert target vector index + if len(targetVectors) > 0 { + if err := g.db.WithContext(ctx). + Clauses(clause.OnConflict{DoNothing: true}).Create(targetVectors).Error; err != nil { + logger.GetLogger(ctx).Errorf("[Postgres] Failed to batch create target index: %v", err) + return err + } + + totalCopied += len(targetVectors) + logger.GetLogger(ctx).Infof( + "[Postgres] Successfully copied batch data, batch size: %d, total copied: %d", + len(targetVectors), + totalCopied, + ) + } + + // Move to the next batch + offset += batchCount + + // If the number of returned records is less than the requested size, it means the last page has been reached + if batchCount < batchSize { + break + } + } + + logger.GetLogger(ctx).Infof("[Postgres] Index copying completed, total copied: %d", totalCopied) + return nil +} diff --git a/internal/application/repository/retriever/postgres/structs.go b/internal/application/repository/retriever/postgres/structs.go new file mode 100644 index 0000000..45fbc0e --- /dev/null +++ b/internal/application/repository/retriever/postgres/structs.go @@ -0,0 +1,88 @@ +package postgres + +import ( + "maps" + "slices" + "strconv" + "time" + + "github.com/Tencent/WeKnora/internal/common" + "github.com/Tencent/WeKnora/internal/types" + "github.com/pgvector/pgvector-go" +) + +// pgVector defines the database model for vector embeddings storage +type pgVector struct { + ID uint `json:"id" gorm:"primarykey"` + CreatedAt time.Time `json:"created_at" gorm:"column:created_at"` + UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"` + SourceID string `json:"source_id" gorm:"column:source_id;not null"` + SourceType int `json:"source_type" gorm:"column:source_type;not null"` + ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` + KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` + KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` + Content string `json:"content" gorm:"column:content;not null"` + Dimension int `json:"dimension" gorm:"column:dimension;not null"` + Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"` +} + +// pgVectorWithScore extends pgVector with similarity score field +type pgVectorWithScore struct { + ID uint `json:"id" gorm:"primarykey"` + CreatedAt time.Time `json:"created_at" gorm:"column:created_at"` + UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"` + SourceID string `json:"source_id" gorm:"column:source_id;not null"` + SourceType int `json:"source_type" gorm:"column:source_type;not null"` + ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` + KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` + KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` + Content string `json:"content" gorm:"column:content;not null"` + Dimension int `json:"dimension" gorm:"column:dimension;not null"` + Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"` + Score float64 `json:"score" gorm:"column:score"` +} + +// TableName specifies the database table name for pgVectorWithScore +func (pgVectorWithScore) TableName() string { + return "embeddings" +} + +// TableName specifies the database table name for pgVector +func (pgVector) TableName() string { + return "embeddings" +} + +// toDBVectorEmbedding converts IndexInfo to pgVector database model +func toDBVectorEmbedding(indexInfo *types.IndexInfo, additionalParams map[string]any) *pgVector { + pgVector := &pgVector{ + SourceID: indexInfo.SourceID, + SourceType: int(indexInfo.SourceType), + ChunkID: indexInfo.ChunkID, + KnowledgeID: indexInfo.KnowledgeID, + KnowledgeBaseID: indexInfo.KnowledgeBaseID, + Content: common.CleanInvalidUTF8(indexInfo.Content), + } + // Add embedding data if available in additionalParams + if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), "embedding") { + if embeddingMap, ok := additionalParams["embedding"].(map[string][]float32); ok { + pgVector.Embedding = pgvector.NewHalfVector(embeddingMap[indexInfo.SourceID]) + pgVector.Dimension = len(pgVector.Embedding.Slice()) + } + } + return pgVector +} + +// fromDBVectorEmbeddingWithScore converts pgVectorWithScore to IndexWithScore domain model +func fromDBVectorEmbeddingWithScore(embedding *pgVectorWithScore, matchType types.MatchType) *types.IndexWithScore { + return &types.IndexWithScore{ + ID: strconv.FormatInt(int64(embedding.ID), 10), + SourceID: embedding.SourceID, + SourceType: types.SourceType(embedding.SourceType), + ChunkID: embedding.ChunkID, + KnowledgeID: embedding.KnowledgeID, + KnowledgeBaseID: embedding.KnowledgeBaseID, + Content: embedding.Content, + Score: embedding.Score, + MatchType: matchType, + } +} diff --git a/internal/application/repository/session.go b/internal/application/repository/session.go new file mode 100644 index 0000000..b0f12fe --- /dev/null +++ b/internal/application/repository/session.go @@ -0,0 +1,89 @@ +package repository + +import ( + "context" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +// sessionRepository implements the SessionRepository interface +type sessionRepository struct { + db *gorm.DB +} + +// NewSessionRepository creates a new session repository instance +func NewSessionRepository(db *gorm.DB) interfaces.SessionRepository { + return &sessionRepository{db: db} +} + +// Create creates a new session +func (r *sessionRepository) Create(ctx context.Context, session *types.Session) (*types.Session, error) { + session.CreatedAt = time.Now() + session.UpdatedAt = time.Now() + if err := r.db.WithContext(ctx).Create(session).Error; err != nil { + return nil, err + } + // Return the session with generated ID + return session, nil +} + +// Get retrieves a session by ID +func (r *sessionRepository) Get(ctx context.Context, tenantID uint, id string) (*types.Session, error) { + var session types.Session + err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).First(&session, "id = ?", id).Error + if err != nil { + return nil, err + } + return &session, nil +} + +// GetByTenantID retrieves all sessions for a tenant +func (r *sessionRepository) GetByTenantID(ctx context.Context, tenantID uint) ([]*types.Session, error) { + var sessions []*types.Session + err := r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Order("created_at DESC").Find(&sessions).Error + if err != nil { + return nil, err + } + return sessions, nil +} + +// GetPagedByTenantID retrieves sessions for a tenant with pagination +func (r *sessionRepository) GetPagedByTenantID( + ctx context.Context, tenantID uint, page *types.Pagination, +) ([]*types.Session, int64, error) { + var sessions []*types.Session + var total int64 + + // First query the total count + err := r.db.WithContext(ctx).Model(&types.Session{}).Where("tenant_id = ?", tenantID).Count(&total).Error + if err != nil { + return nil, 0, err + } + + // Then query the paginated data + err = r.db.WithContext(ctx). + Where("tenant_id = ?", tenantID). + Order("created_at DESC"). + Offset(page.Offset()). + Limit(page.Limit()). + Find(&sessions).Error + if err != nil { + return nil, 0, err + } + + return sessions, total, nil +} + +// Update updates a session +func (r *sessionRepository) Update(ctx context.Context, session *types.Session) error { + session.UpdatedAt = time.Now() + return r.db.WithContext(ctx).Where("tenant_id = ?", session.TenantID).Save(session).Error +} + +// Delete deletes a session +func (r *sessionRepository) Delete(ctx context.Context, tenantID uint, id string) error { + return r.db.WithContext(ctx).Where("tenant_id = ?", tenantID).Delete(&types.Session{}, "id = ?", id).Error +} diff --git a/internal/application/repository/tenant.go b/internal/application/repository/tenant.go new file mode 100644 index 0000000..2faf007 --- /dev/null +++ b/internal/application/repository/tenant.go @@ -0,0 +1,82 @@ +package repository + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +var ( + ErrTenantNotFound = errors.New("tenant not found") + ErrTenantHasKnowledgeBase = errors.New("tenant has associated knowledge bases") +) + +// tenantRepository implements tenant repository interface +type tenantRepository struct { + db *gorm.DB +} + +// NewTenantRepository creates a new tenant repository +func NewTenantRepository(db *gorm.DB) interfaces.TenantRepository { + return &tenantRepository{db: db} +} + +// CreateTenant creates tenant +func (r *tenantRepository) CreateTenant(ctx context.Context, tenant *types.Tenant) error { + return r.db.WithContext(ctx).Create(tenant).Error +} + +// GetTenantByID gets tenant by ID +func (r *tenantRepository) GetTenantByID(ctx context.Context, id uint) (*types.Tenant, error) { + var tenant types.Tenant + if err := r.db.WithContext(ctx).Where("id = ?", id).First(&tenant).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrTenantNotFound + } + return nil, err + } + return &tenant, nil +} + +// ListTenants lists all tenants +func (r *tenantRepository) ListTenants(ctx context.Context) ([]*types.Tenant, error) { + var tenants []*types.Tenant + if err := r.db.WithContext(ctx).Order("created_at DESC").Find(&tenants).Error; err != nil { + return nil, err + } + return tenants, nil +} + +// UpdateTenant updates tenant +func (r *tenantRepository) UpdateTenant(ctx context.Context, tenant *types.Tenant) error { + return r.db.WithContext(ctx).Model(&types.Tenant{}).Where("id = ?", tenant.ID).Updates(tenant).Error +} + +// DeleteTenant deletes tenant +func (r *tenantRepository) DeleteTenant(ctx context.Context, id uint) error { + return r.db.WithContext(ctx).Where("id = ?", id).Delete(&types.Tenant{}).Error +} + +func (r *tenantRepository) AdjustStorageUsed(ctx context.Context, tenantID uint, delta int64) error { + return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var tenant types.Tenant + // 使用悲观锁确保并发安全 + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).First(&tenant, tenantID).Error; err != nil { + return err + } + + tenant.StorageUsed += delta + // 保存更新并验证业务规则 + if tenant.StorageUsed < 0 { + logger.Error(ctx, "tenant storage used is negative %s: %d", tenant.ID, tenant.StorageUsed) + tenant.StorageUsed = 0 + } + + return tx.Save(&tenant).Error + }) +} diff --git a/internal/application/repository/user.go b/internal/application/repository/user.go new file mode 100644 index 0000000..94d0d62 --- /dev/null +++ b/internal/application/repository/user.go @@ -0,0 +1,154 @@ +package repository + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +var ( + ErrUserNotFound = errors.New("user not found") + ErrUserAlreadyExists = errors.New("user already exists") + ErrTokenNotFound = errors.New("token not found") +) + +// userRepository implements user repository interface +type userRepository struct { + db *gorm.DB +} + +// NewUserRepository creates a new user repository +func NewUserRepository(db *gorm.DB) interfaces.UserRepository { + return &userRepository{db: db} +} + +// CreateUser creates a user +func (r *userRepository) CreateUser(ctx context.Context, user *types.User) error { + logger.Infof(ctx, "Creating user in database: %s", user.Email) + return r.db.WithContext(ctx).Create(user).Error +} + +// GetUserByID gets a user by ID +func (r *userRepository) GetUserByID(ctx context.Context, id string) (*types.User, error) { + var user types.User + if err := r.db.WithContext(ctx).Where("id = ?", id).First(&user).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrUserNotFound + } + return nil, err + } + return &user, nil +} + +// GetUserByEmail gets a user by email +func (r *userRepository) GetUserByEmail(ctx context.Context, email string) (*types.User, error) { + var user types.User + if err := r.db.WithContext(ctx).Where("email = ?", email).First(&user).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrUserNotFound + } + return nil, err + } + return &user, nil +} + +// GetUserByUsername gets a user by username +func (r *userRepository) GetUserByUsername(ctx context.Context, username string) (*types.User, error) { + var user types.User + if err := r.db.WithContext(ctx).Where("username = ?", username).First(&user).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrUserNotFound + } + return nil, err + } + return &user, nil +} + +// UpdateUser updates a user +func (r *userRepository) UpdateUser(ctx context.Context, user *types.User) error { + return r.db.WithContext(ctx).Save(user).Error +} + +// DeleteUser deletes a user +func (r *userRepository) DeleteUser(ctx context.Context, id string) error { + return r.db.WithContext(ctx).Where("id = ?", id).Delete(&types.User{}).Error +} + +// ListUsers lists users with pagination +func (r *userRepository) ListUsers(ctx context.Context, offset, limit int) ([]*types.User, error) { + var users []*types.User + query := r.db.WithContext(ctx).Order("created_at DESC") + + if limit > 0 { + query = query.Limit(limit) + } + + if offset > 0 { + query = query.Offset(offset) + } + + if err := query.Find(&users).Error; err != nil { + return nil, err + } + return users, nil +} + +// authTokenRepository implements auth token repository interface +type authTokenRepository struct { + db *gorm.DB +} + +// NewAuthTokenRepository creates a new auth token repository +func NewAuthTokenRepository(db *gorm.DB) interfaces.AuthTokenRepository { + return &authTokenRepository{db: db} +} + +// CreateToken creates an auth token +func (r *authTokenRepository) CreateToken(ctx context.Context, token *types.AuthToken) error { + return r.db.WithContext(ctx).Create(token).Error +} + +// GetTokenByValue gets a token by its value +func (r *authTokenRepository) GetTokenByValue(ctx context.Context, tokenValue string) (*types.AuthToken, error) { + var token types.AuthToken + if err := r.db.WithContext(ctx).Where("token = ?", tokenValue).First(&token).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrTokenNotFound + } + return nil, err + } + return &token, nil +} + +// GetTokensByUserID gets all tokens for a user +func (r *authTokenRepository) GetTokensByUserID(ctx context.Context, userID string) ([]*types.AuthToken, error) { + var tokens []*types.AuthToken + if err := r.db.WithContext(ctx).Where("user_id = ?", userID).Find(&tokens).Error; err != nil { + return nil, err + } + return tokens, nil +} + +// UpdateToken updates a token +func (r *authTokenRepository) UpdateToken(ctx context.Context, token *types.AuthToken) error { + return r.db.WithContext(ctx).Save(token).Error +} + +// DeleteToken deletes a token +func (r *authTokenRepository) DeleteToken(ctx context.Context, id string) error { + return r.db.WithContext(ctx).Where("id = ?", id).Delete(&types.AuthToken{}).Error +} + +// DeleteExpiredTokens deletes all expired tokens +func (r *authTokenRepository) DeleteExpiredTokens(ctx context.Context) error { + return r.db.WithContext(ctx).Where("expires_at < NOW()").Delete(&types.AuthToken{}).Error +} + +// RevokeTokensByUserID revokes all tokens for a user +func (r *authTokenRepository) RevokeTokensByUserID(ctx context.Context, userID string) error { + return r.db.WithContext(ctx).Model(&types.AuthToken{}).Where("user_id = ?", userID).Update("is_revoked", true).Error +} \ No newline at end of file diff --git a/internal/application/service/chat_pipline/chat_completion.go b/internal/application/service/chat_pipline/chat_completion.go new file mode 100644 index 0000000..e589698 --- /dev/null +++ b/internal/application/service/chat_pipline/chat_completion.go @@ -0,0 +1,60 @@ +package chatpipline + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// PluginChatCompletion implements chat completion functionality +// as a plugin that can be registered to EventManager +type PluginChatCompletion struct { + modelService interfaces.ModelService // Interface for model operations +} + +// NewPluginChatCompletion creates a new PluginChatCompletion instance +// and registers it with the EventManager +func NewPluginChatCompletion(eventManager *EventManager, modelService interfaces.ModelService) *PluginChatCompletion { + res := &PluginChatCompletion{ + modelService: modelService, + } + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginChatCompletion) ActivationEvents() []types.EventType { + return []types.EventType{types.CHAT_COMPLETION} +} + +// OnEvent handles chat completion events +// It prepares the chat model, messages, and calls the model to generate responses +func (p *PluginChatCompletion) OnEvent( + ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + logger.Info(ctx, "Starting chat completion") + + // Prepare chat model and options + chatModel, opt, err := prepareChatModel(ctx, p.modelService, chatManage) + if err != nil { + return ErrGetChatModel.WithError(err) + } + + // Prepare messages including conversation history + logger.Info(ctx, "Preparing chat messages with history") + chatMessages := prepareMessagesWithHistory(chatManage) + + // Call the chat model to generate response + logger.Info(ctx, "Calling chat model") + chatResponse, err := chatModel.Chat(ctx, chatMessages, opt) + if err != nil { + logger.Errorf(ctx, "Failed to call chat model: %v", err) + return ErrModelCall.WithError(err) + } + + logger.Info(ctx, "Chat completion successful") + chatManage.ChatResponse = chatResponse + return next() +} diff --git a/internal/application/service/chat_pipline/chat_completion_stream.go b/internal/application/service/chat_pipline/chat_completion_stream.go new file mode 100644 index 0000000..7f5644b --- /dev/null +++ b/internal/application/service/chat_pipline/chat_completion_stream.go @@ -0,0 +1,62 @@ +package chatpipline + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// PluginChatCompletionStream implements streaming chat completion functionality +// as a plugin that can be registered to EventManager +type PluginChatCompletionStream struct { + modelService interfaces.ModelService // Interface for model operations +} + +// NewPluginChatCompletionStream creates a new PluginChatCompletionStream instance +// and registers it with the EventManager +func NewPluginChatCompletionStream(eventManager *EventManager, + modelService interfaces.ModelService, +) *PluginChatCompletionStream { + res := &PluginChatCompletionStream{ + modelService: modelService, + } + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginChatCompletionStream) ActivationEvents() []types.EventType { + return []types.EventType{types.CHAT_COMPLETION_STREAM} +} + +// OnEvent handles streaming chat completion events +// It prepares the chat model, messages, and initiates streaming response +func (p *PluginChatCompletionStream) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + logger.Info(ctx, "Starting chat completion stream") + + // Prepare chat model and options + chatModel, opt, err := prepareChatModel(ctx, p.modelService, chatManage) + if err != nil { + return ErrGetChatModel.WithError(err) + } + + // Prepare base messages without history + logger.Info(ctx, "Preparing chat messages") + chatMessages := prepareMessagesWithHistory(chatManage) + + // Initiate streaming chat model call + logger.Info(ctx, "Calling chat stream model") + responseChan, err := chatModel.ChatStream(ctx, chatMessages, opt) + if err != nil { + logger.Errorf(ctx, "Failed to call chat stream model: %v", err) + return ErrModelCall.WithError(err) + } + + logger.Info(ctx, "Chat stream initiated successfully") + chatManage.ResponseChan = responseChan + return next() +} diff --git a/internal/application/service/chat_pipline/chat_pipline.go b/internal/application/service/chat_pipline/chat_pipline.go new file mode 100644 index 0000000..c82a7bb --- /dev/null +++ b/internal/application/service/chat_pipline/chat_pipline.go @@ -0,0 +1,161 @@ +package chatpipline + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// Plugin defines the interface for chat pipeline plugins +// Plugins can handle specific events in the chat pipeline +type Plugin interface { + // OnEvent handles the event with given context and chat management object + OnEvent( + ctx context.Context, + eventType types.EventType, + chatManage *types.ChatManage, + next func() *PluginError, + ) *PluginError + // ActivationEvents returns the event types this plugin can handle + ActivationEvents() []types.EventType +} + +// EventManager manages plugins and their event handling +type EventManager struct { + // Map of event types to registered plugins + listeners map[types.EventType][]Plugin + // Map of event types to handler functions + handlers map[types.EventType]func(context.Context, types.EventType, *types.ChatManage) *PluginError +} + +// NewEventManager creates and initializes a new EventManager +func NewEventManager() *EventManager { + return &EventManager{ + listeners: make(map[types.EventType][]Plugin), + handlers: make(map[types.EventType]func(context.Context, types.EventType, *types.ChatManage) *PluginError), + } +} + +// Register adds a plugin to the EventManager and sets up its event handlers +func (e *EventManager) Register(plugin Plugin) { + if e.listeners == nil { + e.listeners = make(map[types.EventType][]Plugin) + } + if e.handlers == nil { + e.handlers = make(map[types.EventType]func(context.Context, types.EventType, *types.ChatManage) *PluginError) + } + for _, eventType := range plugin.ActivationEvents() { + e.listeners[eventType] = append(e.listeners[eventType], plugin) + e.handlers[eventType] = e.buildHandler(e.listeners[eventType]) + } +} + +// buildHandler constructs a handler chain for the given plugins +func (e *EventManager) buildHandler(plugins []Plugin) func( + ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, +) *PluginError { + next := func(context.Context, types.EventType, *types.ChatManage) *PluginError { return nil } + for i := len(plugins) - 1; i >= 0; i-- { + current := plugins[i] + prevNext := next + next = func(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage) *PluginError { + return current.OnEvent(ctx, eventType, chatManage, func() *PluginError { + return prevNext(ctx, eventType, chatManage) + }) + } + } + return next +} + +// Trigger invokes the handler for the specified event type +func (e *EventManager) Trigger(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, +) *PluginError { + if handler, ok := e.handlers[eventType]; ok { + return handler(ctx, eventType, chatManage) + } + return nil +} + +// PluginError represents an error in plugin execution +type PluginError struct { + Err error // Original error + Description string // Human-readable description + ErrorType string // Error type identifier +} + +// Predefined plugin errors +var ( + ErrSearchNothing = &PluginError{ + Description: "No relevant content found", + ErrorType: "search_nothing", + } + ErrSearch = &PluginError{ + Description: "Failed to search knowledge base", + ErrorType: "search_failed", + } + ErrRerank = &PluginError{ + Description: "Reranking failed", + ErrorType: "rerank_failed", + } + ErrGetRerankModel = &PluginError{ + Description: "Failed to get rerank model", + ErrorType: "get_rerank_model_failed", + } + ErrGetChatModel = &PluginError{ + Description: "Failed to get chat model", + ErrorType: "get_chat_model_failed", + } + ErrTemplateParse = &PluginError{ + Description: "Failed to parse context template", + ErrorType: "template_parse_failed", + } + ErrTemplateExecute = &PluginError{ + Description: "Failed to generate search content", + ErrorType: "template_execution_failed", + } + ErrModelCall = &PluginError{ + Description: "Failed to call model", + ErrorType: "model_call_failed", + } + ErrGetHistory = &PluginError{ + Description: "Failed to get conversation history", + ErrorType: "get_history_failed", + } +) + +// clone creates a copy of the PluginError +func (p *PluginError) clone() *PluginError { + return &PluginError{ + Description: p.Description, + ErrorType: p.ErrorType, + } +} + +// WithError attaches an error to the PluginError and returns a new instance +func (p *PluginError) WithError(err error) *PluginError { + pp := p.clone() + pp.Err = err + return pp +} + +// NewFallbackChan creates a channel with a single fallback response +func NewFallbackChan(ctx context.Context, fallbackResponse string) <-chan types.StreamResponse { + fallabackChan := make(chan types.StreamResponse) + go func() { + fallabackChan <- NewFallback(ctx, fallbackResponse) + close(fallabackChan) + }() + return fallabackChan +} + +// NewFallback creates a fallback stream response with the given content +func NewFallback(ctx context.Context, fallbackResponse string) types.StreamResponse { + requestID := ctx.Value(types.RequestIDContextKey).(string) + return types.StreamResponse{ + ID: requestID, + ResponseType: types.ResponseTypeAnswer, + Content: fallbackResponse, + Done: true, + } +} diff --git a/internal/application/service/chat_pipline/chat_pipline_test.go b/internal/application/service/chat_pipline/chat_pipline_test.go new file mode 100644 index 0000000..66b69ab --- /dev/null +++ b/internal/application/service/chat_pipline/chat_pipline_test.go @@ -0,0 +1,125 @@ +package chatpipline + +import ( + "context" + "fmt" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +// Define a test Plugin implementation +type testPlugin struct { + name string + events []types.EventType + shouldError bool + errorToReturn *PluginError +} + +func (p *testPlugin) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + if p.shouldError { + return p.errorToReturn + } + fmt.Printf("Plugin %s triggered\n", p.name) + err := next() + fmt.Printf("Plugin %s finished\n", p.name) + return err +} + +func (p *testPlugin) ActivationEvents() []types.EventType { + return p.events +} + +func TestTrigger(t *testing.T) { + // Prepare test data + ctx := context.Background() + chatManage := &types.ChatManage{} + testEvent := types.EventType("test_event") + + // Test scenario 1: No plugins registered + t.Run("NoPluginsRegistered", func(t *testing.T) { + manager := &EventManager{} + err := manager.Trigger(ctx, testEvent, chatManage) + if err != nil { + t.Errorf("Expected nil error, got %v", err) + } + }) + + // Test scenario 2: Register a normally working plugin + t.Run("SinglePluginSuccess", func(t *testing.T) { + manager := &EventManager{} + plugin := &testPlugin{ + name: "test_plugin", + events: []types.EventType{testEvent}, + } + manager.Register(plugin) + + err := manager.Trigger(ctx, testEvent, chatManage) + if err != nil { + t.Errorf("Expected nil error, got %v", err) + } + }) + + // Test scenario 3: Plugin chain call + t.Run("PluginChain", func(t *testing.T) { + manager := &EventManager{} + plugin1 := &testPlugin{ + name: "plugin1", + events: []types.EventType{testEvent}, + } + plugin2 := &testPlugin{ + name: "plugin2", + events: []types.EventType{testEvent}, + } + manager.Register(plugin1) + manager.Register(plugin2) + + err := manager.Trigger(ctx, testEvent, chatManage) + if err != nil { + t.Errorf("Expected nil error, got %v", err) + } + }) + + // Test scenario 4: Plugin returns error + t.Run("PluginReturnsError", func(t *testing.T) { + manager := &EventManager{} + expectedErr := &PluginError{Description: "test error"} + plugin := &testPlugin{ + name: "error_plugin", + events: []types.EventType{testEvent}, + shouldError: true, + errorToReturn: expectedErr, + } + manager.Register(plugin) + + err := manager.Trigger(ctx, testEvent, chatManage) + if err != expectedErr { + t.Errorf("Expected error %v, got %v", expectedErr, err) + } + }) + + // Test scenario 5: A plugin in the chain returns error + t.Run("ErrorInPluginChain", func(t *testing.T) { + manager := &EventManager{} + expectedErr := &PluginError{Description: "test error"} + plugin1 := &testPlugin{ + name: "plugin1", + events: []types.EventType{testEvent}, + } + plugin2 := &testPlugin{ + name: "plugin2", + events: []types.EventType{testEvent}, + shouldError: true, + errorToReturn: expectedErr, + } + manager.Register(plugin1) + manager.Register(plugin2) + + err := manager.Trigger(ctx, testEvent, chatManage) + if err != expectedErr { + t.Errorf("Expected error %v, got %v", expectedErr, err) + } + }) +} diff --git a/internal/application/service/chat_pipline/common.go b/internal/application/service/chat_pipline/common.go new file mode 100644 index 0000000..9dff415 --- /dev/null +++ b/internal/application/service/chat_pipline/common.go @@ -0,0 +1,67 @@ +package chatpipline + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// prepareChatModel shared logic to prepare chat model and options +func prepareChatModel(ctx context.Context, modelService interfaces.ModelService, + chatManage *types.ChatManage, +) (chat.Chat, *chat.ChatOptions, error) { + logger.Infof(ctx, "Getting chat model, model ID: %s", chatManage.ChatModelID) + + chatModel, err := modelService.GetChatModel(ctx, chatManage.ChatModelID) + if err != nil { + logger.Errorf(ctx, "Failed to get chat model: %v", err) + return nil, nil, err + } + + logger.Info(ctx, "Setting up chat options") + opt := &chat.ChatOptions{ + Temperature: chatManage.SummaryConfig.Temperature, + TopP: chatManage.SummaryConfig.TopP, + Seed: chatManage.SummaryConfig.Seed, + MaxTokens: chatManage.SummaryConfig.MaxTokens, + MaxCompletionTokens: chatManage.SummaryConfig.MaxCompletionTokens, + FrequencyPenalty: chatManage.SummaryConfig.FrequencyPenalty, + PresencePenalty: chatManage.SummaryConfig.PresencePenalty, + } + + return chatModel, opt, nil +} + +// prepareBaseMessages prepare basic messages (system prompt and current user content) +func prepareBaseMessages(chatManage *types.ChatManage) []chat.Message { + var chatMessages []chat.Message + chatMessages = append(chatMessages, chat.Message{Role: "system", Content: chatManage.SummaryConfig.Prompt}) + chatMessages = append(chatMessages, chat.Message{Role: "user", Content: chatManage.UserContent}) + return chatMessages +} + +// prepareMessagesWithHistory prepare complete messages including history +func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message { + chatMessages := []chat.Message{ + {Role: "system", Content: chatManage.SummaryConfig.Prompt}, + } + + chatHistory := chatManage.History + if len(chatHistory) > 2 { + chatHistory = chatHistory[len(chatHistory)-2:] + } + + // Add conversation history + for _, history := range chatHistory { + chatMessages = append(chatMessages, chat.Message{Role: "user", Content: history.Query}) + chatMessages = append(chatMessages, chat.Message{Role: "assistant", Content: history.Answer}) + } + + // Add current user message + chatMessages = append(chatMessages, chat.Message{Role: "user", Content: chatManage.UserContent}) + + return chatMessages +} diff --git a/internal/application/service/chat_pipline/extract_entity.go b/internal/application/service/chat_pipline/extract_entity.go new file mode 100644 index 0000000..170a965 --- /dev/null +++ b/internal/application/service/chat_pipline/extract_entity.go @@ -0,0 +1,499 @@ +package chatpipline + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "regexp" + "strings" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// PluginExtractEntity is a plugin for extracting entities from user queries +// It uses historical dialog context and large language models to identify key entities in the user's original query +type PluginExtractEntity struct { + modelService interfaces.ModelService // Model service for calling large language models + template *types.PromptTemplateStructured // Template for generating prompts + knowledgeBaseRepo interfaces.KnowledgeBaseRepository +} + +// NewPluginRewrite creates a new query rewriting plugin instance +// Also registers the plugin with the event manager +func NewPluginExtractEntity( + eventManager *EventManager, + modelService interfaces.ModelService, + knowledgeBaseRepo interfaces.KnowledgeBaseRepository, + config *config.Config, +) *PluginExtractEntity { + res := &PluginExtractEntity{ + modelService: modelService, + template: config.ExtractManager.ExtractEntity, + knowledgeBaseRepo: knowledgeBaseRepo, + } + eventManager.Register(res) + return res +} + +// ActivationEvents returns the list of event types this plugin responds to +// This plugin only responds to REWRITE_QUERY events +func (p *PluginExtractEntity) ActivationEvents() []types.EventType { + return []types.EventType{types.REWRITE_QUERY} +} + +// OnEvent processes triggered events +// When receiving a REWRITE_QUERY event, it rewrites the user query using conversation history and the language model +func (p *PluginExtractEntity) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" { + logger.Debugf(ctx, "skipping extract entity, neo4j is disabled") + return next() + } + + query := chatManage.Query + + model, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID) + if err != nil { + logger.Errorf(ctx, "Failed to get model, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + + kb, err := p.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chatManage.KnowledgeBaseID) + if err != nil { + logger.Errorf(ctx, "failed to get knowledge base: %v", err) + return next() + } + if kb.ExtractConfig == nil { + logger.Warnf(ctx, "failed to get extract config") + return next() + } + + template := &types.PromptTemplateStructured{ + Description: p.template.Description, + Examples: p.template.Examples, + } + extractor := NewExtractor(model, template) + graph, err := extractor.Extract(ctx, query) + if err != nil { + logger.Errorf(ctx, "Failed to extract entities, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + nodes := []string{} + for _, node := range graph.Node { + nodes = append(nodes, node.Name) + } + logger.Debugf(ctx, "extracted node: %v", nodes) + chatManage.Entity = nodes + return next() +} + +type Extractor struct { + chat chat.Chat + formater *Formater + template *types.PromptTemplateStructured + chatOpt *chat.ChatOptions +} + +func NewExtractor( + chatModel chat.Chat, + template *types.PromptTemplateStructured, +) Extractor { + think := false + return Extractor{ + chat: chatModel, + formater: NewFormater(), + template: template, + chatOpt: &chat.ChatOptions{ + Temperature: 0.3, + MaxTokens: 4096, + Thinking: &think, + }, + } +} + +func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphData, error) { + generator := NewQAPromptGenerator(e.formater, e.template) + + // logger.Debugf(ctx, "chat system: %s", generator.System(ctx)) + // logger.Debugf(ctx, "chat user: %s", generator.User(ctx, content)) + + chatResponse, err := e.chat.Chat(ctx, generator.Render(ctx, content), e.chatOpt) + if err != nil { + logger.Errorf(ctx, "failed to chat: %v", err) + return nil, err + } + + graph, err := e.formater.ParseGraph(ctx, chatResponse.Content) + if err != nil { + logger.Errorf(ctx, "failed to parse graph: %v", err) + return nil, err + } + // e.RemoveUnknownRelation(ctx, graph) + return graph, nil +} + +func (e *Extractor) RemoveUnknownRelation(ctx context.Context, graph *types.GraphData) { + relationType := make(map[string]bool) + for _, tag := range e.template.Tags { + relationType[tag] = true + } + + relationNew := make([]*types.GraphRelation, 0) + for _, relation := range graph.Relation { + if _, ok := relationType[relation.Type]; ok { + relationNew = append(relationNew, relation) + } else { + logger.Infof(ctx, "Unknown relation type %s with %v, ignore it", relation.Type, e.template.Tags) + } + } + graph.Relation = relationNew +} + +type QAPromptGenerator struct { + Formater *Formater + Template *types.PromptTemplateStructured + ExamplesHeading string + QuestionHeading string + QuestionPrefix string + AnswerPrefix string +} + +func NewQAPromptGenerator(formater *Formater, template *types.PromptTemplateStructured) *QAPromptGenerator { + return &QAPromptGenerator{ + Formater: formater, + Template: template, + ExamplesHeading: "# Examples", + QuestionHeading: "# Question", + QuestionPrefix: "Q: ", + AnswerPrefix: "A: ", + } +} + +func (qa *QAPromptGenerator) System(ctx context.Context) string { + promptLines := []string{} + + if len(qa.Template.Tags) == 0 { + promptLines = append(promptLines, qa.Template.Description) + } else { + tags, _ := json.Marshal(qa.Template.Tags) + promptLines = append(promptLines, fmt.Sprintf(qa.Template.Description, string(tags))) + } + if len(qa.Template.Examples) > 0 { + promptLines = append(promptLines, qa.ExamplesHeading) + for _, example := range qa.Template.Examples { + // Question + promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.QuestionPrefix, strings.TrimSpace(example.Text))) + + // Answer + answer, err := qa.Formater.formatExtraction(example.Node, example.Relation) + if err != nil { + return "" + } + promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.AnswerPrefix, answer)) + + // new line + promptLines = append(promptLines, "") + } + } + return strings.Join(promptLines, "\n") +} + +func (qa *QAPromptGenerator) User(ctx context.Context, question string) string { + promptLines := []string{} + promptLines = append(promptLines, qa.QuestionHeading) + promptLines = append(promptLines, fmt.Sprintf("%s%s", qa.QuestionPrefix, question)) + promptLines = append(promptLines, qa.AnswerPrefix) + return strings.Join(promptLines, "\n") +} + +func (qa *QAPromptGenerator) Render(ctx context.Context, question string) []chat.Message { + return []chat.Message{ + { + Role: "system", + Content: qa.System(ctx), + }, + { + Role: "user", + Content: qa.User(ctx, question), + }, + } +} + +type FormatType string + +const ( + FormatTypeJSON FormatType = "json" + FormatTypeYAML FormatType = "yaml" +) + +const ( + _FENCE_START = "```" + _LANGUAGE_TAG = `(?P[A-Za-z0-9_+-]+)?` + _FENCE_NEWLINE = `(?:\s*\n)?` + _FENCE_BODY = `(?P[\s\S]*?)` + _FENCE_END = "```" +) + +var _FENCE_RE = regexp.MustCompile( + _FENCE_START + _LANGUAGE_TAG + _FENCE_NEWLINE + _FENCE_BODY + _FENCE_END, +) + +type Formater struct { + attributeSuffix string + formatType FormatType + useFences bool + nodePrefix string + + relationSource string + relationTarget string + relationPrefix string +} + +func NewFormater() *Formater { + return &Formater{ + attributeSuffix: "_attributes", + formatType: FormatTypeJSON, + useFences: true, + nodePrefix: "entity", + relationSource: "entity1", + relationTarget: "entity2", + relationPrefix: "relation", + } +} + +func (f *Formater) formatExtraction(nodes []*types.GraphNode, relations []*types.GraphRelation) (string, error) { + items := make([]map[string]interface{}, 0) + for _, node := range nodes { + item := map[string]interface{}{ + f.nodePrefix: node.Name, + } + if len(node.Attributes) > 0 { + item[fmt.Sprintf("%s%s", f.nodePrefix, f.attributeSuffix)] = node.Attributes + } + items = append(items, item) + } + for _, relation := range relations { + item := map[string]interface{}{ + f.relationSource: relation.Node1, + f.relationTarget: relation.Node2, + f.relationPrefix: relation.Type, + } + items = append(items, item) + } + formatted := "" + switch f.formatType { + default: + formattedBytes, err := json.MarshalIndent(items, "", " ") + if err != nil { + return "", err + } + formatted = string(formattedBytes) + } + if f.useFences { + formatted = f.addFences(formatted) + } + return formatted, nil +} + +func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]interface{}, error) { + if text == "" { + return nil, errors.New("Empty or invalid input string.") + } + content := f.extractContent(ctx, text) + // logger.Debugf(ctx, "Extracted content: %s", content) + if content == "" { + return nil, errors.New("Empty or invalid input string.") + } + + var parsed interface{} + var err error + if f.formatType == FormatTypeJSON { + err = json.Unmarshal([]byte(content), &parsed) + } + if err != nil { + return nil, fmt.Errorf("Failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error()) + } + if parsed == nil { + return nil, fmt.Errorf("Content must be a list of extractions or a dict.") + } + + var items []interface{} + if parsedMap, ok := parsed.(map[string]interface{}); ok { + items = []interface{}{parsedMap} + } else if parsedList, ok := parsed.([]interface{}); ok { + items = parsedList + } else { + return nil, fmt.Errorf("Expected list or dict, got %T", parsed) + } + + itemsList := make([]map[string]interface{}, 0) + for _, item := range items { + if itemMap, ok := item.(map[string]interface{}); ok { + itemsList = append(itemsList, itemMap) + } else { + return nil, fmt.Errorf("Each item in the sequence must be a mapping.") + } + } + return itemsList, nil +} + +func (f *Formater) ParseGraph(ctx context.Context, text string) (*types.GraphData, error) { + matchData, err := f.parseOutput(ctx, text) + if err != nil { + return nil, err + } + if len(matchData) == 0 { + logger.Debugf(ctx, "Received empty extraction data.") + return &types.GraphData{}, nil + } + // mm, _ := json.Marshal(matchData) + // logger.Debugf(ctx, "Parsed graph data: %s", string(mm)) + + var nodes []*types.GraphNode + var relations []*types.GraphRelation + + for _, group := range matchData { + switch { + case group[f.nodePrefix] != nil: + attributes := make([]string, 0) + attributesKey := f.nodePrefix + f.attributeSuffix + if attr, ok := group[attributesKey].([]interface{}); ok { + for _, v := range attr { + attributes = append(attributes, fmt.Sprintf("%v", v)) + } + } + nodes = append(nodes, &types.GraphNode{ + Name: fmt.Sprintf("%v", group[f.nodePrefix]), + Attributes: attributes, + }) + case group[f.relationSource] != nil && group[f.relationTarget] != nil: + relations = append(relations, &types.GraphRelation{ + Node1: fmt.Sprintf("%v", group[f.relationSource]), + Node2: fmt.Sprintf("%v", group[f.relationTarget]), + Type: fmt.Sprintf("%v", group[f.relationPrefix]), + }) + default: + logger.Warnf(ctx, "Unsupported graph group: %v", group) + continue + } + } + graph := &types.GraphData{ + Node: nodes, + Relation: relations, + } + f.rebuildGraph(ctx, graph) + return graph, nil +} + +func (f *Formater) rebuildGraph(ctx context.Context, graph *types.GraphData) { + nodeMap := make(map[string]*types.GraphNode) + nodes := make([]*types.GraphNode, 0, len(graph.Node)) + for _, node := range graph.Node { + if prenode, ok := nodeMap[node.Name]; ok { + logger.Infof(ctx, "Duplicate node ID: %s, merge attribute", node.Name) + // 修复panic：检查Attributes是否为nil + if node.Attributes == nil { + node.Attributes = make([]string, 0) + } + if prenode.Attributes != nil { + for _, attr := range prenode.Attributes { + node.Attributes = append(node.Attributes, attr) + } + } + continue + } + nodeMap[node.Name] = node + nodes = append(nodes, node) + } + + relations := make([]*types.GraphRelation, 0, len(graph.Relation)) + for _, relation := range graph.Relation { + if relation.Node1 == relation.Node2 { + logger.Infof(ctx, "Duplicate relation, ignore it") + continue + } + + if _, ok := nodeMap[relation.Node1]; !ok { + node := &types.GraphNode{Name: relation.Node1} + nodes = append(nodes, node) + nodeMap[relation.Node1] = node + logger.Infof(ctx, "Add unknown source node ID: %s", relation.Node1) + } + if _, ok := nodeMap[relation.Node2]; !ok { + node := &types.GraphNode{Name: relation.Node2} + nodes = append(nodes, node) + nodeMap[relation.Node2] = node + logger.Infof(ctx, "Add unknown target node ID: %s", relation.Node2) + } + + relations = append(relations, relation) + } + *graph = types.GraphData{ + Node: nodes, + Relation: relations, + } +} + +func (f *Formater) extractContent(ctx context.Context, text string) string { + if !f.useFences { + return strings.TrimSpace(text) + } + validTags := map[FormatType]map[string]struct{}{ + FormatTypeYAML: {"yaml": {}, "yml": {}}, + FormatTypeJSON: {"json": {}}, + } + matches := _FENCE_RE.FindAllStringSubmatch(text, -1) + var candidates []string + for _, match := range matches { + lang := match[1] + body := match[2] + if f.isValidLanguageTag(lang, validTags) { + candidates = append(candidates, body) + } + } + switch { + case len(candidates) == 1: + return strings.TrimSpace(candidates[0]) + + case len(candidates) > 1: + logger.Warnf(ctx, "multiple candidates found: %d", len(candidates)) + return strings.TrimSpace(candidates[0]) + + case len(matches) == 1: + logger.Debugf(ctx, "no candidate found, use first match without language tag: %s", matches[0][1]) + return strings.TrimSpace(matches[0][2]) + + case len(matches) > 1: + logger.Warnf(ctx, "multiple matches found: %d", len(matches)) + return strings.TrimSpace(matches[0][2]) + + default: + logger.Warnf(ctx, "no match found") + return strings.TrimSpace(text) + } +} + +func (f *Formater) addFences(content string) string { + content = strings.TrimSpace(content) + return fmt.Sprintf("```%s\n%s\n```", f.formatType, content) +} + +func (f *Formater) isValidLanguageTag(lang string, validTags map[FormatType]map[string]struct{}) bool { + if lang == "" { + return true + } + tag := strings.TrimSpace(strings.ToLower(lang)) + validSet, ok := validTags[f.formatType] + if !ok { + return false + } + _, exists := validSet[tag] + return exists +} diff --git a/internal/application/service/chat_pipline/filter_top_k.go b/internal/application/service/chat_pipline/filter_top_k.go new file mode 100644 index 0000000..d57b4b8 --- /dev/null +++ b/internal/application/service/chat_pipline/filter_top_k.go @@ -0,0 +1,53 @@ +package chatpipline + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// PluginFilterTopK is a plugin that filters search results to keep only the top K items +type PluginFilterTopK struct{} + +// NewPluginFilterTopK creates a new instance of PluginFilterTopK and registers it with the event manager +func NewPluginFilterTopK(eventManager *EventManager) *PluginFilterTopK { + res := &PluginFilterTopK{} + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types that this plugin responds to +func (p *PluginFilterTopK) ActivationEvents() []types.EventType { + return []types.EventType{types.FILTER_TOP_K} +} + +// OnEvent handles the FILTER_TOP_K event by filtering results to keep only the top K items +// It can filter MergeResult, RerankResult, or SearchResult depending on which is available +func (p *PluginFilterTopK) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + logger.Info(ctx, "Starting filter top-K process") + logger.Infof(ctx, "Filter configuration: top-K = %d", chatManage.RerankTopK) + + filterTopK := func(searchResult []*types.SearchResult, topK int) []*types.SearchResult { + if topK > 0 && len(searchResult) > topK { + logger.Infof(ctx, "Filtering results: before=%d, after=%d", len(searchResult), topK) + searchResult = searchResult[:topK] + } + return searchResult + } + + if len(chatManage.MergeResult) > 0 { + chatManage.MergeResult = filterTopK(chatManage.MergeResult, chatManage.RerankTopK) + } else if len(chatManage.RerankResult) > 0 { + chatManage.RerankResult = filterTopK(chatManage.RerankResult, chatManage.RerankTopK) + } else if len(chatManage.SearchResult) > 0 { + chatManage.SearchResult = filterTopK(chatManage.SearchResult, chatManage.RerankTopK) + } else { + logger.Info(ctx, "No results to filter") + } + + logger.Info(ctx, "Filter top-K process completed") + return next() +} diff --git a/internal/application/service/chat_pipline/into_chat_message.go b/internal/application/service/chat_pipline/into_chat_message.go new file mode 100644 index 0000000..b552d3e --- /dev/null +++ b/internal/application/service/chat_pipline/into_chat_message.go @@ -0,0 +1,191 @@ +package chatpipline + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "html/template" + "regexp" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// PluginIntoChatMessage handles the transformation of search results into chat messages +type PluginIntoChatMessage struct{} + +// NewPluginIntoChatMessage creates and registers a new PluginIntoChatMessage instance +func NewPluginIntoChatMessage(eventManager *EventManager) *PluginIntoChatMessage { + res := &PluginIntoChatMessage{} + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginIntoChatMessage) ActivationEvents() []types.EventType { + return []types.EventType{types.INTO_CHAT_MESSAGE} +} + +// OnEvent processes the INTO_CHAT_MESSAGE event to format chat message content +func (p *PluginIntoChatMessage) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + // Extract content from merge results + passages := make([]string, len(chatManage.MergeResult)) + for i, result := range chatManage.MergeResult { + // 合并内容和图片信息 + passages[i] = getEnrichedPassageForChat(ctx, result) + } + + // Parse the context template + tmpl, err := template.New("searchContent").Parse(chatManage.SummaryConfig.ContextTemplate) + if err != nil { + return ErrTemplateParse.WithError(err) + } + + // Prepare weekday names for template + weekdayName := []string{"星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六"} + var userContent bytes.Buffer + + // 验证用户查询的安全性 + safeQuery, isValid := secutils.ValidateInput(chatManage.Query) + if !isValid { + logger.Errorf(ctx, "Invalid user query: %s", chatManage.Query) + return ErrTemplateExecute.WithError(fmt.Errorf("用户查询包含非法内容")) + } + + // Execute template with context data + err = tmpl.Execute(&userContent, map[string]interface{}{ + "Query": safeQuery, // User's original query + "Contexts": passages, // Extracted passages from search results + "CurrentTime": time.Now().Format("2006-01-02 15:04:05"), // Formatted current time + "CurrentWeek": weekdayName[time.Now().Weekday()], // Current weekday in Chinese + }) + if err != nil { + return ErrTemplateExecute.WithError(err) + } + + // Set formatted content back to chat management + chatManage.UserContent = userContent.String() + return next() +} + +// getEnrichedPassageForChat 合并Content和ImageInfo的文本内容，为聊天消息准备 +func getEnrichedPassageForChat(ctx context.Context, result *types.SearchResult) string { + // 如果没有图片信息，直接返回内容 + if result.Content == "" && result.ImageInfo == "" { + return "" + } + + // 如果只有内容，没有图片信息 + if result.ImageInfo == "" { + return result.Content + } + + // 处理图片信息并与内容合并 + return enrichContentWithImageInfo(ctx, result.Content, result.ImageInfo) +} + +// 正则表达式用于匹配Markdown图片链接 +var markdownImageRegex = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) + +// enrichContentWithImageInfo 将图片信息与文本内容合并 +func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJSON string) string { + // 解析ImageInfo + var imageInfos []types.ImageInfo + err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos) + if err != nil { + logger.Warnf(ctx, "Failed to parse ImageInfo: %v, using content only", err) + return content + } + + if len(imageInfos) == 0 { + return content + } + + // 创建图片URL到信息的映射 + imageInfoMap := make(map[string]*types.ImageInfo) + for i := range imageInfos { + if imageInfos[i].URL != "" { + imageInfoMap[imageInfos[i].URL] = &imageInfos[i] + } + // 同时检查原始URL + if imageInfos[i].OriginalURL != "" { + imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i] + } + } + + // 查找内容中的所有Markdown图片链接 + matches := markdownImageRegex.FindAllStringSubmatch(content, -1) + + // 用于存储已处理的图片URL + processedURLs := make(map[string]bool) + + logger.Infof(ctx, "Found %d Markdown image links in content", len(matches)) + + // 替换每个图片链接，添加描述和OCR文本 + for _, match := range matches { + if len(match) < 3 { + continue + } + + // 提取图片URL，忽略alt文本 + imgURL := match[2] + + // 标记该URL已处理 + processedURLs[imgURL] = true + + // 查找匹配的图片信息 + imgInfo, found := imageInfoMap[imgURL] + + // 如果找到匹配的图片信息，添加描述和OCR文本 + if found && imgInfo != nil { + replacement := match[0] + "\n" + if imgInfo.Caption != "" { + replacement += fmt.Sprintf("图片描述: %s\n", imgInfo.Caption) + } + if imgInfo.OCRText != "" { + replacement += fmt.Sprintf("图片文本: %s\n", imgInfo.OCRText) + } + content = strings.Replace(content, match[0], replacement, 1) + } + } + + // 处理未在内容中找到但存在于ImageInfo中的图片 + var additionalImageTexts []string + for _, imgInfo := range imageInfos { + // 如果图片URL已经处理过，跳过 + if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] { + continue + } + + var imgTexts []string + if imgInfo.Caption != "" { + imgTexts = append(imgTexts, fmt.Sprintf("图片 %s 的描述信息: %s", imgInfo.URL, imgInfo.Caption)) + } + if imgInfo.OCRText != "" { + imgTexts = append(imgTexts, fmt.Sprintf("图片 %s 的文本: %s", imgInfo.URL, imgInfo.OCRText)) + } + + if len(imgTexts) > 0 { + additionalImageTexts = append(additionalImageTexts, imgTexts...) + } + } + + // 如果有额外的图片信息，添加到内容末尾 + if len(additionalImageTexts) > 0 { + if content != "" { + content += "\n\n" + } + content += "附加图片信息:\n" + strings.Join(additionalImageTexts, "\n") + } + + logger.Debugf(ctx, "Enhanced content with image info: found %d Markdown images, added %d additional images", + len(matches), len(additionalImageTexts)) + + return content +} diff --git a/internal/application/service/chat_pipline/merge.go b/internal/application/service/chat_pipline/merge.go new file mode 100644 index 0000000..dbdde26 --- /dev/null +++ b/internal/application/service/chat_pipline/merge.go @@ -0,0 +1,175 @@ +package chatpipline + +import ( + "context" + "encoding/json" + "sort" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// PluginMerge handles merging of search result chunks +type PluginMerge struct{} + +// NewPluginMerge creates and registers a new PluginMerge instance +func NewPluginMerge(eventManager *EventManager) *PluginMerge { + res := &PluginMerge{} + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginMerge) ActivationEvents() []types.EventType { + return []types.EventType{types.CHUNK_MERGE} +} + +// OnEvent processes the CHUNK_MERGE event to merge search result chunks +func (p *PluginMerge) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + logger.Info(ctx, "Starting chunk merge process") + + // Use rerank results if available, fallback to search results + searchResult := chatManage.RerankResult + if len(searchResult) == 0 { + logger.Info(ctx, "No rerank results available, using search results") + searchResult = chatManage.SearchResult + } + + logger.Infof(ctx, "Processing %d chunks for merging", len(searchResult)) + + if len(searchResult) == 0 { + logger.Info(ctx, "No chunks available for merging") + return next() + } + + // Group chunks by their knowledge source ID + knowledgeGroup := make(map[string][]*types.SearchResult) + for _, chunk := range searchResult { + knowledgeGroup[chunk.KnowledgeID] = append(knowledgeGroup[chunk.KnowledgeID], chunk) + } + + logger.Infof(ctx, "Grouped chunks by knowledge ID, %d knowledge sources", len(knowledgeGroup)) + + mergedChunks := []*types.SearchResult{} + // Process each knowledge source separately + for knowledgeID, chunks := range knowledgeGroup { + logger.Infof(ctx, "Processing knowledge ID: %s with %d chunks", knowledgeID, len(chunks)) + + // Sort chunks by their start position in the original document + sort.Slice(chunks, func(i, j int) bool { + if chunks[i].StartAt == chunks[j].StartAt { + return chunks[i].EndAt < chunks[j].EndAt + } + return chunks[i].StartAt < chunks[j].StartAt + }) + + // Merge overlapping or adjacent chunks + knowledgeMergedChunks := []*types.SearchResult{} + if chunks[0].ChunkType == types.ChunkTypeSummary { + knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[0]) + // skip the first chunk if it is summary chunk + // This is to avoid merging the summary chunk with the first content chunk + chunks = chunks[1:] + } + if len(chunks) > 0 { + knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[0]) + } + for i := 1; i < len(chunks); i++ { + lastChunk := knowledgeMergedChunks[len(knowledgeMergedChunks)-1] + // If the current chunk starts after the last chunk ends, add it to the merged chunks + if chunks[i].StartAt > lastChunk.EndAt { + knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[i]) + continue + } + // Merge overlapping chunks + if chunks[i].EndAt > lastChunk.EndAt { + lastChunk.Content = lastChunk.Content + + string([]rune(chunks[i].Content)[lastChunk.EndAt-chunks[i].StartAt:]) + lastChunk.EndAt = chunks[i].EndAt + lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID) + + // 合并 ImageInfo + if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil { + logger.Warnf(ctx, "Failed to merge ImageInfo: %v", err) + } + } + if chunks[i].Score > lastChunk.Score { + lastChunk.Score = chunks[i].Score + } + } + + logger.Infof(ctx, "Merged %d chunks into %d chunks for knowledge ID: %s", + len(chunks), len(knowledgeMergedChunks), knowledgeID) + + mergedChunks = append(mergedChunks, knowledgeMergedChunks...) + } + + // Sort merged chunks by their score (highest first) + sort.Slice(mergedChunks, func(i, j int) bool { + return mergedChunks[i].Score > mergedChunks[j].Score + }) + + logger.Infof(ctx, "Final merged result: %d chunks, sorted by score", len(mergedChunks)) + + chatManage.MergeResult = mergedChunks + return next() +} + +// mergeImageInfo 合并两个chunk的ImageInfo +func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error { + // 如果source没有ImageInfo，不需要合并 + if source.ImageInfo == "" { + return nil + } + + var sourceImageInfos []types.ImageInfo + if err := json.Unmarshal([]byte(source.ImageInfo), &sourceImageInfos); err != nil { + logger.Warnf(ctx, "Failed to unmarshal source ImageInfo: %v", err) + return err + } + + // 如果source的ImageInfo为空，不需要合并 + if len(sourceImageInfos) == 0 { + return nil + } + + // 处理target的ImageInfo + var targetImageInfos []types.ImageInfo + if target.ImageInfo != "" { + if err := json.Unmarshal([]byte(target.ImageInfo), &targetImageInfos); err != nil { + logger.Warnf(ctx, "Failed to unmarshal target ImageInfo: %v", err) + // 如果目标解析失败，直接使用源数据 + target.ImageInfo = source.ImageInfo + return nil + } + } + + // 合并ImageInfo + targetImageInfos = append(targetImageInfos, sourceImageInfos...) + + // 去重 + uniqueMap := make(map[string]bool) + uniqueImageInfos := make([]types.ImageInfo, 0, len(targetImageInfos)) + + for _, imgInfo := range targetImageInfos { + // 使用URL作为唯一标识 + if imgInfo.URL != "" && !uniqueMap[imgInfo.URL] { + uniqueMap[imgInfo.URL] = true + uniqueImageInfos = append(uniqueImageInfos, imgInfo) + } + } + + // 序列化合并后的ImageInfo + mergedImageInfoJSON, err := json.Marshal(uniqueImageInfos) + if err != nil { + logger.Warnf(ctx, "Failed to marshal merged ImageInfo: %v", err) + return err + } + + // 更新目标chunk的ImageInfo + target.ImageInfo = string(mergedImageInfoJSON) + logger.Infof(ctx, "Successfully merged ImageInfo, total count: %d", len(uniqueImageInfos)) + return nil +} diff --git a/internal/application/service/chat_pipline/preprocess.go b/internal/application/service/chat_pipline/preprocess.go new file mode 100644 index 0000000..dad7274 --- /dev/null +++ b/internal/application/service/chat_pipline/preprocess.go @@ -0,0 +1,180 @@ +package chatpipline + +import ( + "context" + "regexp" + "strings" + "unicode" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/yanyiwu/gojieba" +) + +// PluginPreprocess Query preprocessing plugin +type PluginPreprocess struct { + config *config.Config + jieba *gojieba.Jieba + stopwords map[string]struct{} +} + +// Regular expressions for text cleaning +var ( + multiSpaceRegex = regexp.MustCompile(`\s+`) // Multiple spaces + urlRegex = regexp.MustCompile(`https?://\S+`) // URLs + emailRegex = regexp.MustCompile(`\b[\w.%+-]+@[\w.-]+\.[a-zA-Z]{2,}\b`) // Email addresses + punctRegex = regexp.MustCompile(`[^\p{L}\p{N}\s]`) // Punctuation marks +) + +// NewPluginPreprocess Creates a new query preprocessing plugin +func NewPluginPreprocess( + eventManager *EventManager, + config *config.Config, + cleaner interfaces.ResourceCleaner, +) *PluginPreprocess { + // Use default dictionary for Jieba tokenizer + jieba := gojieba.NewJieba() + + // Load stopwords from built-in stopword library + stopwords := loadStopwords() + + res := &PluginPreprocess{ + config: config, + jieba: jieba, + stopwords: stopwords, + } + + // Register resource cleanup function + if cleaner != nil { + cleaner.RegisterWithName("JiebaPreprocessor", func() error { + res.Close() + return nil + }) + } + + eventManager.Register(res) + return res +} + +// Load stopwords +func loadStopwords() map[string]struct{} { + // Directly use some common stopwords built into Jieba + commonStopwords := []string{ + "的", "了", "和", "是", "在", "我", "你", "他", "她", "它", + "这", "那", "什么", "怎么", "如何", "为什么", "哪里", "什么时候", + "the", "is", "are", "am", "I", "you", "he", "she", "it", "this", + "that", "what", "how", "a", "an", "and", "or", "but", "if", "of", + "to", "in", "on", "at", "by", "for", "with", "about", "from", + "有", "无", "好", "来", "去", "说", "看", "想", "会", "可以", + "吗", "呢", "啊", "吧", "的话", "就是", "只是", "因为", "所以", + } + + result := make(map[string]struct{}, len(commonStopwords)) + for _, word := range commonStopwords { + result[word] = struct{}{} + } + return result +} + +// ActivationEvents Register activation events +func (p *PluginPreprocess) ActivationEvents() []types.EventType { + return []types.EventType{types.PREPROCESS_QUERY} +} + +// OnEvent Process events +func (p *PluginPreprocess) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError) *PluginError { + if chatManage.RewriteQuery == "" { + return next() + } + + logger.GetLogger(ctx).Infof("Starting query preprocessing, original query: %s", chatManage.RewriteQuery) + + // 1. Basic text cleaning + processed := p.cleanText(chatManage.RewriteQuery) + + // 2. Tokenization + segments := p.segmentText(processed) + + // 3. Stopword filtering and reconstruction + filteredSegments := p.filterStopwords(segments) + + // Update preprocessed query + chatManage.ProcessedQuery = strings.Join(filteredSegments, " ") + + logger.GetLogger(ctx).Infof("Query preprocessing complete, processed query: %s", chatManage.ProcessedQuery) + + return next() +} + +// cleanText Basic text cleaning +func (p *PluginPreprocess) cleanText(text string) string { + // Remove URLs + text = urlRegex.ReplaceAllString(text, " ") + + // Remove email addresses + text = emailRegex.ReplaceAllString(text, " ") + + // Remove excessive spaces + text = multiSpaceRegex.ReplaceAllString(text, " ") + + // Remove punctuation marks + text = punctRegex.ReplaceAllString(text, " ") + + // Trim leading and trailing spaces + text = strings.TrimSpace(text) + + return text +} + +// segmentText Text tokenization +func (p *PluginPreprocess) segmentText(text string) []string { + // Use Jieba tokenizer for tokenization, using search engine mode + segments := p.jieba.CutForSearch(text, true) + return segments +} + +// filterStopwords Filter stopwords +func (p *PluginPreprocess) filterStopwords(segments []string) []string { + var filtered []string + + for _, word := range segments { + // If not a stopword and not blank, keep it + if _, isStopword := p.stopwords[word]; !isStopword && !isBlank(word) { + filtered = append(filtered, word) + } + } + + // If filtering results in empty list, return original tokenization results + if len(filtered) == 0 { + return segments + } + + return filtered +} + +// isBlank Check if a string is blank +func isBlank(str string) bool { + for _, r := range str { + if !unicode.IsSpace(r) { + return false + } + } + return true +} + +// Ensure resources are properly released +func (p *PluginPreprocess) Close() { + if p.jieba != nil { + p.jieba.Free() + p.jieba = nil + } +} + +// ShutdownHandler Returns shutdown function +func (p *PluginPreprocess) ShutdownHandler() func() { + return func() { + p.Close() + } +} diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go new file mode 100644 index 0000000..c8ceea2 --- /dev/null +++ b/internal/application/service/chat_pipline/rerank.go @@ -0,0 +1,171 @@ +package chatpipline + +import ( + "context" + "encoding/json" + "fmt" + "math" + "strings" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// PluginRerank implements reranking functionality for chat pipeline +type PluginRerank struct { + modelService interfaces.ModelService // Service to access rerank models +} + +// NewPluginRerank creates a new rerank plugin instance +func NewPluginRerank(eventManager *EventManager, modelService interfaces.ModelService) *PluginRerank { + res := &PluginRerank{ + modelService: modelService, + } + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginRerank) ActivationEvents() []types.EventType { + return []types.EventType{types.CHUNK_RERANK} +} + +// OnEvent handles reranking events in the chat pipeline +func (p *PluginRerank) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + logger.Info(ctx, "Starting reranking process") + logger.Infof(ctx, "Getting rerank model, model ID: %s", chatManage.RerankModelID) + if len(chatManage.SearchResult) == 0 { + logger.Infof(ctx, "No search result, skip reranking") + return next() + } + if chatManage.RerankModelID == "" { + logger.Warn(ctx, "Rerank model ID is empty, skipping reranking") + return next() + } + + // Get rerank model from service + rerankModel, err := p.modelService.GetRerankModel(ctx, chatManage.RerankModelID) + if err != nil { + logger.Errorf(ctx, "Failed to get rerank model: %v, rerank model ID: %s", err, chatManage.RerankModelID) + return ErrGetRerankModel.WithError(err) + } + + // Prepare passages for reranking + logger.Infof(ctx, "Preparing passages for reranking, search result count: %d", len(chatManage.SearchResult)) + var passages []string + for _, result := range chatManage.SearchResult { + // 合并Content和ImageInfo的文本内容 + passage := getEnrichedPassage(ctx, result) + passages = append(passages, passage) + } + + // Try reranking with different query variants in priority order + rerankResp := p.rerank(ctx, chatManage, rerankModel, chatManage.RewriteQuery, passages) + if len(rerankResp) == 0 { + rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.ProcessedQuery, passages) + if len(rerankResp) == 0 { + rerankResp = p.rerank(ctx, chatManage, rerankModel, chatManage.Query, passages) + } + } + + // Update search results with reranked scores + logger.Infof(ctx, "Filtered rerank results, original: %d, filtered: %d", len(rerankResp), len(rerankResp)) + result := []*types.SearchResult{} + for _, rr := range rerankResp { + chatManage.SearchResult[rr.Index].Score = rr.RelevanceScore + result = append(result, chatManage.SearchResult[rr.Index]) + } + chatManage.RerankResult = result + + if len(chatManage.RerankResult) == 0 { + logger.Warn(ctx, "Reranking produced no results above threshold") + return ErrSearchNothing + } + + logger.Infof(ctx, "Reranking process completed successfully, result count: %d", len(chatManage.RerankResult)) + return next() +} + +// rerank performs the actual reranking operation with given query and passages +func (p *PluginRerank) rerank(ctx context.Context, + chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string, +) []rerank.RankResult { + logger.Infof(ctx, "Executing reranking with query: %s, passage count: %d", query, len(passages)) + rerankResp, err := rerankModel.Rerank(ctx, query, passages) + if err != nil { + logger.Errorf(ctx, "Reranking failed: %v", err) + return nil + } + + // Log top scores for debugging + logger.Infof(ctx, "Reranking completed, filtering results with threshold: %f", chatManage.RerankThreshold) + for i := range min(3, len(rerankResp)) { + logger.Infof(ctx, "Top %d score of rerankResp: %f, passages: %s, index: %d", + i+1, rerankResp[i].RelevanceScore, rerankResp[i].Document.Text, rerankResp[i].Index, + ) + } + + // Filter results based on threshold with special handling for history matches + rankFilter := []rerank.RankResult{} + for _, result := range rerankResp { + th := chatManage.RerankThreshold + matchType := chatManage.SearchResult[result.Index].MatchType + if matchType == types.MatchTypeHistory { + th = math.Max(th-0.1, 0.5) // Lower threshold for history matches + } + if result.RelevanceScore > th { + rankFilter = append(rankFilter, result) + } + } + return rankFilter +} + +// getEnrichedPassage 合并Content和ImageInfo的文本内容 +func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string { + if result.ImageInfo == "" { + return result.Content + } + + // 解析ImageInfo + var imageInfos []types.ImageInfo + err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos) + if err != nil { + logger.Warnf(ctx, "Failed to parse ImageInfo: %v, using content only", err) + return result.Content + } + + if len(imageInfos) == 0 { + return result.Content + } + + // 提取所有图片的描述和OCR文本 + var imageTexts []string + for _, img := range imageInfos { + if img.Caption != "" { + imageTexts = append(imageTexts, fmt.Sprintf("图片描述: %s", img.Caption)) + } + if img.OCRText != "" { + imageTexts = append(imageTexts, fmt.Sprintf("图片文本: %s", img.OCRText)) + } + } + + if len(imageTexts) == 0 { + return result.Content + } + + // 组合内容和图片信息 + combinedText := result.Content + if combinedText != "" { + combinedText += "\n\n" + } + combinedText += strings.Join(imageTexts, "\n") + + logger.Debugf(ctx, "Enhanced passage with image info: content length %d, image texts length %d", + len(result.Content), len(strings.Join(imageTexts, "\n"))) + + return combinedText +} diff --git a/internal/application/service/chat_pipline/rewrite.go b/internal/application/service/chat_pipline/rewrite.go new file mode 100644 index 0000000..16e69f8 --- /dev/null +++ b/internal/application/service/chat_pipline/rewrite.go @@ -0,0 +1,176 @@ +// Package chatpipline provides chat pipeline processing capabilities +// Including query rewriting, history processing, model invocation and other features +package chatpipline + +import ( + "bytes" + "context" + "html/template" + "regexp" + "slices" + "sort" + "time" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// PluginRewrite is a plugin for rewriting user queries +// It uses historical dialog context and large language models to optimize the user's original query +type PluginRewrite struct { + modelService interfaces.ModelService // Model service for calling large language models + messageService interfaces.MessageService // Message service for retrieving historical messages + config *config.Config // System configuration +} + +// reg is a regular expression used to match and remove content between tags +var reg = regexp.MustCompile(`(?s).*?`) + +// NewPluginRewrite creates a new query rewriting plugin instance +// Also registers the plugin with the event manager +func NewPluginRewrite(eventManager *EventManager, + modelService interfaces.ModelService, messageService interfaces.MessageService, + config *config.Config, +) *PluginRewrite { + res := &PluginRewrite{ + modelService: modelService, + messageService: messageService, + config: config, + } + eventManager.Register(res) + return res +} + +// ActivationEvents returns the list of event types this plugin responds to +// This plugin only responds to REWRITE_QUERY events +func (p *PluginRewrite) ActivationEvents() []types.EventType { + return []types.EventType{types.REWRITE_QUERY} +} + +// OnEvent processes triggered events +// When receiving a REWRITE_QUERY event, it rewrites the user query using conversation history and the language model +func (p *PluginRewrite) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + // Initialize rewritten query as original query + chatManage.RewriteQuery = chatManage.Query + + // Get conversation history + history, err := p.messageService.GetRecentMessagesBySession(ctx, chatManage.SessionID, 20) + if err != nil { + logger.Errorf(ctx, "Failed to get conversation history, session_id: %s, error: %v", chatManage.SessionID, err) + } + + // Convert historical messages to conversation history structure + historyMap := make(map[string]*types.History) + + // Process historical messages, grouped by requestID + for _, message := range history { + history, ok := historyMap[message.RequestID] + if !ok { + history = &types.History{} + } + if message.Role == "user" { + // User message as query + history.Query = message.Content + history.CreateAt = message.CreatedAt + } else { + // System message as answer, while removing thinking process + history.Answer = reg.ReplaceAllString(message.Content, "") + history.KnowledgeReferences = message.KnowledgeReferences + } + historyMap[message.RequestID] = history + } + + // Convert to list and filter incomplete conversations + historyList := make([]*types.History, 0) + for _, history := range historyMap { + if history.Answer != "" && history.Query != "" { + historyList = append(historyList, history) + } + } + + // Sort by time, keep the most recent conversations + sort.Slice(historyList, func(i, j int) bool { + return historyList[i].CreateAt.After(historyList[j].CreateAt) + }) + + // Limit the number of historical records + if len(historyList) > p.config.Conversation.MaxRounds { + historyList = historyList[:p.config.Conversation.MaxRounds] + } + + // Reverse to chronological order + slices.Reverse(historyList) + chatManage.History = historyList + + userTmpl, err := template.New("rewriteContent").Parse(p.config.Conversation.RewritePromptUser) + if err != nil { + logger.Errorf(ctx, "Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + systemTmpl, err := template.New("rewriteContent").Parse(p.config.Conversation.RewritePromptSystem) + if err != nil { + logger.GetLogger(ctx).Errorf("Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + currentTime := time.Now().Format("2006-01-02 15:04:05") + var userContent, systemContent bytes.Buffer + err = userTmpl.Execute(&userContent, map[string]interface{}{ + "Query": chatManage.Query, + "CurrentTime": currentTime, + "Yesterday": time.Now().AddDate(0, 0, -1).Format("2006-01-02"), + "Conversation": historyList, + }) + if err != nil { + logger.GetLogger(ctx).Errorf("Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + err = systemTmpl.Execute(&systemContent, map[string]interface{}{ + "Query": chatManage.Query, + "CurrentTime": currentTime, + "Yesterday": time.Now().AddDate(0, 0, -1).Format("2006-01-02"), + "Conversation": historyList, + }) + if err != nil { + logger.Errorf(ctx, "Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + rewriteModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID) + if err != nil { + logger.Errorf(ctx, "Failed to get model, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + + // Call model to rewrite query + thinking := false + response, err := rewriteModel.Chat(ctx, []chat.Message{ + { + Role: "system", + Content: systemContent.String(), + }, + { + Role: "user", + Content: userContent.String(), + }, + }, &chat.ChatOptions{ + Temperature: 0.3, + MaxCompletionTokens: 50, + Thinking: &thinking, + }) + if err != nil { + logger.Errorf(ctx, "Failed to execute model, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + + if response.Content != "" { + // Update rewritten query + chatManage.RewriteQuery = response.Content + } + logger.GetLogger(ctx).Infof("Rewritten query, session_id: %s, rewrite_query: %s", + chatManage.SessionID, chatManage.RewriteQuery) + return next() +} diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go new file mode 100644 index 0000000..a5b2904 --- /dev/null +++ b/internal/application/service/chat_pipline/search.go @@ -0,0 +1,125 @@ +package chatpipline + +import ( + "context" + "strings" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// PluginSearch implements search functionality for chat pipeline +type PluginSearch struct { + knowledgeBaseService interfaces.KnowledgeBaseService + modelService interfaces.ModelService + config *config.Config +} + +func NewPluginSearch(eventManager *EventManager, + knowledgeBaseService interfaces.KnowledgeBaseService, + modelService interfaces.ModelService, + config *config.Config, +) *PluginSearch { + res := &PluginSearch{ + knowledgeBaseService: knowledgeBaseService, + modelService: modelService, + config: config, + } + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginSearch) ActivationEvents() []types.EventType { + return []types.EventType{types.CHUNK_SEARCH} +} + +// OnEvent handles search events in the chat pipeline +func (p *PluginSearch) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + // Prepare search parameters + searchParams := types.SearchParams{ + QueryText: strings.TrimSpace(chatManage.RewriteQuery), + VectorThreshold: chatManage.VectorThreshold, + KeywordThreshold: chatManage.KeywordThreshold, + MatchCount: chatManage.EmbeddingTopK, + } + logger.Infof(ctx, "Search parameters: %v", searchParams) + + // Perform initial hybrid search + searchResults, err := p.knowledgeBaseService.HybridSearch(ctx, chatManage.KnowledgeBaseID, searchParams) + logger.Infof(ctx, "Search results count: %d, error: %v", len(searchResults), err) + if err != nil { + return ErrSearch.WithError(err) + } + chatManage.SearchResult = searchResults + logger.Infof(ctx, "Search result count: %d", len(chatManage.SearchResult)) + + // Add relevant results from chat history + historyResult := p.getSearchResultFromHistory(chatManage) + if historyResult != nil { + logger.Infof(ctx, "Add history result, result count: %d", len(historyResult)) + chatManage.SearchResult = append(chatManage.SearchResult, historyResult...) + } + + // Try search with processed query if different from rewrite query + if chatManage.RewriteQuery != chatManage.ProcessedQuery { + searchParams.QueryText = strings.TrimSpace(chatManage.ProcessedQuery) + searchResults, err = p.knowledgeBaseService.HybridSearch(ctx, chatManage.KnowledgeBaseID, searchParams) + logger.Infof(ctx, "Search by processed query: %s, results count: %d, error: %v", + searchParams.QueryText, len(searchResults), err, + ) + if err != nil { + return ErrSearch.WithError(err) + } + chatManage.SearchResult = append(chatManage.SearchResult, searchResults...) + } + + // remove duplicate results + chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult) + + // Return if we have results + if len(chatManage.SearchResult) != 0 { + logger.Infof( + ctx, + "Get search results, count: %d, session_id: %s", + len(chatManage.SearchResult), chatManage.SessionID, + ) + return next() + } + logger.Infof(ctx, "No search result, session_id: %s", chatManage.SessionID) + return ErrSearchNothing +} + +// getSearchResultFromHistory retrieves relevant knowledge references from chat history +func (p *PluginSearch) getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult { + if len(chatManage.History) == 0 { + return nil + } + // Search history in reverse chronological order + for i := len(chatManage.History) - 1; i >= 0; i-- { + if len(chatManage.History[i].KnowledgeReferences) > 0 { + // Mark all references as history matches + for _, reference := range chatManage.History[i].KnowledgeReferences { + reference.MatchType = types.MatchTypeHistory + } + return chatManage.History[i].KnowledgeReferences + } + } + return nil +} + +func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult { + seen := make(map[string]bool) + var uniqueResults []*types.SearchResult + for _, result := range results { + if !seen[result.ID] { + seen[result.ID] = true + uniqueResults = append(uniqueResults, result) + } + } + return uniqueResults +} diff --git a/internal/application/service/chat_pipline/search_entity.go b/internal/application/service/chat_pipline/search_entity.go new file mode 100644 index 0000000..a059174 --- /dev/null +++ b/internal/application/service/chat_pipline/search_entity.go @@ -0,0 +1,136 @@ +package chatpipline + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// PluginSearch implements search functionality for chat pipeline +type PluginSearchEntity struct { + graphRepo interfaces.RetrieveGraphRepository + chunkRepo interfaces.ChunkRepository + knowledgeRepo interfaces.KnowledgeRepository +} + +func NewPluginSearchEntity( + eventManager *EventManager, + graphRepository interfaces.RetrieveGraphRepository, + chunkRepository interfaces.ChunkRepository, + knowledgeRepository interfaces.KnowledgeRepository, +) *PluginSearchEntity { + res := &PluginSearchEntity{ + graphRepo: graphRepository, + chunkRepo: chunkRepository, + knowledgeRepo: knowledgeRepository, + } + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginSearchEntity) ActivationEvents() []types.EventType { + return []types.EventType{types.ENTITY_SEARCH} +} + +// OnEvent handles search events in the chat pipeline +func (p *PluginSearchEntity) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + entity := chatManage.Entity + if len(entity) == 0 { + logger.Infof(ctx, "No entity found") + return next() + } + + graph, err := p.graphRepo.SearchNode(ctx, types.NameSpace{KnowledgeBase: chatManage.KnowledgeBaseID}, entity) + if err != nil { + logger.Errorf(ctx, "Failed to search node, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + chatManage.GraphResult = graph + logger.Infof(ctx, "search entity result count: %d", len(graph.Node)) + // graphStr, _ := json.Marshal(graph) + // logger.Debugf(ctx, "search entity result: %s", string(graphStr)) + + chunkIDs := filterSeenChunk(ctx, graph, chatManage.SearchResult) + if len(chunkIDs) == 0 { + logger.Infof(ctx, "No new chunk found") + return next() + } + chunks, err := p.chunkRepo.ListChunksByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), chunkIDs) + if err != nil { + logger.Errorf(ctx, "Failed to list chunks, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + knowledgeIDs := []string{} + for _, chunk := range chunks { + knowledgeIDs = append(knowledgeIDs, chunk.KnowledgeID) + } + knowledges, err := p.knowledgeRepo.GetKnowledgeBatch(ctx, ctx.Value(types.TenantIDContextKey).(uint), knowledgeIDs) + if err != nil { + logger.Errorf(ctx, "Failed to list knowledge, session_id: %s, error: %v", chatManage.SessionID, err) + return next() + } + + knowledgeMap := map[string]*types.Knowledge{} + for _, knowledge := range knowledges { + knowledgeMap[knowledge.ID] = knowledge + } + for _, chunk := range chunks { + searchResult := chunk2SearchResult(chunk, knowledgeMap[chunk.KnowledgeID]) + chatManage.SearchResult = append(chatManage.SearchResult, searchResult) + } + // remove duplicate results + chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult) + if len(chatManage.SearchResult) == 0 { + logger.Infof(ctx, "No new search result, session_id: %s", chatManage.SessionID) + return ErrSearchNothing + } + logger.Infof(ctx, "search entity result count: %d, session_id: %s", len(chatManage.SearchResult), chatManage.SessionID) + return next() +} + +func filterSeenChunk(ctx context.Context, graph *types.GraphData, searchResult []*types.SearchResult) []string { + seen := map[string]bool{} + for _, chunk := range searchResult { + seen[chunk.ID] = true + } + logger.Infof(ctx, "filterSeenChunk: seen count: %d", len(seen)) + + chunkIDs := []string{} + for _, node := range graph.Node { + for _, chunkID := range node.Chunks { + if seen[chunkID] { + continue + } + seen[chunkID] = true + chunkIDs = append(chunkIDs, chunkID) + } + } + logger.Infof(ctx, "filterSeenChunk: new chunkIDs count: %d", len(chunkIDs)) + return chunkIDs +} + +func chunk2SearchResult(chunk *types.Chunk, knowledge *types.Knowledge) *types.SearchResult { + return &types.SearchResult{ + ID: chunk.ID, + Content: chunk.Content, + KnowledgeID: chunk.KnowledgeID, + ChunkIndex: chunk.ChunkIndex, + KnowledgeTitle: knowledge.Title, + StartAt: chunk.StartAt, + EndAt: chunk.EndAt, + Seq: chunk.ChunkIndex, + Score: 1.0, + MatchType: types.MatchTypeGraph, + Metadata: knowledge.GetMetadata(), + ChunkType: string(chunk.ChunkType), + ParentChunkID: chunk.ParentChunkID, + ImageInfo: chunk.ImageInfo, + KnowledgeFilename: knowledge.FileName, + KnowledgeSource: knowledge.Source, + } +} diff --git a/internal/application/service/chat_pipline/stream_filter.go b/internal/application/service/chat_pipline/stream_filter.go new file mode 100644 index 0000000..06d65a5 --- /dev/null +++ b/internal/application/service/chat_pipline/stream_filter.go @@ -0,0 +1,84 @@ +package chatpipline + +import ( + "context" + "strings" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// PluginStreamFilter implements stream filtering functionality for chat pipeline +type PluginStreamFilter struct{} + +// NewPluginStreamFilter creates a new stream filter plugin instance +func NewPluginStreamFilter(eventManager *EventManager) *PluginStreamFilter { + res := &PluginStreamFilter{} + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginStreamFilter) ActivationEvents() []types.EventType { + return []types.EventType{types.STREAM_FILTER} +} + +// OnEvent handles stream filtering events in the chat pipeline +func (p *PluginStreamFilter) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + logger.Info(ctx, "Starting stream filter") + logger.Info(ctx, "Creating new stream channel") + + // Create new stream channel and replace the original one + oldStream := chatManage.ResponseChan + newStream := make(chan types.StreamResponse) + chatManage.ResponseChan = newStream + + // Initialize response builder and check if no-match prefix filtering is needed + responseBuilder := &strings.Builder{} + matchNoMatchBuilderPrefix := chatManage.SummaryConfig.NoMatchPrefix != "" + + if matchNoMatchBuilderPrefix { + logger.Infof(ctx, "Using no match prefix filter: %s", chatManage.SummaryConfig.NoMatchPrefix) + } + + // Start goroutine to filter the stream + go func() { + logger.Info(ctx, "Starting stream filter goroutine") + for resp := range oldStream { + // Accumulate answer content + if resp.ResponseType == types.ResponseTypeAnswer { + responseBuilder.WriteString(resp.Content) + } + + // Skip filtering if no prefix matching is required + if !matchNoMatchBuilderPrefix { + newStream <- resp + continue + } + + // Check if content matches the no-match prefix + if !strings.HasPrefix(chatManage.SummaryConfig.NoMatchPrefix, responseBuilder.String()) { + resp.Content = responseBuilder.String() + newStream <- resp + logger.Info( + ctx, "Content does not match no-match prefix, passing through, content: ", + responseBuilder.String(), + ) + matchNoMatchBuilderPrefix = false + } + } + + // Handle NO_MATCH case when stream ends + if matchNoMatchBuilderPrefix { + logger.Info(ctx, "Content matches no-match prefix, using fallback response") + newStream <- NewFallback(ctx, chatManage.FallbackResponse) + } + logger.Info(ctx, "Stream filter completed, closing new stream") + close(newStream) + }() + + logger.Info(ctx, "Stream filter initialized") + return next() +} diff --git a/internal/application/service/chat_pipline/tracing.go b/internal/application/service/chat_pipline/tracing.go new file mode 100644 index 0000000..8a10e48 --- /dev/null +++ b/internal/application/service/chat_pipline/tracing.go @@ -0,0 +1,270 @@ +package chatpipline + +import ( + "context" + "encoding/json" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/tracing" + "github.com/Tencent/WeKnora/internal/types" + "go.opentelemetry.io/otel/attribute" +) + +// PluginTracing implements tracing functionality for chat pipeline events +type PluginTracing struct{} + +// NewPluginTracing creates a new tracing plugin instance +func NewPluginTracing(eventManager *EventManager) *PluginTracing { + res := &PluginTracing{} + eventManager.Register(res) + return res +} + +// ActivationEvents returns the event types this plugin handles +func (p *PluginTracing) ActivationEvents() []types.EventType { + return []types.EventType{ + types.CHUNK_SEARCH, + types.CHUNK_RERANK, + types.CHUNK_MERGE, + types.INTO_CHAT_MESSAGE, + types.CHAT_COMPLETION, + types.CHAT_COMPLETION_STREAM, + types.FILTER_TOP_K, + types.REWRITE_QUERY, + types.PREPROCESS_QUERY, + } +} + +// OnEvent handles incoming events and routes them to the appropriate tracing handler based on event type. +// It acts as the central dispatcher for all tracing-related events in the chat pipeline. +// +// Parameters: +// - ctx: context.Context for request-scoped values, cancellation signals, and deadlines +// - eventType: the type of event being processed (e.g., CHUNK_SEARCH, CHAT_COMPLETION) +// - chatManage: contains all the chat-related data and state for the current request +// - next: callback function to continue processing in the pipeline +// +// Returns: +// - *PluginError: error if any occurred during processing, or nil if successful +func (p *PluginTracing) OnEvent(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + switch eventType { + case types.CHUNK_SEARCH: + return p.Search(ctx, eventType, chatManage, next) + case types.CHUNK_RERANK: + return p.Rerank(ctx, eventType, chatManage, next) + case types.CHUNK_MERGE: + return p.Merge(ctx, eventType, chatManage, next) + case types.INTO_CHAT_MESSAGE: + return p.IntoChatMessage(ctx, eventType, chatManage, next) + case types.CHAT_COMPLETION: + return p.ChatCompletion(ctx, eventType, chatManage, next) + case types.CHAT_COMPLETION_STREAM: + return p.ChatCompletionStream(ctx, eventType, chatManage, next) + case types.FILTER_TOP_K: + return p.FilterTopK(ctx, eventType, chatManage, next) + case types.REWRITE_QUERY: + return p.RewriteQuery(ctx, eventType, chatManage, next) + case types.PREPROCESS_QUERY: + return p.PreprocessQuery(ctx, eventType, chatManage, next) + } + return next() +} + +// Search traces search operations in the chat pipeline +func (p *PluginTracing) Search(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search") + defer span.End() + span.SetAttributes( + attribute.String("query", chatManage.Query), + attribute.Float64("vector_threshold", chatManage.VectorThreshold), + attribute.Float64("keyword_threshold", chatManage.KeywordThreshold), + attribute.Int("match_count", chatManage.EmbeddingTopK), + ) + err := next() + searchResultJson, _ := json.Marshal(chatManage.SearchResult) + span.SetAttributes( + attribute.String("hybrid_search", string(searchResultJson)), + attribute.String("processed_query", chatManage.ProcessedQuery), + ) + return err +} + +// Rerank traces rerank operations in the chat pipeline +func (p *PluginTracing) Rerank(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank") + defer span.End() + span.SetAttributes( + attribute.String("query", chatManage.Query), + attribute.Int("passages_count", len(chatManage.SearchResult)), + attribute.String("rerank_model_id", chatManage.RerankModelID), + attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold), + attribute.Int("rerank_filter_topk", chatManage.RerankTopK), + ) + err := next() + resultJson, _ := json.Marshal(chatManage.RerankResult) + span.SetAttributes( + attribute.Int("rerank_resp_count", len(chatManage.RerankResult)), + attribute.String("rerank_resp_results", string(resultJson)), + ) + return err +} + +// Merge traces merge operations in the chat pipeline +func (p *PluginTracing) Merge(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.Merge") + defer span.End() + span.SetAttributes( + attribute.Int("search_results_count", len(chatManage.SearchResult)), + attribute.Int("rerank_results_count", len(chatManage.RerankResult)), + ) + err := next() + mergeResultJson, _ := json.Marshal(chatManage.MergeResult) + span.SetAttributes( + attribute.Int("merge_results_count", len(chatManage.MergeResult)), + attribute.String("merge_results", string(mergeResultJson)), + ) + return err +} + +// IntoChatMessage traces message conversion operations +func (p *PluginTracing) IntoChatMessage(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.IntoChatMessage") + defer span.End() + span.SetAttributes( + attribute.Int("search_results_count", len(chatManage.SearchResult)), + attribute.Int("rerank_results_count", len(chatManage.RerankResult)), + attribute.Int("merge_results_count", len(chatManage.MergeResult)), + ) + err := next() + span.SetAttributes(attribute.Int("generated_content_length", len(chatManage.UserContent))) + return err +} + +// ChatCompletion traces chat completion operations +func (p *PluginTracing) ChatCompletion(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.ChatCompletion") + defer span.End() + span.SetAttributes( + attribute.String("model_id", chatManage.ChatModelID), + attribute.String("system_prompt", chatManage.SummaryConfig.Prompt), + attribute.String("user_prompt", chatManage.UserContent), + attribute.Int("total_references", len(chatManage.RerankResult)), + ) + err := next() + span.SetAttributes( + attribute.String("chat_response", chatManage.ChatResponse.Content), + attribute.Int("chat_response_tokens", chatManage.ChatResponse.Usage.TotalTokens), + attribute.Int("chat_response_prompt_tokens", chatManage.ChatResponse.Usage.PromptTokens), + attribute.Int("chat_response_completion_tokens", chatManage.ChatResponse.Usage.CompletionTokens), + ) + return err +} + +// ChatCompletionStream traces streaming chat completion operations +func (p *PluginTracing) ChatCompletionStream(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + ctx, span := tracing.ContextWithSpan(ctx, "PluginTracing.ChatCompletionStream") + startTime := time.Now() + span.SetAttributes( + attribute.String("model_id", chatManage.ChatModelID), + attribute.String("system_prompt", chatManage.SummaryConfig.Prompt), + attribute.String("user_prompt", chatManage.UserContent), + attribute.Int("total_references", len(chatManage.RerankResult)), + ) + + responseBuilder := &strings.Builder{} + oldStream := chatManage.ResponseChan + newStream := make(chan types.StreamResponse) + chatManage.ResponseChan = newStream + + go func(ctx context.Context) { + for resp := range oldStream { + if resp.ResponseType == types.ResponseTypeAnswer { + responseBuilder.WriteString(resp.Content) + } + newStream <- resp + } + elapsedMS := time.Since(startTime).Milliseconds() + span.SetAttributes( + attribute.Bool("chat_completion_success", true), + attribute.Int64("response_time_ms", elapsedMS), + attribute.String("chat_response", responseBuilder.String()), + attribute.Int("final_response_length", responseBuilder.Len()), + attribute.Float64("tokens_per_second", float64(responseBuilder.Len())/float64(elapsedMS)*1000), + ) + span.End() + close(newStream) + }(ctx) + + return next() +} + +// FilterTopK traces filtering operations in the chat pipeline +func (p *PluginTracing) FilterTopK(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.FilterTopK") + defer span.End() + span.SetAttributes( + attribute.Int("before_filter_search_results_count", len(chatManage.SearchResult)), + attribute.Int("before_filter_rerank_results_count", len(chatManage.RerankResult)), + attribute.Int("before_filter_merge_results_count", len(chatManage.MergeResult)), + ) + err := next() + span.SetAttributes( + attribute.Int("after_filter_search_results_count", len(chatManage.SearchResult)), + attribute.Int("after_filter_rerank_results_count", len(chatManage.RerankResult)), + attribute.Int("after_filter_merge_results_count", len(chatManage.MergeResult)), + ) + return err +} + +// RewriteQuery traces query rewriting operations +func (p *PluginTracing) RewriteQuery(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.RewriteQuery") + defer span.End() + span.SetAttributes( + attribute.String("query", chatManage.Query), + ) + err := next() + span.SetAttributes( + attribute.String("rewrite_query", chatManage.RewriteQuery), + ) + return err +} + +// PreprocessQuery traces query preprocessing operations +func (p *PluginTracing) PreprocessQuery(ctx context.Context, + eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, +) *PluginError { + _, span := tracing.ContextWithSpan(ctx, "PluginTracing.PreprocessQuery") + defer span.End() + + span.SetAttributes( + attribute.String("query", chatManage.Query), + ) + + err := next() + + span.SetAttributes( + attribute.String("processed_query", chatManage.ProcessedQuery), + ) + + return err +} diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go new file mode 100644 index 0000000..b82a4bd --- /dev/null +++ b/internal/application/service/chunk.go @@ -0,0 +1,278 @@ +// Package service provides business logic implementations for WeKnora application +// This package contains service layer implementations that coordinate between +// repositories and handlers, applying business rules and transaction management +package service + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// chunkService implements the ChunkService interface +// It provides operations for managing document chunks in the knowledge base +// Chunks are segments of documents that have been processed and prepared for indexing +type chunkService struct { + chunkRepository interfaces.ChunkRepository // Repository for chunk data persistence + kbRepository interfaces.KnowledgeBaseRepository + modelService interfaces.ModelService +} + +// NewChunkService creates a new chunk service +// It initializes a service with the provided chunk repository +// Parameters: +// - chunkRepository: Repository for chunk operations +// +// Returns: +// - interfaces.ChunkService: Initialized chunk service implementation +func NewChunkService( + chunkRepository interfaces.ChunkRepository, + kbRepository interfaces.KnowledgeBaseRepository, + modelService interfaces.ModelService, +) interfaces.ChunkService { + return &chunkService{ + chunkRepository: chunkRepository, + kbRepository: kbRepository, + modelService: modelService, + } +} + +// CreateChunks creates multiple chunks +// This method persists a batch of document chunks to the repository +// Parameters: +// - ctx: Context with authentication and request information +// - chunks: Slice of document chunks to create +// +// Returns: +// - error: Any error encountered during chunk creation +func (s *chunkService) CreateChunks(ctx context.Context, chunks []*types.Chunk) error { + logger.Info(ctx, "Start creating chunks") + logger.Infof(ctx, "Creating %d chunks", len(chunks)) + + err := s.chunkRepository.CreateChunks(ctx, chunks) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "chunk_count": len(chunks), + }) + return err + } + + logger.Info(ctx, "Chunks created successfully") + return nil +} + +// GetChunkByID retrieves a chunk by its ID +// This method fetches a specific chunk using its ID and validates tenant access +// Parameters: +// - ctx: Context with authentication and request information +// - knowledgeID: ID of the knowledge document containing the chunk +// - id: ID of the chunk to retrieve +// +// Returns: +// - *types.Chunk: Retrieved chunk if found +// - error: Any error encountered during retrieval +func (s *chunkService) GetChunkByID(ctx context.Context, knowledgeID string, id string) (*types.Chunk, error) { + logger.Info(ctx, "Start getting chunk by ID") + logger.Infof(ctx, "Getting chunk, ID: %s, knowledge ID: %s", id, knowledgeID) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + chunk, err := s.chunkRepository.GetChunkByID(ctx, tenantID, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "chunk_id": id, + "knowledge_id": knowledgeID, + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Info(ctx, "Chunk retrieved successfully") + return chunk, nil +} + +// ListChunksByKnowledgeID lists all chunks for a knowledge ID +// This method retrieves all chunks belonging to a specific knowledge document +// Parameters: +// - ctx: Context with authentication and request information +// - knowledgeID: ID of the knowledge document +// +// Returns: +// - []*types.Chunk: List of chunks belonging to the knowledge document +// - error: Any error encountered during retrieval +func (s *chunkService) ListChunksByKnowledgeID(ctx context.Context, knowledgeID string) ([]*types.Chunk, error) { + logger.Info(ctx, "Start listing chunks by knowledge ID") + logger.Infof(ctx, "Knowledge ID: %s", knowledgeID) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + chunks, err := s.chunkRepository.ListChunksByKnowledgeID(ctx, tenantID, knowledgeID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": knowledgeID, + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Infof(ctx, "Retrieved %d chunks successfully", len(chunks)) + return chunks, nil +} + +// ListPagedChunksByKnowledgeID lists chunks for a knowledge ID with pagination +// This method retrieves chunks with pagination support for better performance with large datasets +// Parameters: +// - ctx: Context with authentication and request information +// - knowledgeID: ID of the knowledge document +// - page: Pagination parameters including page number and page size +// +// Returns: +// - *types.PageResult: Paginated result containing chunks and pagination metadata +// - error: Any error encountered during retrieval +func (s *chunkService) ListPagedChunksByKnowledgeID(ctx context.Context, + knowledgeID string, page *types.Pagination, +) (*types.PageResult, error) { + logger.Info(ctx, "Start listing paged chunks by knowledge ID") + logger.Infof(ctx, "Knowledge ID: %s, page: %d, page size: %d", knowledgeID, page.Page, page.PageSize) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + chunkType := []types.ChunkType{types.ChunkTypeText} + chunks, total, err := s.chunkRepository.ListPagedChunksByKnowledgeID(ctx, tenantID, knowledgeID, page, chunkType) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": knowledgeID, + "tenant_id": tenantID, + "page": page.Page, + "page_size": page.PageSize, + }) + return nil, err + } + + logger.Infof(ctx, "Retrieved %d chunks out of %d total chunks", len(chunks), total) + return types.NewPageResult(total, page, chunks), nil +} + +// updateChunk updates a chunk +// This method updates an existing chunk in the repository +// Parameters: +// - ctx: Context with authentication and request information +// - chunk: Chunk with updated fields +// +// Returns: +// - error: Any error encountered during update +// +// This method handles the actual update logic for a chunk, including updating the vector database representation +func (s *chunkService) UpdateChunk(ctx context.Context, chunk *types.Chunk) error { + logger.Infof(ctx, "Updating chunk, ID: %s, knowledge ID: %s", chunk.ID, chunk.KnowledgeID) + + // Update the chunk in the repository + err := s.chunkRepository.UpdateChunk(ctx, chunk) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "chunk_id": chunk.ID, + "knowledge_id": chunk.KnowledgeID, + }) + return err + } + + logger.Info(ctx, "Chunk updated successfully") + return nil +} + +// DeleteChunk deletes a chunk by ID +// This method removes a specific chunk from the repository +// Parameters: +// - ctx: Context with authentication and request information +// - id: ID of the chunk to delete +// +// Returns: +// - error: Any error encountered during deletion +func (s *chunkService) DeleteChunk(ctx context.Context, id string) error { + logger.Info(ctx, "Start deleting chunk") + logger.Infof(ctx, "Deleting chunk, ID: %s", id) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + err := s.chunkRepository.DeleteChunk(ctx, tenantID, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "chunk_id": id, + "tenant_id": tenantID, + }) + return err + } + + logger.Info(ctx, "Chunk deleted successfully") + return nil +} + +// DeleteChunksByKnowledgeID deletes all chunks for a knowledge ID +// This method removes all chunks belonging to a specific knowledge document +// Parameters: +// - ctx: Context with authentication and request information +// - knowledgeID: ID of the knowledge document +// +// Returns: +// - error: Any error encountered during bulk deletion +func (s *chunkService) DeleteChunksByKnowledgeID(ctx context.Context, knowledgeID string) error { + logger.Info(ctx, "Start deleting all chunks by knowledge ID") + logger.Infof(ctx, "Knowledge ID: %s", knowledgeID) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + err := s.chunkRepository.DeleteChunksByKnowledgeID(ctx, tenantID, knowledgeID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": knowledgeID, + "tenant_id": tenantID, + }) + return err + } + + logger.Info(ctx, "All chunks under knowledge deleted successfully") + return nil +} + +func (s *chunkService) DeleteByKnowledgeList(ctx context.Context, ids []string) error { + logger.Info(ctx, "Start deleting all chunks by knowledge IDs") + logger.Infof(ctx, "Knowledge IDs: %v", ids) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + err := s.chunkRepository.DeleteByKnowledgeList(ctx, tenantID, ids) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": ids, + "tenant_id": tenantID, + }) + return err + } + + logger.Info(ctx, "All chunks under knowledge deleted successfully") + return nil +} + +func (s *chunkService) ListChunkByParentID(ctx context.Context, tenantID uint, parentID string) ([]*types.Chunk, error) { + logger.Info(ctx, "Start listing chunk by parent ID") + logger.Infof(ctx, "Parent ID: %s", parentID) + + chunks, err := s.chunkRepository.ListChunkByParentID(ctx, tenantID, parentID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "parent_id": parentID, + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Info(ctx, "Chunk listed successfully") + return chunks, nil +} diff --git a/internal/application/service/dataset.go b/internal/application/service/dataset.go new file mode 100644 index 0000000..259c656 --- /dev/null +++ b/internal/application/service/dataset.go @@ -0,0 +1,243 @@ +package service + +import ( + "context" + "errors" + "fmt" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/parquet-go/parquet-go" +) + +// DatasetService provides operations for working with datasets +type DatasetService struct{} + +// NewDatasetService creates a new DatasetService instance +func NewDatasetService() interfaces.DatasetService { + return &DatasetService{} +} + +// TextInfo represents text data with ID in parquet format +type TextInfo struct { + ID int64 `parquet:"id"` // Unique identifier + Text string `parquet:"text"` // Text content +} + +// RelsInfo represents question-passage relations in parquet format +type RelsInfo struct { + QID int64 `parquet:"qid"` // Question ID + PID int64 `parquet:"pid"` // Passage ID +} + +// QaInfo represents question-answer relations in parquet format +type QaInfo struct { + QID int64 `parquet:"qid"` // Question ID + AID int64 `parquet:"aid"` // Answer ID +} + +// GetDatasetByID retrieves QA pairs from dataset by ID +func (d *DatasetService) GetDatasetByID(ctx context.Context, datasetID string) ([]*types.QAPair, error) { + logger.Info(ctx, "Start getting dataset by ID") + logger.Infof(ctx, "Getting dataset with ID: %s", datasetID) + + dataset := DefaultDataset() + dataset.PrintStats(ctx) + qaPairs := dataset.Iterate() + + logger.Infof(ctx, "Retrieved %d QA pairs from dataset", len(qaPairs)) + return qaPairs, nil +} + +// DefaultDataset loads and initializes the default dataset from parquet files +func DefaultDataset() dataset { + datasetDir := "./dataset/samples" + queries, err := loadParquet[TextInfo](fmt.Sprintf("%s/queries.parquet", datasetDir)) + if err != nil { + panic(err) + } + corpus, err := loadParquet[TextInfo](fmt.Sprintf("%s/corpus.parquet", datasetDir)) + if err != nil { + panic(err) + } + answers, err := loadParquet[TextInfo](fmt.Sprintf("%s/answers.parquet", datasetDir)) + if err != nil { + panic(err) + } + qrels, err := loadParquet[RelsInfo](fmt.Sprintf("%s/qrels.parquet", datasetDir)) + if err != nil { + panic(err) + } + qas, err := loadParquet[QaInfo](fmt.Sprintf("%s/qas.parquet", datasetDir)) + if err != nil { + panic(err) + } + + res := dataset{ + queries: make(map[int64]string), // qid -> question text + corpus: make(map[int64]string), // pid -> passage text + answers: make(map[int64]string), // aid -> answer text + qrels: make(map[int64][]int64), // qid -> list of pid + qas: make(map[int64]int64), // qid -> aid + } + for _, qi := range queries { + res.queries[qi.ID] = qi.Text + } + for _, ci := range corpus { + res.corpus[ci.ID] = ci.Text + } + for _, ai := range answers { + res.answers[ai.ID] = ai.Text + } + for _, ri := range qrels { + res.qrels[ri.QID] = append(res.qrels[ri.QID], ri.PID) + } + for _, qi := range qas { + res.qas[qi.QID] = qi.AID + } + return res +} + +// dataset represents the in-memory dataset structure +type dataset struct { + queries map[int64]string // qid -> question text + corpus map[int64]string // pid -> passage text + answers map[int64]string // aid -> answer text + qrels map[int64][]int64 // qid -> list of related pids + qas map[int64]int64 // qid -> aid +} + +// Iterate generates QA pairs from the dataset +func (d *dataset) Iterate() []*types.QAPair { + var pairs []*types.QAPair + + for qid, question := range d.queries { + // Get answer info + aid, hasAnswer := d.qas[qid] + answer := "" + if hasAnswer { + answer = d.answers[aid] + } + + // Get related passages + pids := d.qrels[qid] + var pidStr []int + for _, pid := range pids { + pidStr = append(pidStr, int(pid)) + } + var passages []string + for _, pid := range pids { + passages = append(passages, d.corpus[pid]) + } + + pairs = append(pairs, &types.QAPair{ + QID: int(qid), + Question: question, + PIDs: pidStr, + Passages: passages, + AID: int(aid), + Answer: answer, + }) + } + + return pairs +} + +// GetContextForQID retrieves context passages for a given question ID +func (d *dataset) GetContextForQID(qid int64) ([]string, error) { + pids, ok := d.qrels[qid] + if !ok { + return nil, errors.New("question ID not found") + } + + var contextParts []string + for _, pid := range pids { + if text, exists := d.corpus[pid]; exists { + contextParts = append(contextParts, text) + } + } + + return contextParts, nil +} + +// PrintStats prints dataset statistics to the logger +func (d *dataset) PrintStats(ctx context.Context) { + logger.Infof(ctx, "QA System Statistics:") + logger.Infof(ctx, "- Total queries: %d", len(d.queries)) + logger.Infof(ctx, "- Total corpus passages: %d", len(d.corpus)) + logger.Infof(ctx, "- Total answers: %d", len(d.answers)) + + // Calculate average passages per query + totalRelations := 0 + for _, pids := range d.qrels { + totalRelations += len(pids) + } + avgPassages := float64(totalRelations) / float64(len(d.qrels)) + logger.Infof(ctx, "- Average passages per query: %.2f", avgPassages) + + // Calculate coverage + coveredQueries := len(d.qas) + coverage := float64(coveredQueries) / float64(len(d.queries)) * 100 + logger.Infof(ctx, "- Answer coverage: %.2f%% (%d/%d)", coverage, coveredQueries, len(d.queries)) +} + +// PrintRandomQA prints a random question with its related passages and answer +func (d *dataset) PrintRandomQA() error { + // Get a random qid + var qid int64 + for k := range d.qas { + qid = k + break + } + if qid == 0 { + return errors.New("no questions available") + } + + // Get question text + question, ok := d.queries[qid] + if !ok { + return fmt.Errorf("question %d not found", qid) + } + + // Get answer info + aid, ok := d.qas[qid] + if !ok { + return fmt.Errorf("answer for question %d not found", qid) + } + answer, ok := d.answers[aid] + if !ok { + return fmt.Errorf("answer %d not found", aid) + } + + // Print formatted QA + fmt.Println("===== Random QA =====") + fmt.Printf("QID: %d\n", qid) + fmt.Printf("Question: %s\n", question) + + // Print passages if available + if pids, exists := d.qrels[qid]; exists && len(pids) > 0 { + fmt.Println("\nRelated passages:") + for i, pid := range pids { + if text, exists := d.corpus[pid]; exists { + fmt.Printf("\nPassage %d (PID: %d):\n%s\n", i+1, pid, text) + } + } + } else { + fmt.Println("\nNo related passages found") + } + + // Print answer + fmt.Printf("\nAnswer (AID: %d):\n%s\n", aid, answer) + + return nil +} + +// loadParquet loads data from parquet file into specified type +func loadParquet[T any](filePath string) ([]T, error) { + rows, err := parquet.ReadFile[T](filePath) + if err != nil { + return nil, err + } + return rows, nil +} diff --git a/internal/application/service/evaluation.go b/internal/application/service/evaluation.go new file mode 100644 index 0000000..391a580 --- /dev/null +++ b/internal/application/service/evaluation.go @@ -0,0 +1,454 @@ +package service + +import ( + "context" + "errors" + "fmt" + "runtime" + "sync" + "time" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" + "golang.org/x/sync/errgroup" +) + +/* +corpus: pid -> content +queries: qid -> content +answers: aid -> content +qrels: qid -> pid +arels: qid -> aid +*/ + +// EvaluationService handles evaluation tasks for knowledge base and chat models +type EvaluationService struct { + config *config.Config // Application configuration + dataset interfaces.DatasetService // Service for dataset operations + knowledgeBaseService interfaces.KnowledgeBaseService // Service for knowledge base operations + knowledgeService interfaces.KnowledgeService // Service for knowledge operations + sessionService interfaces.SessionService // Service for chat sessions + modelService interfaces.ModelService // Service for model operations + + evaluationMemoryStorage *evaluationMemoryStorage // In-memory storage for evaluation tasks +} + +func NewEvaluationService( + config *config.Config, + dataset interfaces.DatasetService, + knowledgeBaseService interfaces.KnowledgeBaseService, + knowledgeService interfaces.KnowledgeService, + sessionService interfaces.SessionService, + modelService interfaces.ModelService, +) interfaces.EvaluationService { + evaluationMemoryStorage := newEvaluationMemoryStorage() + return &EvaluationService{ + config: config, + dataset: dataset, + knowledgeBaseService: knowledgeBaseService, + knowledgeService: knowledgeService, + sessionService: sessionService, + modelService: modelService, + evaluationMemoryStorage: evaluationMemoryStorage, + } +} + +// evaluationMemoryStorage stores evaluation tasks in memory with thread-safe access +type evaluationMemoryStorage struct { + store map[string]*types.EvaluationDetail // Map of taskID to evaluation details + mu *sync.RWMutex // Read-write lock for concurrent access +} + +func newEvaluationMemoryStorage() *evaluationMemoryStorage { + res := &evaluationMemoryStorage{ + store: make(map[string]*types.EvaluationDetail), + mu: &sync.RWMutex{}, + } + return res +} + +func (e *evaluationMemoryStorage) register(params *types.EvaluationDetail) { + e.mu.Lock() + defer e.mu.Unlock() + logger.Infof(context.Background(), "Registering evaluation task: %s", params.Task.ID) + e.store[params.Task.ID] = params +} + +func (e *evaluationMemoryStorage) get(taskID string) (*types.EvaluationDetail, error) { + e.mu.RLock() + defer e.mu.RUnlock() + logger.Infof(context.Background(), "Getting evaluation task: %s", taskID) + res, ok := e.store[taskID] + if !ok { + return nil, errors.New("task not found") + } + return res, nil +} + +func (e *evaluationMemoryStorage) update(taskID string, fn func(params *types.EvaluationDetail)) error { + e.mu.Lock() + defer e.mu.Unlock() + params, ok := e.store[taskID] + if !ok { + return errors.New("task not found") + } + fn(params) + return nil +} + +func (e *EvaluationService) EvaluationResult(ctx context.Context, taskID string) (*types.EvaluationDetail, error) { + logger.Info(ctx, "Start getting evaluation result") + logger.Infof(ctx, "Task ID: %s", taskID) + + detail, err := e.evaluationMemoryStorage.get(taskID) + if err != nil { + logger.Errorf(ctx, "Failed to get evaluation task: %v", err) + return nil, err + } + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof( + ctx, + "Checking tenant ID match, task tenant ID: %d, current tenant ID: %d", + detail.Task.TenantID, tenantID, + ) + + if tenantID != detail.Task.TenantID { + logger.Error(ctx, "Tenant ID mismatch") + return nil, errors.New("tenant ID does not match") + } + + logger.Info(ctx, "Evaluation result retrieved successfully") + return detail, nil +} + +// Evaluation starts a new evaluation task with given parameters +// datasetID: ID of the dataset to evaluate against +// knowledgeBaseID: ID of the knowledge base to use (empty to create new) +// chatModelID: ID of the chat model to evaluate +// rerankModelID: ID of the rerank model to evaluate +func (e *EvaluationService) Evaluation(ctx context.Context, + datasetID string, knowledgeBaseID string, chatModelID string, rerankModelID string, +) (*types.EvaluationDetail, error) { + logger.Info(ctx, "Start evaluation") + logger.Infof(ctx, "Dataset ID: %s, Knowledge Base ID: %s, Chat Model ID: %s, Rerank Model ID: %s", + datasetID, knowledgeBaseID, chatModelID, rerankModelID) + + // Get tenant ID from context for multi-tenancy support + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + // Handle knowledge base creation if not provided + if knowledgeBaseID == "" { + logger.Info(ctx, "No knowledge base ID provided, creating new knowledge base") + // Create new knowledge base with default evaluation settings + // 获取默认的嵌入模型和LLM模型 + models, err := e.modelService.ListModels(ctx) + if err != nil { + logger.Errorf(ctx, "Failed to list models: %v", err) + return nil, err + } + + var embeddingModelID, llmModelID string + for _, model := range models { + if model.Type == types.ModelTypeEmbedding { + embeddingModelID = model.ID + } + if model.Type == types.ModelTypeKnowledgeQA { + llmModelID = model.ID + } + } + + if embeddingModelID == "" || llmModelID == "" { + return nil, fmt.Errorf("no default models found for evaluation") + } + + kb, err := e.knowledgeBaseService.CreateKnowledgeBase(ctx, &types.KnowledgeBase{ + Name: "evaluation", + Description: "evaluation", + EmbeddingModelID: embeddingModelID, + SummaryModelID: llmModelID, + }) + if err != nil { + logger.Errorf(ctx, "Failed to create knowledge base: %v", err) + return nil, err + } + knowledgeBaseID = kb.ID + logger.Infof(ctx, "Created new knowledge base with ID: %s", knowledgeBaseID) + } else { + logger.Infof(ctx, "Using existing knowledge base ID: %s", knowledgeBaseID) + // Create evaluation-specific knowledge base based on existing one + kb, err := e.knowledgeBaseService.GetKnowledgeBaseByID(ctx, knowledgeBaseID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge base: %v", err) + return nil, err + } + + kb, err = e.knowledgeBaseService.CreateKnowledgeBase(ctx, &types.KnowledgeBase{ + Name: "evaluation", + Description: "evaluation", + EmbeddingModelID: kb.EmbeddingModelID, + SummaryModelID: kb.SummaryModelID, + }) + if err != nil { + logger.Errorf(ctx, "Failed to create knowledge base: %v", err) + return nil, err + } + knowledgeBaseID = kb.ID + logger.Infof(ctx, "Created new knowledge base with ID: %s based on existing one", knowledgeBaseID) + } + + // Set default values for optional parameters + if datasetID == "" { + datasetID = "default" + logger.Info(ctx, "Using default dataset") + } + + if rerankModelID == "" { + // 获取默认的重排模型 + models, err := e.modelService.ListModels(ctx) + if err == nil { + for _, model := range models { + if model.Type == types.ModelTypeRerank { + rerankModelID = model.ID + break + } + } + } + if rerankModelID == "" { + logger.Warnf(ctx, "No rerank model found, skipping rerank") + } else { + logger.Infof(ctx, "Using default rerank model: %s", rerankModelID) + } + } + + if chatModelID == "" { + // 获取默认的LLM模型 + models, err := e.modelService.ListModels(ctx) + if err == nil { + for _, model := range models { + if model.Type == types.ModelTypeKnowledgeQA { + chatModelID = model.ID + break + } + } + } + if chatModelID == "" { + return nil, fmt.Errorf("no default chat model found") + } + logger.Infof(ctx, "Using default chat model: %s", chatModelID) + } + + // Create evaluation task with unique ID + logger.Info(ctx, "Creating evaluation task") + taskID := uuid.New().String() + logger.Infof(ctx, "Generated task ID: %s", taskID) + + // Prepare evaluation detail with all parameters + detail := &types.EvaluationDetail{ + Task: &types.EvaluationTask{ + ID: taskID, + TenantID: tenantID, + DatasetID: datasetID, + Status: types.EvaluationStatuePending, + StartTime: time.Now(), + }, + Params: &types.ChatManage{ + KnowledgeBaseID: knowledgeBaseID, + VectorThreshold: e.config.Conversation.VectorThreshold, + KeywordThreshold: e.config.Conversation.KeywordThreshold, + EmbeddingTopK: e.config.Conversation.EmbeddingTopK, + RerankModelID: rerankModelID, + RerankTopK: e.config.Conversation.RerankTopK, + RerankThreshold: e.config.Conversation.RerankThreshold, + ChatModelID: chatModelID, + SummaryConfig: types.SummaryConfig{ + MaxTokens: e.config.Conversation.Summary.MaxTokens, + RepeatPenalty: e.config.Conversation.Summary.RepeatPenalty, + TopK: e.config.Conversation.Summary.TopK, + TopP: e.config.Conversation.Summary.TopP, + Prompt: e.config.Conversation.Summary.Prompt, + ContextTemplate: e.config.Conversation.Summary.ContextTemplate, + FrequencyPenalty: e.config.Conversation.Summary.FrequencyPenalty, + PresencePenalty: e.config.Conversation.Summary.PresencePenalty, + NoMatchPrefix: e.config.Conversation.Summary.NoMatchPrefix, + Temperature: e.config.Conversation.Summary.Temperature, + Seed: e.config.Conversation.Summary.Seed, + MaxCompletionTokens: e.config.Conversation.Summary.MaxCompletionTokens, + }, + FallbackResponse: e.config.Conversation.FallbackResponse, + }, + } + + // Store evaluation task in memory storage + logger.Info(ctx, "Registering evaluation task") + e.evaluationMemoryStorage.register(detail) + + // Start evaluation in background goroutine + logger.Info(ctx, "Starting evaluation in background") + go func() { + // Create new context with logger for background task + newCtx := logger.CloneContext(ctx) + logger.Infof(newCtx, "Background evaluation started for task ID: %s", taskID) + + // Update task status to running + detail.Task.Status = types.EvaluationStatueRunning + logger.Info(newCtx, "Evaluation task status set to running") + + // Execute actual evaluation + if err := e.EvalDataset(newCtx, detail); err != nil { + detail.Task.Status = types.EvaluationStatueFailed + detail.Task.ErrMsg = err.Error() + logger.Errorf(newCtx, "Evaluation task failed: %v, task ID: %s", err, taskID) + return + } + + // Mark task as completed successfully + logger.Infof(newCtx, "Evaluation task completed successfully, task ID: %s", taskID) + detail.Task.Status = types.EvaluationStatueSuccess + }() + + logger.Infof(ctx, "Evaluation task created successfully, task ID: %s", taskID) + return detail, nil +} + +// EvalDataset performs the actual evaluation of a dataset +// Processes each QA pair in parallel and records metrics +func (e *EvaluationService) EvalDataset(ctx context.Context, detail *types.EvaluationDetail) error { + logger.Info(ctx, "Start evaluating dataset") + logger.Infof(ctx, "Task ID: %s, Dataset ID: %s", detail.Task.ID, detail.Task.DatasetID) + + // Retrieve dataset from storage + dataset, err := e.dataset.GetDatasetByID(ctx, detail.Task.DatasetID) + if err != nil { + logger.Errorf(ctx, "Failed to get dataset: %v", err) + return err + } + logger.Infof(ctx, "Dataset retrieved successfully with %d QA pairs", len(dataset)) + + // Update total QA pairs count in task details + e.evaluationMemoryStorage.update(detail.Task.ID, func(params *types.EvaluationDetail) { + params.Task.Total = len(dataset) + logger.Infof(ctx, "Updated task total to %d QA pairs", params.Task.Total) + }) + + // Extract and organize passages from dataset + passages := getPassageList(dataset) + logger.Infof(ctx, "Creating knowledge from %d passages", len(passages)) + + // Create knowledge base from passages + knowledge, err := e.knowledgeService.CreateKnowledgeFromPassage(ctx, detail.Params.KnowledgeBaseID, passages) + if err != nil { + logger.Errorf(ctx, "Failed to create knowledge from passages: %v", err) + return err + } + logger.Infof(ctx, "Knowledge created successfully, ID: %s", knowledge.ID) + + // Setup cleanup of temporary resources + defer func() { + logger.Infof(ctx, "Cleaning up resources - deleting knowledge: %s", knowledge.ID) + if err := e.knowledgeService.DeleteKnowledge(ctx, knowledge.ID); err != nil { + logger.Errorf(ctx, "Failed to delete knowledge: %v, knowledge ID: %s", err, knowledge.ID) + } + + logger.Infof(ctx, "Cleaning up resources - deleting knowledge base: %s", detail.Params.KnowledgeBaseID) + if err := e.knowledgeBaseService.DeleteKnowledgeBase(ctx, detail.Params.KnowledgeBaseID); err != nil { + logger.Errorf( + ctx, + "Failed to delete knowledge base: %v, knowledge base ID: %s", + err, detail.Params.KnowledgeBaseID, + ) + } + }() + + // Initialize parallel evaluation metrics + var finished int + var mu sync.Mutex + var g errgroup.Group + metricHook := NewHookMetric(len(dataset)) + + // Set worker limit based on available CPUs + g.SetLimit(max(runtime.GOMAXPROCS(0)-1, 1)) + logger.Infof(ctx, "Starting evaluation with %d parallel workers", max(runtime.GOMAXPROCS(0)-1, 1)) + + // Process each QA pair in parallel + for i, qaPair := range dataset { + qaPair := qaPair + i := i + g.Go(func() error { + logger.Infof(ctx, "Processing QA pair %d, question: %s", i, qaPair.Question) + + // Prepare chat management parameters for this QA pair + chatManage := detail.Params.Clone() + chatManage.Query = qaPair.Question + + // Execute knowledge QA pipeline + logger.Infof(ctx, "Running knowledge QA for question: %s", qaPair.Question) + err = e.sessionService.KnowledgeQAByEvent(ctx, chatManage, types.Pipline["rag"]) + if err != nil { + logger.Errorf(ctx, "Failed to process question %d: %v", i, err) + return err + } + + // Record evaluation metrics + logger.Infof(ctx, "Recording metrics for QA pair %d", i) + metricHook.recordInit(i) + metricHook.recordQaPair(i, qaPair) + metricHook.recordSearchResult(i, chatManage.SearchResult) + metricHook.recordRerankResult(i, chatManage.RerankResult) + metricHook.recordChatResponse(i, chatManage.ChatResponse) + metricHook.recordFinish(i) + + // Update progress metrics + mu.Lock() + finished += 1 + metricResult := metricHook.MetricResult() + mu.Unlock() + e.evaluationMemoryStorage.update(detail.Task.ID, func(params *types.EvaluationDetail) { + params.Metric = metricResult + params.Task.Finished = finished + logger.Infof(ctx, "Updated task progress: %d/%d completed", finished, params.Task.Total) + }) + return nil + }) + } + + // Wait for all parallel evaluations to complete + logger.Info(ctx, "Waiting for all evaluation tasks to complete") + if err := g.Wait(); err != nil { + logger.Errorf(ctx, "Evaluation error: %v", err) + return err + } + + // Final update of evaluation metrics + e.evaluationMemoryStorage.update(detail.Task.ID, func(params *types.EvaluationDetail) { + params.Metric = metricHook.MetricResult() + params.Task.Finished = finished + }) + + logger.Infof(ctx, "Dataset evaluation completed successfully, task ID: %s", detail.Task.ID) + return nil +} + +// getPassageList extracts and organizes passages from QA pairs +// Returns a slice of passages indexed by their passage IDs +func getPassageList(dataset []*types.QAPair) []string { + pIDMap := make(map[int]string) + maxPID := 0 + for _, qaPair := range dataset { + for i := 0; i < len(qaPair.PIDs); i++ { + pIDMap[qaPair.PIDs[i]] = qaPair.Passages[i] + maxPID = max(maxPID, qaPair.PIDs[i]) + } + } + passages := make([]string, maxPID) + for i := 0; i < maxPID; i++ { + if _, ok := pIDMap[i]; ok { + passages[i] = pIDMap[i] + } + } + return passages +} diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go new file mode 100644 index 0000000..06f3db3 --- /dev/null +++ b/internal/application/service/extract.go @@ -0,0 +1,137 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" + "github.com/hibiken/asynq" +) + +func NewChunkExtractTask(ctx context.Context, client *asynq.Client, tenantID uint, chunkID string, modelID string) error { + if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" { + logger.Debugf(ctx, "NOT SUPPORT RETRIEVE GRAPH") + return nil + } + payload, err := json.Marshal(types.ExtractChunkPayload{ + TenantID: tenantID, + ChunkID: chunkID, + ModelID: modelID, + }) + if err != nil { + return err + } + task := asynq.NewTask(types.TypeChunkExtract, payload, asynq.MaxRetry(3)) + info, err := client.Enqueue(task) + if err != nil { + logger.Errorf(ctx, "failed to enqueue task: %v", err) + return fmt.Errorf("failed to enqueue task: %v", err) + } + logger.Infof(ctx, "enqueued task: id=%s queue=%s chunk=%s", info.ID, info.Queue, chunkID) + return nil +} + +type ChunkExtractService struct { + template *types.PromptTemplateStructured + modelService interfaces.ModelService + knowledgeBaseRepo interfaces.KnowledgeBaseRepository + chunkRepo interfaces.ChunkRepository + graphEngine interfaces.RetrieveGraphRepository +} + +func NewChunkExtractService( + config *config.Config, + modelService interfaces.ModelService, + knowledgeBaseRepo interfaces.KnowledgeBaseRepository, + chunkRepo interfaces.ChunkRepository, + graphEngine interfaces.RetrieveGraphRepository, +) interfaces.Extracter { + generator := chatpipline.NewQAPromptGenerator(chatpipline.NewFormater(), config.ExtractManager.ExtractGraph) + ctx := context.Background() + logger.Debugf(ctx, "chunk extract system prompt: %s", generator.System(ctx)) + logger.Debugf(ctx, "chunk extract user prompt: %s", generator.User(ctx, "demo")) + return &ChunkExtractService{ + template: config.ExtractManager.ExtractGraph, + modelService: modelService, + knowledgeBaseRepo: knowledgeBaseRepo, + chunkRepo: chunkRepo, + graphEngine: graphEngine, + } +} + +func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error { + var p types.ExtractChunkPayload + if err := json.Unmarshal(t.Payload(), &p); err != nil { + logger.Errorf(ctx, "failed to unmarshal task payload: %v", err) + return err + } + ctx = logger.WithRequestID(ctx, uuid.New().String()) + ctx = logger.WithField(ctx, "extract", p.ChunkID) + ctx = context.WithValue(ctx, types.TenantIDContextKey, p.TenantID) + + chunk, err := s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID) + if err != nil { + logger.Errorf(ctx, "failed to get chunk: %v", err) + return err + } + kb, err := s.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chunk.KnowledgeBaseID) + if err != nil { + logger.Errorf(ctx, "failed to get knowledge base: %v", err) + return err + } + if kb.ExtractConfig == nil { + logger.Warnf(ctx, "failed to get extract config") + return err + } + + chatModel, err := s.modelService.GetChatModel(ctx, p.ModelID) + if err != nil { + logger.Errorf(ctx, "failed to get chat model: %v", err) + return err + } + + template := &types.PromptTemplateStructured{ + Description: s.template.Description, + Tags: kb.ExtractConfig.Tags, + Examples: []types.GraphData{ + { + Text: kb.ExtractConfig.Text, + Node: kb.ExtractConfig.Nodes, + Relation: kb.ExtractConfig.Relations, + }, + }, + } + extractor := chatpipline.NewExtractor(chatModel, template) + graph, err := extractor.Extract(ctx, chunk.Content) + if err != nil { + return err + } + + chunk, err = s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID) + if err != nil { + logger.Warnf(ctx, "graph ignore chunk %s: %v", p.ChunkID, err) + return nil + } + + for _, node := range graph.Node { + node.Chunks = []string{chunk.ID} + } + if err = s.graphEngine.AddGraph(ctx, + types.NameSpace{KnowledgeBase: chunk.KnowledgeBaseID, Knowledge: chunk.KnowledgeID}, + []*types.GraphData{graph}, + ); err != nil { + logger.Errorf(ctx, "failed to add graph: %v", err) + return err + } + // gg, _ := json.Marshal(graph) + // logger.Infof(ctx, "extracted graph: %s", string(gg)) + return nil +} diff --git a/internal/application/service/file/cos.go b/internal/application/service/file/cos.go new file mode 100644 index 0000000..69c817f --- /dev/null +++ b/internal/application/service/file/cos.go @@ -0,0 +1,83 @@ +package file + +import ( + "context" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "path/filepath" + "strings" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" + "github.com/tencentyun/cos-go-sdk-v5" +) + +// cosFileService implements the FileService interface for Tencent Cloud COS +type cosFileService struct { + client *cos.Client + bucketURL string + cosPathPrefix string +} + +// NewCosFileService creates a new COS file service instance +func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) { + bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region) + u, err := url.Parse(bucketURL) + if err != nil { + return nil, fmt.Errorf("failed to parse bucketURL: %w", err) + } + b := &cos.BaseURL{BucketURL: u} + client := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: secretId, + SecretKey: secretKey, + }, + }) + return &cosFileService{ + client: client, + bucketURL: bucketURL, + cosPathPrefix: cosPathPrefix, + }, nil +} + +// SaveFile saves a file to COS storage +// It generates a unique name for the file and organizes it by tenant and knowledge ID +func (s *cosFileService) SaveFile(ctx context.Context, + file *multipart.FileHeader, tenantID uint, knowledgeID string, +) (string, error) { + ext := filepath.Ext(file.Filename) + objectName := fmt.Sprintf("%s/%d/%s/%s%s", s.cosPathPrefix, tenantID, knowledgeID, uuid.New().String(), ext) + src, err := file.Open() + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer src.Close() + _, err = s.client.Object.Put(ctx, objectName, src, nil) + if err != nil { + return "", fmt.Errorf("failed to upload file to COS: %w", err) + } + return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil +} + +// GetFile retrieves a file from COS storage by its path URL +func (s *cosFileService) GetFile(ctx context.Context, filePathUrl string) (io.ReadCloser, error) { + objectName := strings.TrimPrefix(filePathUrl, s.bucketURL) + resp, err := s.client.Object.Get(ctx, objectName, nil) + if err != nil { + return nil, fmt.Errorf("failed to get file from COS: %w", err) + } + return resp.Body, nil +} + +// DeleteFile removes a file from COS storage +func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error { + objectName := strings.TrimPrefix(filePath, s.bucketURL) + _, err := s.client.Object.Delete(ctx, objectName) + if err != nil { + return fmt.Errorf("failed to delete file: %w", err) + } + return nil +} diff --git a/internal/application/service/file/dummy.go b/internal/application/service/file/dummy.go new file mode 100644 index 0000000..3c9e5ce --- /dev/null +++ b/internal/application/service/file/dummy.go @@ -0,0 +1,38 @@ +package file + +import ( + "context" + "errors" + "io" + "mime/multipart" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" +) + +// DummyFileService is a no-op implementation of the FileService interface +// used for testing or when file storage is not required +type DummyFileService struct{} + +// NewDummyFileService creates a new instance of DummyFileService +func NewDummyFileService() interfaces.FileService { + return &DummyFileService{} +} + +// SaveFile pretends to save a file but just returns a random UUID +// This is useful for testing without actual file operations +func (s *DummyFileService) SaveFile(ctx context.Context, + file *multipart.FileHeader, tenantID uint, knowledgeID string, +) (string, error) { + return uuid.New().String(), nil +} + +// GetFile always returns an error as dummy service doesn't store files +func (s *DummyFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + return nil, errors.New("not implemented") +} + +// DeleteFile is a no-op operation that always succeeds +func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) error { + return nil +} diff --git a/internal/application/service/file/local.go b/internal/application/service/file/local.go new file mode 100644 index 0000000..de2c682 --- /dev/null +++ b/internal/application/service/file/local.go @@ -0,0 +1,111 @@ +package file + +import ( + "context" + "fmt" + "io" + "mime/multipart" + "os" + "path/filepath" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// localFileService implements the FileService interface for local file system storage +type localFileService struct { + baseDir string // Base directory for file storage +} + +// NewLocalFileService creates a new local file service instance +func NewLocalFileService(baseDir string) interfaces.FileService { + return &localFileService{ + baseDir: baseDir, + } +} + +// SaveFile stores an uploaded file to the local file system +// The file is stored in a directory structure: baseDir/tenantID/knowledgeID/filename +// Returns the full file path or an error if saving fails +func (s *localFileService) SaveFile(ctx context.Context, + file *multipart.FileHeader, tenantID uint, knowledgeID string, +) (string, error) { + logger.Info(ctx, "Starting to save file locally") + logger.Infof(ctx, "File information: name=%s, size=%d, tenant ID=%d, knowledge ID=%s", + file.Filename, file.Size, tenantID, knowledgeID) + + // Create storage directory with tenant and knowledge ID + dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), knowledgeID) + logger.Infof(ctx, "Creating directory: %s", dir) + if err := os.MkdirAll(dir, 0o755); err != nil { + logger.Errorf(ctx, "Failed to create directory: %v", err) + return "", fmt.Errorf("failed to create directory: %w", err) + } + + // Generate unique filename using timestamp + ext := filepath.Ext(file.Filename) + filename := fmt.Sprintf("%d%s", time.Now().UnixNano(), ext) + filePath := filepath.Join(dir, filename) + logger.Infof(ctx, "Generated file path: %s", filePath) + + // Open source file for reading + logger.Info(ctx, "Opening source file") + src, err := file.Open() + if err != nil { + logger.Errorf(ctx, "Failed to open source file: %v", err) + return "", fmt.Errorf("failed to open file: %w", err) + } + defer src.Close() + + // Create destination file for writing + logger.Info(ctx, "Creating destination file") + dst, err := os.Create(filePath) + if err != nil { + logger.Errorf(ctx, "Failed to create destination file: %v", err) + return "", fmt.Errorf("failed to create file: %w", err) + } + defer dst.Close() + + // Copy content from source to destination + logger.Info(ctx, "Copying file content") + if _, err := io.Copy(dst, src); err != nil { + logger.Errorf(ctx, "Failed to copy file content: %v", err) + return "", fmt.Errorf("failed to save file: %w", err) + } + + logger.Infof(ctx, "File saved successfully: %s", filePath) + return filePath, nil +} + +// GetFile retrieves a file from the local file system by its path +// Returns a ReadCloser for reading the file content +func (s *localFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + logger.Infof(ctx, "Getting file: %s", filePath) + + // Open the file for reading + file, err := os.Open(filePath) + if err != nil { + logger.Errorf(ctx, "Failed to open file: %v", err) + return nil, fmt.Errorf("failed to open file: %w", err) + } + + logger.Info(ctx, "File opened successfully") + return file, nil +} + +// DeleteFile removes a file from the local file system +// Returns an error if deletion fails +func (s *localFileService) DeleteFile(ctx context.Context, filePath string) error { + logger.Infof(ctx, "Deleting file: %s", filePath) + + // Remove the file + err := os.Remove(filePath) + if err != nil { + logger.Errorf(ctx, "Failed to delete file: %v", err) + return fmt.Errorf("failed to delete file: %w", err) + } + + logger.Info(ctx, "File deleted successfully") + return nil +} diff --git a/internal/application/service/file/minio.go b/internal/application/service/file/minio.go new file mode 100644 index 0000000..af2141f --- /dev/null +++ b/internal/application/service/file/minio.go @@ -0,0 +1,126 @@ +package file + +import ( + "context" + "fmt" + "io" + "mime/multipart" + "path/filepath" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// minioFileService MinIO file service implementation +type minioFileService struct { + client *minio.Client + bucketName string +} + +// NewMinioFileService creates a MinIO file service +func NewMinioFileService(endpoint, + accessKeyID, secretAccessKey, bucketName string, useSSL bool) (interfaces.FileService, error) { + // Initialize MinIO client + client, err := minio.New(endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: useSSL, + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize MinIO client: %w", err) + } + + // Check if bucket exists, create if not + exists, err := client.BucketExists(context.Background(), bucketName) + if err != nil { + return nil, fmt.Errorf("failed to check bucket: %w", err) + } + + if !exists { + err = client.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create bucket: %w", err) + } + } + + return &minioFileService{ + client: client, + bucketName: bucketName, + }, nil +} + +// SaveFile saves a file to MinIO +func (s *minioFileService) SaveFile(ctx context.Context, + file *multipart.FileHeader, tenantID uint, knowledgeID string, +) (string, error) { + // Generate object name + ext := filepath.Ext(file.Filename) + objectName := fmt.Sprintf("%d/%s/%s%s", tenantID, knowledgeID, uuid.New().String(), ext) + + // Open file + src, err := file.Open() + if err != nil { + return "", fmt.Errorf("failed to open file: %w", err) + } + defer src.Close() + + // Upload file to MinIO + _, err = s.client.PutObject(ctx, s.bucketName, objectName, src, file.Size, minio.PutObjectOptions{ + ContentType: file.Header.Get("Content-Type"), + }) + if err != nil { + return "", fmt.Errorf("failed to upload file to MinIO: %w", err) + } + + // Return the complete path to the object + return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil +} + +// GetFile gets a file from MinIO +func (s *minioFileService) GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) { + // Parse MinIO path + // Format: minio://bucketName/objectName + if len(filePath) < 9 || filePath[:8] != "minio://" { + return nil, fmt.Errorf("invalid MinIO file path: %s", filePath) + } + + // Extract object name + objectName := filePath[9+len(s.bucketName):] + if objectName[0] == '/' { + objectName = objectName[1:] + } + + // Get object + obj, err := s.client.GetObject(ctx, s.bucketName, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get file from MinIO: %w", err) + } + + return obj, nil +} + +// DeleteFile deletes a file +func (s *minioFileService) DeleteFile(ctx context.Context, filePath string) error { + // Parse MinIO path + // Format: minio://bucketName/objectName + if len(filePath) < 9 || filePath[:8] != "minio://" { + return fmt.Errorf("invalid MinIO file path: %s", filePath) + } + + // Extract object name + objectName := filePath[9+len(s.bucketName):] + if objectName[0] == '/' { + objectName = objectName[1:] + } + + // Delete object + err := s.client.RemoveObject(ctx, s.bucketName, objectName, minio.RemoveObjectOptions{ + GovernanceBypass: true, + }) + if err != nil { + return fmt.Errorf("failed to delete file: %w", err) + } + + return nil +} diff --git a/internal/application/service/graph.go b/internal/application/service/graph.go new file mode 100644 index 0000000..d1271db --- /dev/null +++ b/internal/application/service/graph.go @@ -0,0 +1,995 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "math" + "slices" + "strings" + "sync" + "time" + + "github.com/Tencent/WeKnora/internal/common" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "golang.org/x/sync/errgroup" +) + +const ( + // DefaultLLMTemperature Use low temperature for more deterministic results + DefaultLLMTemperature = 0.1 + + // PMIWeight Proportion of PMI in calculating relationship weight + PMIWeight = 0.6 + + // StrengthWeight Proportion of relationship strength in calculating relationship weight + StrengthWeight = 0.4 + + // IndirectRelationWeightDecay Decay coefficient for indirect relationship weights + IndirectRelationWeightDecay = 0.5 + + // MaxConcurrentEntityExtractions Maximum concurrency for entity extraction + MaxConcurrentEntityExtractions = 4 + + // MaxConcurrentRelationExtractions Maximum concurrency for relationship extraction + MaxConcurrentRelationExtractions = 4 + + // DefaultRelationBatchSize Default batch size for relationship extraction + DefaultRelationBatchSize = 5 + + // MinEntitiesForRelation Minimum number of entities required for relationship extraction + MinEntitiesForRelation = 2 + + // MinWeightValue Minimum weight value to avoid division by zero + MinWeightValue = 1.0 + + // WeightScaleFactor Weight scaling factor to normalize weights to 1-10 range + WeightScaleFactor = 9.0 +) + +// ChunkRelation represents a relationship between two Chunks +type ChunkRelation struct { + // Weight relationship weight, calculated based on PMI and strength + Weight float64 + + // Degree total degree of related entities + Degree int +} + +// graphBuilder implements knowledge graph construction functionality +type graphBuilder struct { + config *config.Config + entityMap map[string]*types.Entity // Entities indexed by ID + entityMapByTitle map[string]*types.Entity // Entities indexed by title + relationshipMap map[string]*types.Relationship // Relationship mapping + chatModel chat.Chat + chunkGraph map[string]map[string]*ChunkRelation // Document chunk relationship graph + mutex sync.RWMutex // Mutex for concurrent operations +} + +// NewGraphBuilder creates a new graph builder +func NewGraphBuilder(config *config.Config, chatModel chat.Chat) types.GraphBuilder { + logger.Info(context.Background(), "Creating new graph builder") + return &graphBuilder{ + config: config, + chatModel: chatModel, + entityMap: make(map[string]*types.Entity), + entityMapByTitle: make(map[string]*types.Entity), + relationshipMap: make(map[string]*types.Relationship), + chunkGraph: make(map[string]map[string]*ChunkRelation), + } +} + +// extractEntities extracts entities from text chunks +// It uses LLM to analyze text content and identify relevant entities +func (b *graphBuilder) extractEntities(ctx context.Context, chunk *types.Chunk) ([]*types.Entity, error) { + log := logger.GetLogger(ctx) + log.Infof("Extracting entities from chunk: %s", chunk.ID) + + if chunk.Content == "" { + log.Warn("Empty chunk content, skipping entity extraction") + return []*types.Entity{}, nil + } + + // Create prompt for entity extraction + thinking := false + messages := []chat.Message{ + { + Role: "system", + Content: b.config.Conversation.ExtractEntitiesPrompt, + }, + { + Role: "user", + Content: chunk.Content, + }, + } + + // Call LLM to extract entities + log.Debug("Calling LLM to extract entities") + resp, err := b.chatModel.Chat(ctx, messages, &chat.ChatOptions{ + Temperature: DefaultLLMTemperature, + Thinking: &thinking, + }) + if err != nil { + log.WithError(err).Error("Failed to extract entities from chunk") + return nil, fmt.Errorf("LLM entity extraction failed: %w", err) + } + + // Parse JSON response + var extractedEntities []*types.Entity + if err := common.ParseLLMJsonResponse(resp.Content, &extractedEntities); err != nil { + log.WithError(err).Errorf("Failed to parse entity extraction response, rsp content: %s", resp.Content) + return nil, fmt.Errorf("failed to parse entity extraction response: %w", err) + } + log.Infof("Extracted %d entities from chunk", len(extractedEntities)) + + // Print detailed entity information in a clear format + log.Info("=========== EXTRACTED ENTITIES ===========") + for i, entity := range extractedEntities { + log.Infof("[Entity %d] Title: '%s', Description: '%s'", i+1, entity.Title, entity.Description) + } + log.Info("=========================================") + + var entities []*types.Entity + + // Process entities and update entityMap + b.mutex.Lock() + defer b.mutex.Unlock() + + for _, entity := range extractedEntities { + if entity.Title == "" || entity.Description == "" { + log.WithField("entity", entity).Warn("Invalid entity with empty title or description") + continue + } + if existEntity, exists := b.entityMapByTitle[entity.Title]; !exists { + // This is a new entity + entity.ID = uuid.New().String() + entity.ChunkIDs = []string{chunk.ID} + entity.Frequency = 1 + b.entityMapByTitle[entity.Title] = entity + b.entityMap[entity.ID] = entity + entities = append(entities, entity) + log.Debugf("New entity added: %s (ID: %s)", entity.Title, entity.ID) + } else { + // Entity already exists, update its ChunkIDs + if !slices.Contains(existEntity.ChunkIDs, chunk.ID) { + existEntity.ChunkIDs = append(existEntity.ChunkIDs, chunk.ID) + log.Debugf("Updated existing entity: %s with chunk: %s", entity.Title, chunk.ID) + } + existEntity.Frequency++ + entities = append(entities, existEntity) + } + } + + log.Infof("Completed entity extraction for chunk %s: %d entities", chunk.ID, len(entities)) + return entities, nil +} + +// extractRelationships extracts relationships between entities +// It analyzes semantic connections between multiple entities and establishes relationships +func (b *graphBuilder) extractRelationships(ctx context.Context, + chunks []*types.Chunk, entities []*types.Entity) error { + log := logger.GetLogger(ctx) + log.Infof("Extracting relationships from %d entities across %d chunks", len(entities), len(chunks)) + + if len(entities) < MinEntitiesForRelation { + log.Info("Not enough entities to form relationships (minimum 2)") + return nil + } + + // Serialize entities to build prompt + entitiesJSON, err := json.Marshal(entities) + if err != nil { + log.WithError(err).Error("Failed to serialize entities to JSON") + return fmt.Errorf("failed to serialize entities: %w", err) + } + + // Merge chunk contents + content := b.mergeChunkContents(chunks) + if content == "" { + log.Warn("No content to extract relationships from") + return nil + } + + // Create relationship extraction prompt + thinking := false + messages := []chat.Message{ + { + Role: "system", + Content: b.config.Conversation.ExtractRelationshipsPrompt, + }, + { + Role: "user", + Content: fmt.Sprintf("Entities: %s\n\nText: %s", string(entitiesJSON), content), + }, + } + + // Call LLM to extract relationships + log.Debug("Calling LLM to extract relationships") + resp, err := b.chatModel.Chat(ctx, messages, &chat.ChatOptions{ + Temperature: DefaultLLMTemperature, + Thinking: &thinking, + }) + if err != nil { + log.WithError(err).Error("Failed to extract relationships") + return fmt.Errorf("LLM relationship extraction failed: %w", err) + } + + // Parse JSON response + var extractedRelationships []*types.Relationship + if err := common.ParseLLMJsonResponse(resp.Content, &extractedRelationships); err != nil { + log.WithError(err).Error("Failed to parse relationship extraction response") + return fmt.Errorf("failed to parse relationship extraction response: %w", err) + } + log.Infof("Extracted %d relationships", len(extractedRelationships)) + + // Print detailed relationship information in a clear format + log.Info("========= EXTRACTED RELATIONSHIPS =========") + for i, rel := range extractedRelationships { + log.Infof("[Relation %d] Source: '%s', Target: '%s', Description: '%s', Strength: %d", + i+1, rel.Source, rel.Target, rel.Description, rel.Strength) + } + log.Info("===========================================") + + // Process relationships and update relationshipMap + b.mutex.Lock() + defer b.mutex.Unlock() + + relationshipsAdded := 0 + relationshipsUpdated := 0 + for _, relationship := range extractedRelationships { + key := fmt.Sprintf("%s#%s", relationship.Source, relationship.Target) + relationChunkIDs := b.findRelationChunkIDs(relationship.Source, relationship.Target, entities) + if len(relationChunkIDs) == 0 { + log.Debugf("Skipping relationship %s -> %s: no common chunks", relationship.Source, relationship.Target) + continue + } + if existingRel, exists := b.relationshipMap[key]; !exists { + // This is a new relationship + relationship.ID = uuid.New().String() + relationship.ChunkIDs = relationChunkIDs + b.relationshipMap[key] = relationship + relationshipsAdded++ + log.Debugf("New relationship added: %s -> %s (ID: %s)", + relationship.Source, relationship.Target, relationship.ID) + } else { + // This relationship already exists, update its properties + chunkIDsAdded := 0 + for _, chunkID := range relationChunkIDs { + if !slices.Contains(existingRel.ChunkIDs, chunkID) { + existingRel.ChunkIDs = append(existingRel.ChunkIDs, chunkID) + chunkIDsAdded++ + } + } + // Update strength, considering weighted average of existing strength and new relationship strength + if len(existingRel.ChunkIDs) > 0 { + existingRel.Strength = (existingRel.Strength*len(existingRel.ChunkIDs) + relationship.Strength) / + (len(existingRel.ChunkIDs) + 1) + } + + if chunkIDsAdded > 0 { + relationshipsUpdated++ + log.Debugf("Updated relationship: %s -> %s with %d new chunks", + relationship.Source, relationship.Target, chunkIDsAdded) + } + } + } + + log.Infof("Relationship extraction completed: added %d, updated %d relationships", + relationshipsAdded, relationshipsUpdated) + return nil +} + +// findRelationChunkIDs finds common document chunk IDs between two entities +func (b *graphBuilder) findRelationChunkIDs(source, target string, entities []*types.Entity) []string { + relationChunkIDs := make(map[string]struct{}) + + // Collect all document chunk IDs for source and target entities + for _, entity := range entities { + if entity.Title == source || entity.Title == target { + for _, chunkID := range entity.ChunkIDs { + relationChunkIDs[chunkID] = struct{}{} + } + } + } + + if len(relationChunkIDs) == 0 { + return []string{} + } + + // Convert map keys to slice + result := make([]string, 0, len(relationChunkIDs)) + for chunkID := range relationChunkIDs { + result = append(result, chunkID) + } + return result +} + +// mergeChunkContents merges content from multiple document chunks +// It accounts for overlapping portions between chunks to ensure coherent content +func (b *graphBuilder) mergeChunkContents(chunks []*types.Chunk) string { + if len(chunks) == 0 { + return "" + } + + var chunkContents = chunks[0].Content + preChunk := chunks[0] + + for i := 1; i < len(chunks); i++ { + // Only add non-overlapping content parts + if preChunk.EndAt > chunks[i].StartAt { + // Calculate overlap starting position + startPos := preChunk.EndAt - chunks[i].StartAt + if startPos >= 0 && startPos < len([]rune(chunks[i].Content)) { + chunkContents = chunkContents + string([]rune(chunks[i].Content)[startPos:]) + } + } else { + // If there's no overlap between chunks, add all content + chunkContents = chunkContents + chunks[i].Content + } + preChunk = chunks[i] + } + + return chunkContents +} + +// BuildGraph constructs the knowledge graph +// It serves as the main entry point for the graph building process, coordinating all components +func (b *graphBuilder) BuildGraph(ctx context.Context, chunks []*types.Chunk) error { + log := logger.GetLogger(ctx) + log.Infof("Building knowledge graph from %d chunks", len(chunks)) + startTime := time.Now() + + // Concurrently extract entities from each document chunk + var chunkEntities = make([][]*types.Entity, len(chunks)) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(MaxConcurrentEntityExtractions) // Limit concurrency + + for i, chunk := range chunks { + i, chunk := i, chunk // Create local variables to avoid closure issues + g.Go(func() error { + log.Debugf("Processing chunk %d/%d (ID: %s)", i+1, len(chunks), chunk.ID) + entities, err := b.extractEntities(gctx, chunk) + if err != nil { + log.WithError(err).Errorf("Failed to extract entities from chunk %s", chunk.ID) + return fmt.Errorf("entity extraction failed for chunk %s: %w", chunk.ID, err) + } + chunkEntities[i] = entities + return nil + }) + } + + // Wait for all entity extractions to complete + if err := g.Wait(); err != nil { + log.WithError(err).Error("Entity extraction failed") + return fmt.Errorf("entity extraction process failed: %w", err) + } + + // Count total extracted entities + totalEntityCount := 0 + for _, entities := range chunkEntities { + totalEntityCount += len(entities) + } + log.Infof("Successfully extracted %d total entities across %d chunks", + totalEntityCount, len(chunks)) + + // Process relationships in batches concurrently + relationChunkSize := DefaultRelationBatchSize + log.Infof("Processing relationships concurrently in batches of %d chunks", relationChunkSize) + + // prepare relationship extraction batches + var relationBatches []struct { + batchChunks []*types.Chunk + relationUseEntities []*types.Entity + batchIndex int + } + + for i, batchChunks := range utils.ChunkSlice(chunks, relationChunkSize) { + start := i * relationChunkSize + end := start + relationChunkSize + if end > len(chunkEntities) { + end = len(chunkEntities) + } + + // Merge all entities in this batch + relationUseEntities := make([]*types.Entity, 0) + for j := start; j < end; j++ { + if j < len(chunkEntities) { + relationUseEntities = append(relationUseEntities, chunkEntities[j]...) + } + } + + if len(relationUseEntities) < MinEntitiesForRelation { + log.Debugf("Skipping batch %d: not enough entities (%d)", i+1, len(relationUseEntities)) + continue + } + + relationBatches = append(relationBatches, struct { + batchChunks []*types.Chunk + relationUseEntities []*types.Entity + batchIndex int + }{ + batchChunks: batchChunks, + relationUseEntities: relationUseEntities, + batchIndex: i, + }) + } + + // extract relationships concurrently + relG, relGctx := errgroup.WithContext(ctx) + relG.SetLimit(MaxConcurrentRelationExtractions) // use dedicated relationship extraction concurrency limit + + for _, batch := range relationBatches { + relG.Go(func() error { + log.Debugf("Processing relationship batch %d (chunks %d)", batch.batchIndex+1, len(batch.batchChunks)) + err := b.extractRelationships(relGctx, batch.batchChunks, batch.relationUseEntities) + if err != nil { + log.WithError(err).Errorf("Failed to extract relationships for batch %d", batch.batchIndex+1) + } + return nil // continue to process other batches even if the current batch fails + }) + } + + // wait for all relationship extractions to complete + if err := relG.Wait(); err != nil { + log.WithError(err).Error("Some relationship extraction tasks failed") + // but we continue to process the next steps because some relationship extractions are still useful + } + + // Calculate relationship weights + log.Info("Calculating weights for relationships") + b.calculateWeights(ctx) + + // Calculate entity degrees + log.Info("Calculating degrees for entities") + b.calculateDegrees(ctx) + + // Build Chunk graph + log.Info("Building chunk relationship graph") + b.buildChunkGraph(ctx) + + log.Infof("Graph building completed in %.2f seconds: %d entities, %d relationships", + time.Since(startTime).Seconds(), len(b.entityMap), len(b.relationshipMap)) + + // generate knowledge graph visualization diagram + mermaidDiagram := b.generateKnowledgeGraphDiagram(ctx) + log.Info("Knowledge graph visualization diagram:") + log.Info(mermaidDiagram) + + return nil +} + +// calculateWeights calculates relationship weights +// It uses Point Mutual Information (PMI) and strength values to calculate relationship weights +func (b *graphBuilder) calculateWeights(ctx context.Context) { + log := logger.GetLogger(ctx) + log.Info("Calculating relationship weights using PMI and strength") + + // Calculate total entity occurrences + totalEntityOccurrences := 0 + entityFrequency := make(map[string]int) + + for _, entity := range b.entityMap { + frequency := len(entity.ChunkIDs) + entityFrequency[entity.Title] = frequency + totalEntityOccurrences += frequency + } + + // Calculate total relationship occurrences + totalRelOccurrences := 0 + for _, rel := range b.relationshipMap { + totalRelOccurrences += len(rel.ChunkIDs) + } + + // Skip calculation if insufficient data + if totalEntityOccurrences == 0 || totalRelOccurrences == 0 { + log.Warn("Insufficient data for weight calculation") + return + } + + // Track maximum PMI and Strength values for normalization + maxPMI := 0.0 + maxStrength := MinWeightValue // Avoid division by zero + + // First calculate PMI and find maximum values + pmiValues := make(map[string]float64) + for _, rel := range b.relationshipMap { + sourceFreq := entityFrequency[rel.Source] + targetFreq := entityFrequency[rel.Target] + relFreq := len(rel.ChunkIDs) + + if sourceFreq > 0 && targetFreq > 0 && relFreq > 0 { + sourceProbability := float64(sourceFreq) / float64(totalEntityOccurrences) + targetProbability := float64(targetFreq) / float64(totalEntityOccurrences) + relProbability := float64(relFreq) / float64(totalRelOccurrences) + + // PMI calculation: log(P(x,y) / (P(x) * P(y))) + pmi := math.Max(math.Log2(relProbability/(sourceProbability*targetProbability)), 0) + pmiValues[rel.ID] = pmi + + if pmi > maxPMI { + maxPMI = pmi + } + } + + // Record maximum Strength value + if float64(rel.Strength) > maxStrength { + maxStrength = float64(rel.Strength) + } + } + + // Combine PMI and Strength to calculate final weights + for _, rel := range b.relationshipMap { + pmi := pmiValues[rel.ID] + + // Normalize PMI and Strength (0-1 range) + normalizedPMI := 0.0 + if maxPMI > 0 { + normalizedPMI = pmi / maxPMI + } + + normalizedStrength := float64(rel.Strength) / maxStrength + + // Combine PMI and Strength using configured weights + combinedWeight := normalizedPMI*PMIWeight + normalizedStrength*StrengthWeight + + // Scale weight to 1-10 range + scaledWeight := 1.0 + WeightScaleFactor*combinedWeight + + rel.Weight = scaledWeight + } + + log.Infof("Weight calculation completed for %d relationships", len(b.relationshipMap)) +} + +// calculateDegrees calculates entity degrees +// Degree represents the number of connections an entity has with other entities, a key metric in graph structures +func (b *graphBuilder) calculateDegrees(ctx context.Context) { + log := logger.GetLogger(ctx) + log.Info("Calculating entity degrees") + + // Calculate in-degree and out-degree for each entity + inDegree := make(map[string]int) + outDegree := make(map[string]int) + + for _, rel := range b.relationshipMap { + outDegree[rel.Source]++ + inDegree[rel.Target]++ + } + + // Set degree for each entity + for _, entity := range b.entityMap { + entity.Degree = inDegree[entity.Title] + outDegree[entity.Title] + } + + // Set combined degree for relationships + for _, rel := range b.relationshipMap { + sourceEntity := b.getEntityByTitle(rel.Source) + targetEntity := b.getEntityByTitle(rel.Target) + + if sourceEntity != nil && targetEntity != nil { + rel.CombinedDegree = sourceEntity.Degree + targetEntity.Degree + } + } + + log.Info("Entity degree calculation completed") +} + +// buildChunkGraph builds relationship graph between Chunks +// It creates a network of relationships between document chunks based on entity relationships +func (b *graphBuilder) buildChunkGraph(ctx context.Context) { + log := logger.GetLogger(ctx) + log.Info("Building chunk relationship graph") + + // Create document chunk relationship graph based on entity relationships + for _, rel := range b.relationshipMap { + // Ensure source and target entities exist for the relationship + sourceEntity := b.entityMapByTitle[rel.Source] + targetEntity := b.entityMapByTitle[rel.Target] + + if sourceEntity == nil || targetEntity == nil { + log.Warnf("Missing entity for relationship %s -> %s", rel.Source, rel.Target) + continue + } + + // Build Chunk graph - connect all related document chunks + for _, sourceChunkID := range sourceEntity.ChunkIDs { + if _, exists := b.chunkGraph[sourceChunkID]; !exists { + b.chunkGraph[sourceChunkID] = make(map[string]*ChunkRelation) + } + + for _, targetChunkID := range targetEntity.ChunkIDs { + if _, exists := b.chunkGraph[targetChunkID]; !exists { + b.chunkGraph[targetChunkID] = make(map[string]*ChunkRelation) + } + + relation := &ChunkRelation{ + Weight: rel.Weight, + Degree: rel.CombinedDegree, + } + + b.chunkGraph[sourceChunkID][targetChunkID] = relation + b.chunkGraph[targetChunkID][sourceChunkID] = relation + } + } + } + + log.Infof("Chunk graph built with %d nodes", len(b.chunkGraph)) +} + +// GetAllEntities returns all entities +func (b *graphBuilder) GetAllEntities() []*types.Entity { + b.mutex.RLock() + defer b.mutex.RUnlock() + + entities := make([]*types.Entity, 0, len(b.entityMap)) + for _, entity := range b.entityMap { + entities = append(entities, entity) + } + return entities +} + +// GetAllRelationships returns all relationships +func (b *graphBuilder) GetAllRelationships() []*types.Relationship { + b.mutex.RLock() + defer b.mutex.RUnlock() + + relationships := make([]*types.Relationship, 0, len(b.relationshipMap)) + for _, relationship := range b.relationshipMap { + relationships = append(relationships, relationship) + } + return relationships +} + +// GetRelationChunks retrieves document chunks directly related to the given chunkID +// It returns a list of related document chunk IDs sorted by weight and degree +func (b *graphBuilder) GetRelationChunks(chunkID string, topK int) []string { + b.mutex.RLock() + defer b.mutex.RUnlock() + + log := logger.GetLogger(context.Background()) + log.Debugf("Getting related chunks for %s (topK=%d)", chunkID, topK) + + // Create weighted chunk structure for sorting + type weightedChunk struct { + id string + weight float64 + degree int + } + + // Collect related chunks with their weights and degrees + weightedChunks := make([]weightedChunk, 0) + for relationChunkID, relation := range b.chunkGraph[chunkID] { + weightedChunks = append(weightedChunks, weightedChunk{ + id: relationChunkID, + weight: relation.Weight, + degree: relation.Degree, + }) + } + + // Sort by weight and degree in descending order + slices.SortFunc(weightedChunks, func(a, b weightedChunk) int { + // Sort by weight first + if a.weight > b.weight { + return -1 // Descending order + } else if a.weight < b.weight { + return 1 + } + + // If weights are equal, sort by degree + if a.degree > b.degree { + return -1 // Descending order + } else if a.degree < b.degree { + return 1 + } + + return 0 + }) + + // Take top K results + resultCount := len(weightedChunks) + if topK > 0 && topK < resultCount { + resultCount = topK + } + + // Extract chunk IDs + chunks := make([]string, 0, resultCount) + for i := 0; i < resultCount; i++ { + chunks = append(chunks, weightedChunks[i].id) + } + + log.Debugf("Found %d related chunks for %s (limited to %d)", + len(weightedChunks), chunkID, resultCount) + return chunks +} + +// GetIndirectRelationChunks retrieves document chunks indirectly related to the given chunkID +// It returns document chunk IDs found through second-degree connections +func (b *graphBuilder) GetIndirectRelationChunks(chunkID string, topK int) []string { + b.mutex.RLock() + defer b.mutex.RUnlock() + + log := logger.GetLogger(context.Background()) + log.Debugf("Getting indirectly related chunks for %s (topK=%d)", chunkID, topK) + + // Create weighted chunk structure for sorting + type weightedChunk struct { + id string + weight float64 + degree int + } + + // Get directly related chunks (first-degree connections) + directChunks := make(map[string]struct{}) + directChunks[chunkID] = struct{}{} // Add original chunkID + for directChunkID := range b.chunkGraph[chunkID] { + directChunks[directChunkID] = struct{}{} + } + log.Debugf("Found %d directly related chunks to exclude", len(directChunks)) + + // Use map to deduplicate and store second-degree connections + indirectChunkMap := make(map[string]*ChunkRelation) + + // Get first-degree connections + for directChunkID, directRelation := range b.chunkGraph[chunkID] { + // Get second-degree connections + for indirectChunkID, indirectRelation := range b.chunkGraph[directChunkID] { + // Skip self and all direct connections + if _, isDirect := directChunks[indirectChunkID]; isDirect { + continue + } + + // Weight decay: second-degree relationship weight is the product of two direct relationship weights + // multiplied by decay coefficient + combinedWeight := directRelation.Weight * indirectRelation.Weight * IndirectRelationWeightDecay + // Degree calculation: take the maximum degree from the two path segments + combinedDegree := max(directRelation.Degree, indirectRelation.Degree) + + // If already exists, take the higher weight + if existingRel, exists := indirectChunkMap[indirectChunkID]; !exists || + combinedWeight > existingRel.Weight { + indirectChunkMap[indirectChunkID] = &ChunkRelation{ + Weight: combinedWeight, + Degree: combinedDegree, + } + } + } + } + + // Convert to sortable slice + weightedChunks := make([]weightedChunk, 0, len(indirectChunkMap)) + for id, relation := range indirectChunkMap { + weightedChunks = append(weightedChunks, weightedChunk{ + id: id, + weight: relation.Weight, + degree: relation.Degree, + }) + } + + // Sort by weight and degree in descending order + slices.SortFunc(weightedChunks, func(a, b weightedChunk) int { + // Sort by weight first + if a.weight > b.weight { + return -1 // Descending order + } else if a.weight < b.weight { + return 1 + } + + // If weights are equal, sort by degree + if a.degree > b.degree { + return -1 // Descending order + } else if a.degree < b.degree { + return 1 + } + + return 0 + }) + + // Take top K results + resultCount := len(weightedChunks) + if topK > 0 && topK < resultCount { + resultCount = topK + } + + // Extract chunk IDs + chunks := make([]string, 0, resultCount) + for i := 0; i < resultCount; i++ { + chunks = append(chunks, weightedChunks[i].id) + } + + log.Debugf("Found %d indirect related chunks for %s (limited to %d)", + len(weightedChunks), chunkID, resultCount) + return chunks +} + +// getEntityByTitle retrieves an entity by its title +func (b *graphBuilder) getEntityByTitle(title string) *types.Entity { + return b.entityMapByTitle[title] +} + +// dfs depth-first search to find connected components +func dfs(entityTitle string, + adjacencyList map[string]map[string]*types.Relationship, + visited map[string]bool, component *[]string) { + visited[entityTitle] = true + *component = append(*component, entityTitle) + + // traverse all relationships of the current entity + for targetEntity := range adjacencyList[entityTitle] { + if !visited[targetEntity] { + dfs(targetEntity, adjacencyList, visited, component) + } + } + + // check reverse relationships (check if other entities point to the current entity) + for source, targets := range adjacencyList { + for target := range targets { + if target == entityTitle && !visited[source] { + dfs(source, adjacencyList, visited, component) + } + } + } +} + +// generateKnowledgeGraphDiagram generate Mermaid diagram for knowledge graph +func (b *graphBuilder) generateKnowledgeGraphDiagram(ctx context.Context) string { + log := logger.GetLogger(ctx) + log.Info("Generating knowledge graph visualization diagram...") + + var sb strings.Builder + + // Mermaid diagram header + sb.WriteString("```mermaid\ngraph TD\n") + sb.WriteString(" %% entity style definition\n") + sb.WriteString(" classDef entity fill:#f9f,stroke:#333,stroke-width:1px;\n") + sb.WriteString(" classDef highFreq fill:#bbf,stroke:#333,stroke-width:2px;\n\n") + + // get all entities and sort by frequency + entities := b.GetAllEntities() + slices.SortFunc(entities, func(a, b *types.Entity) int { + if a.Frequency > b.Frequency { + return -1 + } else if a.Frequency < b.Frequency { + return 1 + } + return 0 + }) + + // get relationships and sort by weight + relationships := b.GetAllRelationships() + slices.SortFunc(relationships, func(a, b *types.Relationship) int { + if a.Weight > b.Weight { + return -1 + } else if a.Weight < b.Weight { + return 1 + } + return 0 + }) + + // create entity ID mapping + entityMap := make(map[string]string) // store entity title to node ID mapping + for i, entity := range entities { + nodeID := fmt.Sprintf("E%d", i) + entityMap[entity.Title] = nodeID + } + + // create adjacency list to represent graph structure + adjacencyList := make(map[string]map[string]*types.Relationship) + for _, entity := range entities { + adjacencyList[entity.Title] = make(map[string]*types.Relationship) + } + + // fill adjacency list + for _, rel := range relationships { + if _, sourceExists := entityMap[rel.Source]; sourceExists { + if _, targetExists := entityMap[rel.Target]; targetExists { + adjacencyList[rel.Source][rel.Target] = rel + } + } + } + + // use DFS to find connected components (subgraphs) + visited := make(map[string]bool) + subgraphs := make([][]string, 0) // store entity titles in each subgraph + + for _, entity := range entities { + if !visited[entity.Title] { + component := make([]string, 0) + dfs(entity.Title, adjacencyList, visited, &component) + if len(component) > 0 { + subgraphs = append(subgraphs, component) + } + } + } + + // generate Mermaid subgraphs + subgraphCount := 0 + for _, component := range subgraphs { + // check if this component has relationships + hasRelations := false + nodeCount := len(component) + + // if there is only 1 node, check if it has relationships + if nodeCount == 1 { + entityTitle := component[0] + // check if this entity appears as source or target in any relationship + for _, rel := range relationships { + if rel.Source == entityTitle || rel.Target == entityTitle { + hasRelations = true + break + } + } + + // if there is only 1 node and no relationships, skip this subgraph + if !hasRelations { + continue + } + } else if nodeCount > 1 { + // a subgraph with more than 1 node must have relationships + hasRelations = true + } + + // only draw if there are multiple entities or at least one relationship in the subgraph + if hasRelations { + subgraphCount++ + sb.WriteString(fmt.Sprintf("\n subgraph 子图%d\n", subgraphCount)) + + // add all entities in this subgraph + entitiesInComponent := make(map[string]bool) + for _, entityTitle := range component { + nodeID := entityMap[entityTitle] + entitiesInComponent[entityTitle] = true + + // add node definition for each entity + entity := b.entityMapByTitle[entityTitle] + if entity != nil { + sb.WriteString(fmt.Sprintf(" %s[\"%s\"]\n", nodeID, entityTitle)) + } + } + + // add relationships in this subgraph + for _, rel := range relationships { + if entitiesInComponent[rel.Source] && entitiesInComponent[rel.Target] { + sourceID := entityMap[rel.Source] + targetID := entityMap[rel.Target] + + linkStyle := "-->" + // adjust link style based on relationship strength + if rel.Strength > 7 { + linkStyle = "==>" + } + + sb.WriteString(fmt.Sprintf(" %s %s|%s| %s\n", + sourceID, linkStyle, rel.Description, targetID)) + } + } + + // subgraph ends + sb.WriteString(" end\n") + + // apply style class + for _, entityTitle := range component { + nodeID := entityMap[entityTitle] + entity := b.entityMapByTitle[entityTitle] + if entity != nil { + if entity.Frequency > 5 { + sb.WriteString(fmt.Sprintf(" class %s highFreq;\n", nodeID)) + } else { + sb.WriteString(fmt.Sprintf(" class %s entity;\n", nodeID)) + } + } + } + } + } + + // close Mermaid diagram + sb.WriteString("```\n") + + log.Infof("Knowledge graph visualization diagram generated with %d subgraphs", subgraphCount) + return sb.String() +} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go new file mode 100644 index 0000000..24a6db9 --- /dev/null +++ b/internal/application/service/knowledge.go @@ -0,0 +1,1810 @@ +package service + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "os" + "regexp" + "slices" + "sort" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/application/service/retriever" + "github.com/Tencent/WeKnora/internal/config" + werrors "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/Tencent/WeKnora/internal/tracing" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" + "github.com/Tencent/WeKnora/services/docreader/src/client" + "github.com/Tencent/WeKnora/services/docreader/src/proto" + "github.com/google/uuid" + "github.com/hibiken/asynq" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sync/errgroup" +) + +// Error definitions for knowledge service operations +var ( + // ErrInvalidFileType is returned when an unsupported file type is provided + ErrInvalidFileType = errors.New("unsupported file type") + // ErrInvalidURL is returned when an invalid URL is provided + ErrInvalidURL = errors.New("invalid URL") + // ErrChunkNotFound is returned when a requested chunk cannot be found + ErrChunkNotFound = errors.New("chunk not found") + // ErrDuplicateFile is returned when trying to add a file that already exists + ErrDuplicateFile = errors.New("file already exists") + // ErrDuplicateURL is returned when trying to add a URL that already exists + ErrDuplicateURL = errors.New("URL already exists") + // ErrImageNotParse is returned when trying to update image information without enabling multimodel + ErrImageNotParse = errors.New("image not parse without enable multimodel") +) + +// knowledgeService implements the knowledge service interface +// service 实现知识服务接口 +type knowledgeService struct { + config *config.Config + repo interfaces.KnowledgeRepository + kbService interfaces.KnowledgeBaseService + tenantRepo interfaces.TenantRepository + docReaderClient *client.Client + chunkService interfaces.ChunkService + chunkRepo interfaces.ChunkRepository + fileSvc interfaces.FileService + modelService interfaces.ModelService + task *asynq.Client + graphEngine interfaces.RetrieveGraphRepository +} + +// NewKnowledgeService creates a new knowledge service instance +func NewKnowledgeService( + config *config.Config, + repo interfaces.KnowledgeRepository, + docReaderClient *client.Client, + kbService interfaces.KnowledgeBaseService, + tenantRepo interfaces.TenantRepository, + chunkService interfaces.ChunkService, + chunkRepo interfaces.ChunkRepository, + fileSvc interfaces.FileService, + modelService interfaces.ModelService, + task *asynq.Client, + graphEngine interfaces.RetrieveGraphRepository, +) (interfaces.KnowledgeService, error) { + return &knowledgeService{ + config: config, + repo: repo, + kbService: kbService, + tenantRepo: tenantRepo, + docReaderClient: docReaderClient, + chunkService: chunkService, + chunkRepo: chunkRepo, + fileSvc: fileSvc, + modelService: modelService, + task: task, + graphEngine: graphEngine, + }, nil +} + +// CreateKnowledgeFromFile creates a knowledge entry from an uploaded file +func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, + kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, +) (*types.Knowledge, error) { + logger.Info(ctx, "Start creating knowledge from file") + logger.Infof(ctx, "Knowledge base ID: %s, file: %s", kbID, file.Filename) + if metadata != nil { + logger.Infof(ctx, "Received metadata: %v", metadata) + } + + // Get knowledge base configuration + logger.Info(ctx, "Getting knowledge base configuration") + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge base: %v", err) + return nil, err + } + + // 检查多模态配置完整性 - 只在图片文件时校验 + // 检查是否为图片文件 + if !IsImageType(getFileType(file.Filename)) { + logger.Info(ctx, "Non-image file with multimodal enabled, skipping COS/VLM validation") + } else { + // 检查COS配置 + switch kb.StorageConfig.Provider { + case "cos": + if kb.StorageConfig.SecretID == "" || kb.StorageConfig.SecretKey == "" || + kb.StorageConfig.Region == "" || kb.StorageConfig.BucketName == "" || + kb.StorageConfig.AppID == "" { + logger.Error(ctx, "COS configuration incomplete for image multimodal processing") + return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往系统设置页面进行补全") + } + case "minio": + if kb.StorageConfig.BucketName == "" { + logger.Error(ctx, "MinIO configuration incomplete for image multimodal processing") + return nil, werrors.NewBadRequestError("上传图片文件需要完整的对象存储配置信息, 请前往系统设置页面进行补全") + } + } + + // 检查VLM配置 + if kb.VLMConfig.ModelName == "" || kb.VLMConfig.BaseURL == "" { + logger.Error(ctx, "VLM configuration incomplete for image multimodal processing") + return nil, werrors.NewBadRequestError("上传图片文件需要完整的VLM配置信息, 请前往系统设置页面进行补全") + } + + logger.Info(ctx, "Image multimodal configuration validation passed") + } + + // Validate file type + logger.Infof(ctx, "Checking file type: %s", file.Filename) + if !isValidFileType(file.Filename) { + logger.Error(ctx, "Invalid file type") + return nil, ErrInvalidFileType + } + + // Calculate file hash for deduplication + logger.Info(ctx, "Calculating file hash") + hash, err := calculateFileHash(file) + if err != nil { + logger.Errorf(ctx, "Failed to calculate file hash: %v", err) + return nil, err + } + + // Check if file already exists + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if file exists, tenant ID: %d", tenantID) + exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ + Type: "file", + FileName: file.Filename, + FileSize: file.Size, + FileHash: hash, + }) + if err != nil { + logger.Errorf(ctx, "Failed to check knowledge existence: %v", err) + return nil, err + } + if exists { + logger.Infof(ctx, "File already exists: %s", file.Filename) + // Update creation time for existing knowledge + if err := s.repo.UpdateKnowledgeColumn(ctx, existingKnowledge.ID, "created_at", time.Now()); err != nil { + logger.Errorf(ctx, "Failed to update existing knowledge: %v", err) + return nil, err + } + return existingKnowledge, types.NewDuplicateFileError(existingKnowledge) + } + + // Check storage quota + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota { + logger.Error(ctx, "Storage quota exceeded") + return nil, types.NewStorageQuotaExceededError() + } + + // Convert metadata to JSON format if provided + var metadataJSON types.JSON + if metadata != nil { + metadataBytes, err := json.Marshal(metadata) + if err != nil { + logger.Errorf(ctx, "Failed to marshal metadata: %v", err) + return nil, err + } + metadataJSON = types.JSON(metadataBytes) + } + + // 验证文件名安全性 + safeFilename, isValid := secutils.ValidateInput(file.Filename) + if !isValid { + logger.Errorf(ctx, "Invalid filename: %s", file.Filename) + return nil, werrors.NewValidationError("文件名包含非法字符") + } + + // Create knowledge record + logger.Info(ctx, "Creating knowledge record") + knowledge := &types.Knowledge{ + TenantID: tenantID, + KnowledgeBaseID: kbID, + Type: "file", + Title: safeFilename, + FileName: safeFilename, + FileType: getFileType(safeFilename), + FileSize: file.Size, + FileHash: hash, + ParseStatus: "pending", + EnableStatus: "disabled", + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + EmbeddingModelID: kb.EmbeddingModelID, + Metadata: metadataJSON, + } + // Save knowledge record to database + logger.Info(ctx, "Saving knowledge record to database") + if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { + logger.Errorf(ctx, "Failed to create knowledge record, ID: %s, error: %v", knowledge.ID, err) + return nil, err + } + // Save the file to storage + logger.Infof(ctx, "Saving file, knowledge ID: %s", knowledge.ID) + filePath, err := s.fileSvc.SaveFile(ctx, file, knowledge.TenantID, knowledge.ID) + if err != nil { + logger.Errorf(ctx, "Failed to save file, knowledge ID: %s, error: %v", knowledge.ID, err) + return nil, err + } + knowledge.FilePath = filePath + + // Update knowledge record with file path + logger.Info(ctx, "Updating knowledge record with file path") + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + logger.Errorf(ctx, "Failed to update knowledge with file path, ID: %s, error: %v", knowledge.ID, err) + return nil, err + } + + // Process document asynchronously + logger.Info(ctx, "Starting asynchronous document processing") + newCtx := logger.CloneContext(ctx) + if enableMultimodel == nil { + enableMultimodel = &kb.ChunkingConfig.EnableMultimodal + } + go s.processDocument(newCtx, kb, knowledge, file, *enableMultimodel) + + logger.Infof(ctx, "Knowledge from file created successfully, ID: %s", knowledge.ID) + return knowledge, nil +} + +// CreateKnowledgeFromURL creates a knowledge entry from a URL source +func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, + kbID string, url string, enableMultimodel *bool, +) (*types.Knowledge, error) { + logger.Info(ctx, "Start creating knowledge from URL") + logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url) + + // Get knowledge base configuration + logger.Info(ctx, "Getting knowledge base configuration") + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge base: %v", err) + return nil, err + } + + // Validate URL format and security + logger.Info(ctx, "Validating URL") + if !isValidURL(url) || !secutils.IsValidURL(url) { + logger.Error(ctx, "Invalid or unsafe URL format") + return nil, ErrInvalidURL + } + + // Check if URL already exists in the knowledge base + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if URL exists, tenant ID: %d", tenantID) + fileHash := calculateStr(url) + exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{ + Type: "url", + URL: url, + FileHash: fileHash, + }) + if err != nil { + logger.Errorf(ctx, "Failed to check knowledge existence: %v", err) + return nil, err + } + if exists { + logger.Infof(ctx, "URL already exists: %s", url) + // Update creation time for existing knowledge + existingKnowledge.CreatedAt = time.Now() + existingKnowledge.UpdatedAt = time.Now() + if err := s.repo.UpdateKnowledge(ctx, existingKnowledge); err != nil { + logger.Errorf(ctx, "Failed to update existing knowledge: %v", err) + return nil, err + } + return existingKnowledge, types.NewDuplicateURLError(existingKnowledge) + } + + // Check storage quota + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota { + logger.Error(ctx, "Storage quota exceeded") + return nil, types.NewStorageQuotaExceededError() + } + + // Create knowledge record + logger.Info(ctx, "Creating knowledge record") + knowledge := &types.Knowledge{ + ID: uuid.New().String(), + TenantID: tenantID, + KnowledgeBaseID: kbID, + Type: "url", + Source: url, + FileHash: fileHash, + ParseStatus: "pending", + EnableStatus: "disabled", + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + EmbeddingModelID: kb.EmbeddingModelID, + } + + // Save knowledge record + logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID) + if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { + logger.Errorf(ctx, "Failed to create knowledge record: %v", err) + return nil, err + } + + // Process URL asynchronously + logger.Info(ctx, "Starting asynchronous URL processing") + if enableMultimodel == nil { + enableMultimodel = &kb.ChunkingConfig.EnableMultimodal + } + newCtx := logger.CloneContext(ctx) + go s.processDocumentFromURL(newCtx, kb, knowledge, url, *enableMultimodel) + + logger.Infof(ctx, "Knowledge from URL created successfully, ID: %s", knowledge.ID) + return knowledge, nil +} + +// CreateKnowledgeFromPassage creates a knowledge entry from text passages +func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context, + kbID string, passage []string, +) (*types.Knowledge, error) { + logger.Info(ctx, "Start creating knowledge from passage") + logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage)) + + // 验证段落内容安全性 + safePassages := make([]string, 0, len(passage)) + for i, p := range passage { + safePassage, isValid := secutils.ValidateInput(p) + if !isValid { + logger.Errorf(ctx, "Invalid passage content at index %d", i) + return nil, werrors.NewValidationError(fmt.Sprintf("段落 %d 包含非法内容", i+1)) + } + safePassages = append(safePassages, safePassage) + } + + // Get knowledge base configuration + logger.Info(ctx, "Getting knowledge base configuration") + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge base: %v", err) + return nil, err + } + + // Create knowledge record + logger.Info(ctx, "Creating knowledge record") + knowledge := &types.Knowledge{ + ID: uuid.New().String(), + TenantID: ctx.Value(types.TenantIDContextKey).(uint), + KnowledgeBaseID: kbID, + Type: "passage", + ParseStatus: "pending", + EnableStatus: "disabled", + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + EmbeddingModelID: kb.EmbeddingModelID, + } + + // Save knowledge record + logger.Infof(ctx, "Saving knowledge record to database, ID: %s", knowledge.ID) + if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil { + logger.Errorf(ctx, "Failed to create knowledge record: %v", err) + return nil, err + } + + // Process passages asynchronously + logger.Info(ctx, "Starting asynchronous passage processing") + go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages) + + logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID) + return knowledge, nil +} + +// GetKnowledgeByID retrieves a knowledge entry by its ID +func (s *knowledgeService) GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error) { + logger.Info(ctx, "Start getting knowledge by ID") + logger.Infof(ctx, "Knowledge ID: %s", id) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_id": id, + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Infof(ctx, "Knowledge retrieved successfully, ID: %s, type: %s", knowledge.ID, knowledge.Type) + return knowledge, nil +} + +// ListKnowledgeByKnowledgeBaseID returns all knowledge entries in a knowledge base +func (s *knowledgeService) ListKnowledgeByKnowledgeBaseID(ctx context.Context, + kbID string, +) ([]*types.Knowledge, error) { + return s.repo.ListKnowledgeByKnowledgeBaseID(ctx, ctx.Value(types.TenantIDContextKey).(uint), kbID) +} + +// ListPagedKnowledgeByKnowledgeBaseID returns paginated knowledge entries in a knowledge base +func (s *knowledgeService) ListPagedKnowledgeByKnowledgeBaseID(ctx context.Context, + kbID string, page *types.Pagination, +) (*types.PageResult, error) { + knowledges, total, err := s.repo.ListPagedKnowledgeByKnowledgeBaseID(ctx, + ctx.Value(types.TenantIDContextKey).(uint), kbID, page) + if err != nil { + return nil, err + } + + return types.NewPageResult(total, page, knowledges), nil +} + +// DeleteKnowledge deletes a knowledge entry and all related resources +func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error { + // Get the knowledge entry + knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) + if err != nil { + return err + } + wg := errgroup.Group{} + // Delete knowledge embeddings from vector store + wg.Go(func() error { + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(tenantInfo.RetrieverEngines.Engines) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") + return err + } + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, knowledge.EmbeddingModelID) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") + return err + } + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") + return err + } + return nil + }) + + // Delete all chunks associated with this knowledge + wg.Go(func() error { + if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") + return err + } + return nil + }) + + // Delete the physical file if it exists + wg.Go(func() error { + if knowledge.FilePath != "" { + if err := s.fileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed") + } + } + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + tenantInfo.StorageUsed -= knowledge.StorageSize + if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, -knowledge.StorageSize); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") + } + return nil + }) + + // Delete the knowledge graph + wg.Go(func() error { + namespace := types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID} + if err := s.graphEngine.DelGraph(ctx, []types.NameSpace{namespace}); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed") + return err + } + return nil + }) + + if err = wg.Wait(); err != nil { + return err + } + // Delete the knowledge entry itself from the database + return s.repo.DeleteKnowledge(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) +} + +// DeleteKnowledge deletes a knowledge entry and all related resources +func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string) error { + if len(ids) == 0 { + return nil + } + // 1. Get the knowledge entry + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + knowledgeList, err := s.repo.GetKnowledgeBatch(ctx, tenantInfo.ID, ids) + if err != nil { + return err + } + + wg := errgroup.Group{} + // 2. Delete knowledge embeddings from vector store + wg.Go(func() error { + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(tenantInfo.RetrieverEngines.Engines) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") + return err + } + group := map[string][]string{} + for _, knowledge := range knowledgeList { + group[knowledge.EmbeddingModelID] = append(group[knowledge.EmbeddingModelID], knowledge.ID) + } + for embeddingModelID, knowledgeList := range group { + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge get embedding model failed") + return err + } + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeList, embeddingModel.GetDimensions()); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") + return err + } + } + return nil + }) + + // 3. Delete all chunks associated with this knowledge + wg.Go(func() error { + if err := s.chunkService.DeleteByKnowledgeList(ctx, ids); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete chunks failed") + return err + } + return nil + }) + + // 4. Delete the physical file if it exists + wg.Go(func() error { + storageAdjust := int64(0) + for _, knowledge := range knowledgeList { + if knowledge.FilePath != "" { + if err := s.fileSvc.DeleteFile(ctx, knowledge.FilePath); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete file failed") + } + } + storageAdjust -= knowledge.StorageSize + } + tenantInfo.StorageUsed += storageAdjust + if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, storageAdjust); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge update tenant storage used failed") + } + return nil + }) + + // Delete the knowledge graph + wg.Go(func() error { + namespaces := []types.NameSpace{} + for _, knowledge := range knowledgeList { + namespaces = append(namespaces, types.NameSpace{KnowledgeBase: knowledge.KnowledgeBaseID, Knowledge: knowledge.ID}) + } + if err := s.graphEngine.DelGraph(ctx, namespaces); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge graph failed") + return err + } + return nil + }) + + if err = wg.Wait(); err != nil { + return err + } + // 5. Delete the knowledge entry itself from the database + return s.repo.DeleteKnowledgeList(ctx, tenantInfo.ID, ids) +} + +func (s *knowledgeService) cloneKnowledge(ctx context.Context, src *types.Knowledge, targetKB *types.KnowledgeBase) (err error) { + if src.ParseStatus != "completed" { + logger.GetLogger(ctx).WithField("knowledge_id", src.ID).Errorf("MoveKnowledge parse status is not completed") + return nil + } + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + dst := &types.Knowledge{ + ID: uuid.New().String(), + TenantID: targetKB.TenantID, + KnowledgeBaseID: targetKB.ID, + Type: src.Type, + Title: src.Title, + Description: src.Description, + Source: src.Source, + ParseStatus: "processing", + EnableStatus: "disabled", + EmbeddingModelID: targetKB.EmbeddingModelID, + FileName: src.FileName, + FileType: src.FileType, + FileSize: src.FileSize, + FileHash: src.FileHash, + FilePath: src.FilePath, + StorageSize: src.StorageSize, + Metadata: src.Metadata, + } + defer func() { + if err != nil { + dst.ParseStatus = "failed" + dst.ErrorMessage = err.Error() + _ = s.repo.UpdateKnowledge(ctx, dst) + logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge failed to move knowledge") + } else { + dst.ParseStatus = "completed" + dst.EnableStatus = "enabled" + _ = s.repo.UpdateKnowledge(ctx, dst) + logger.GetLogger(ctx).WithField("knowledge_id", dst.ID).Infof("MoveKnowledge move knowledge successfully") + } + }() + + if err = s.repo.CreateKnowledge(ctx, dst); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge create knowledge failed") + return + } + tenantInfo.StorageUsed += dst.StorageSize + if err = s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, dst.StorageSize); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("MoveKnowledge update tenant storage used failed") + return + } + if err = s.CloneChunk(ctx, src, dst); err != nil { + logger.GetLogger(ctx).WithField("knowledge_id", dst.ID). + WithField("error", err).Errorf("MoveKnowledge move chunks failed") + return + } + return +} + +// processDocument handles asynchronous processing of document files +func (s *knowledgeService) processDocument(ctx context.Context, + kb *types.KnowledgeBase, knowledge *types.Knowledge, file *multipart.FileHeader, enableMultimodel bool, +) { + logger.GetLogger(ctx).Infof("processDocument enableMultimodel: %v", enableMultimodel) + + ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processDocument") + defer span.End() + span.SetAttributes( + attribute.String("request_id", ctx.Value(types.RequestIDContextKey).(string)), + attribute.String("knowledge_base_id", kb.ID), + attribute.Int("tenant_id", int(kb.TenantID)), + attribute.String("knowledge_id", knowledge.ID), + attribute.String("file_name", knowledge.FileName), + attribute.String("file_type", knowledge.FileType), + attribute.String("file_path", knowledge.FilePath), + attribute.Int64("file_size", knowledge.FileSize), + attribute.String("embedding_model", knowledge.EmbeddingModelID), + attribute.Bool("enable_multimodal", enableMultimodel), + ) + logger.GetLogger(ctx).Infof("processDocument trace id: %s", span.SpanContext().TraceID().String()) + + if !enableMultimodel && IsImageType(knowledge.FileType) { + logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). + WithField("error", ErrImageNotParse).Errorf("processDocument image without enable multimodel") + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = ErrImageNotParse.Error() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(ErrImageNotParse) + return + } + + // Update status to processing + knowledge.ParseStatus = "processing" + knowledge.UpdatedAt = time.Now() + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + span.RecordError(err) + return + } + + // Read and chunk the document + f, err := file.Open() + if err != nil { + logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). + WithField("error", err).Errorf("processDocument open file failed") + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(err) + return + } + defer f.Close() + + span.AddEvent("start read file") + contentBytes, err := io.ReadAll(f) + if err != nil { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(err) + return + } + + // Split file into chunks using document reader service + span.AddEvent("start split file") + resp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{ + FileContent: contentBytes, + FileName: knowledge.FileName, + FileType: knowledge.FileType, + ReadConfig: &proto.ReadConfig{ + ChunkSize: int32(kb.ChunkingConfig.ChunkSize), + ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), + Separators: kb.ChunkingConfig.Separators, + EnableMultimodal: enableMultimodel, + StorageConfig: &proto.StorageConfig{ + Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), + Region: kb.StorageConfig.Region, + BucketName: kb.StorageConfig.BucketName, + AccessKeyId: kb.StorageConfig.SecretID, + SecretAccessKey: kb.StorageConfig.SecretKey, + AppId: kb.StorageConfig.AppID, + PathPrefix: kb.StorageConfig.PathPrefix, + }, + VlmConfig: &proto.VLMConfig{ + ModelName: kb.VLMConfig.ModelName, + BaseUrl: kb.VLMConfig.BaseURL, + ApiKey: kb.VLMConfig.APIKey, + InterfaceType: kb.VLMConfig.InterfaceType, + }, + }, + RequestId: ctx.Value(types.RequestIDContextKey).(string), + }) + if err != nil { + logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). + WithField("error", err).Errorf("processDocument read file failed") + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(err) + return + } + + // Process and store chunks + span.AddEvent("start process chunks") + s.processChunks(ctx, kb, knowledge, resp.Chunks) +} + +// processDocumentFromURL handles asynchronous processing of URL content +func (s *knowledgeService) processDocumentFromURL(ctx context.Context, + kb *types.KnowledgeBase, knowledge *types.Knowledge, url string, enableMultimodel bool, +) { + // Update status to processing + knowledge.ParseStatus = "processing" + knowledge.UpdatedAt = time.Now() + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + return + } + logger.GetLogger(ctx).Infof("processDocumentFromURL enableMultimodel: %v", enableMultimodel) + + // Fetch and chunk content from URL + resp, err := s.docReaderClient.ReadFromURL(ctx, &proto.ReadFromURLRequest{ + Url: url, + Title: knowledge.Title, + ReadConfig: &proto.ReadConfig{ + ChunkSize: int32(kb.ChunkingConfig.ChunkSize), + ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap), + Separators: kb.ChunkingConfig.Separators, + EnableMultimodal: enableMultimodel, + StorageConfig: &proto.StorageConfig{ + Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]), + Region: kb.StorageConfig.Region, + BucketName: kb.StorageConfig.BucketName, + AccessKeyId: kb.StorageConfig.SecretID, + SecretAccessKey: kb.StorageConfig.SecretKey, + AppId: kb.StorageConfig.AppID, + PathPrefix: kb.StorageConfig.PathPrefix, + }, + VlmConfig: &proto.VLMConfig{ + ModelName: kb.VLMConfig.ModelName, + BaseUrl: kb.VLMConfig.BaseURL, + ApiKey: kb.VLMConfig.APIKey, + InterfaceType: kb.VLMConfig.InterfaceType, + }, + }, + RequestId: ctx.Value(types.RequestIDContextKey).(string), + }) + if err != nil { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + return + } + + // Update knowledge title if extracted from HTML + if resp.GetTitle() != "" && knowledge.Title == "" { + knowledge.Title = resp.GetTitle() + logger.GetLogger(ctx).Infof("Extracted title from URL: %s", knowledge.Title) + } + + // Process and store chunks + s.processChunks(ctx, kb, knowledge, resp.Chunks) +} + +// processDocumentFromPassage handles asynchronous processing of text passages +func (s *knowledgeService) processDocumentFromPassage(ctx context.Context, + kb *types.KnowledgeBase, knowledge *types.Knowledge, passage []string, +) { + // Update status to processing + knowledge.ParseStatus = "processing" + knowledge.UpdatedAt = time.Now() + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + return + } + + // Convert passages to chunks + chunks := make([]*proto.Chunk, 0, len(passage)) + start, end := 0, 0 + for i, p := range passage { + if p == "" { + continue + } + end += len([]rune(p)) + chunk := &proto.Chunk{ + Content: p, + Seq: int32(i), + Start: int32(start), + End: int32(end), + } + start = end + chunks = append(chunks, chunk) + } + // Process and store chunks + s.processChunks(ctx, kb, knowledge, chunks) +} + +// processChunks processes chunks and creates embeddings for knowledge content +func (s *knowledgeService) processChunks(ctx context.Context, + kb *types.KnowledgeBase, knowledge *types.Knowledge, chunks []*proto.Chunk, +) { + ctx, span := tracing.ContextWithSpan(ctx, "knowledgeService.processChunks") + defer span.End() + span.SetAttributes( + attribute.Int("tenant_id", int(knowledge.TenantID)), + attribute.String("knowledge_base_id", knowledge.KnowledgeBaseID), + attribute.String("knowledge_id", knowledge.ID), + attribute.String("embedding_model_id", kb.EmbeddingModelID), + attribute.Int("chunk_count", len(chunks)), + ) + + // Get embedding model for vectorization + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get embedding model failed") + span.RecordError(err) + return + } + + // Generate document summary - 只使用文本类型的 Chunk + chatModel, err := s.modelService.GetChatModel(ctx, kb.SummaryModelID) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks get summary model failed") + span.RecordError(err) + return + } + + enableGraphRAG := os.Getenv("ENABLE_GRAPH_RAG") == "true" + + // Create chunk objects from proto chunks + maxSeq := 0 + + // 统计图片相关的子Chunk数量，用于扩展insertChunks的容量 + imageChunkCount := 0 + for _, chunkData := range chunks { + if len(chunkData.Images) > 0 { + // 为每个图片的OCR和Caption分别创建一个Chunk + imageChunkCount += len(chunkData.Images) * 2 + } + if int(chunkData.Seq) > maxSeq { + maxSeq = int(chunkData.Seq) + } + } + + // 重新分配容量，考虑图片相关的Chunk + insertChunks := make([]*types.Chunk, 0, len(chunks)+imageChunkCount) + + for _, chunkData := range chunks { + if strings.TrimSpace(chunkData.Content) == "" { + continue + } + + // 创建主文本Chunk + textChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: knowledge.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + Content: chunkData.Content, + ChunkIndex: int(chunkData.Seq), + IsEnabled: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StartAt: int(chunkData.Start), + EndAt: int(chunkData.End), + ChunkType: types.ChunkTypeText, + } + var chunkImages []types.ImageInfo + insertChunks = append(insertChunks, textChunk) + + // 处理图片信息 + if len(chunkData.Images) > 0 { + logger.GetLogger(ctx).Infof("Processing %d images in chunk #%d", len(chunkData.Images), chunkData.Seq) + + for i, img := range chunkData.Images { + // 保存图片信息到文本Chunk + imageInfo := types.ImageInfo{ + URL: img.Url, + OriginalURL: img.OriginalUrl, + StartPos: int(img.Start), + EndPos: int(img.End), + OCRText: img.OcrText, + Caption: img.Caption, + } + chunkImages = append(chunkImages, imageInfo) + + // 将ImageInfo序列化为JSON + imageInfoJSON, err := json.Marshal([]types.ImageInfo{imageInfo}) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to marshal image info to JSON") + continue + } + + // 如果有OCR文本，创建OCR Chunk + if img.OcrText != "" { + ocrChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: knowledge.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + Content: img.OcrText, + ChunkIndex: maxSeq + i*100 + 1, // 使用不冲突的索引方式 + IsEnabled: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StartAt: int(img.Start), + EndAt: int(img.End), + ChunkType: types.ChunkTypeImageOCR, + ParentChunkID: textChunk.ID, + ImageInfo: string(imageInfoJSON), + } + insertChunks = append(insertChunks, ocrChunk) + logger.GetLogger(ctx).Infof("Created OCR chunk for image %d in chunk #%d", i, chunkData.Seq) + } + + // 如果有图片描述，创建Caption Chunk + if img.Caption != "" { + captionChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: knowledge.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + Content: img.Caption, + ChunkIndex: maxSeq + i*100 + 2, // 使用不冲突的索引方式 + IsEnabled: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StartAt: int(img.Start), + EndAt: int(img.End), + ChunkType: types.ChunkTypeImageCaption, + ParentChunkID: textChunk.ID, + ImageInfo: string(imageInfoJSON), + } + insertChunks = append(insertChunks, captionChunk) + logger.GetLogger(ctx).Infof("Created caption chunk for image %d in chunk #%d", i, chunkData.Seq) + } + } + + imageInfoJSON, err := json.Marshal(chunkImages) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to marshal image info to JSON") + continue + } + textChunk.ImageInfo = string(imageInfoJSON) + } + } + + // Sort chunks by index for proper ordering + sort.Slice(insertChunks, func(i, j int) bool { + return insertChunks[i].ChunkIndex < insertChunks[j].ChunkIndex + }) + + // 仅为文本类型的Chunk设置前后关系 + textChunks := make([]*types.Chunk, 0, len(chunks)) + for _, chunk := range insertChunks { + if chunk.ChunkType == types.ChunkTypeText { + textChunks = append(textChunks, chunk) + } + } + + // 设置文本Chunk之间的前后关系 + for i, chunk := range textChunks { + if i > 0 { + textChunks[i-1].NextChunkID = chunk.ID + } + if i < len(textChunks)-1 { + textChunks[i+1].PreChunkID = chunk.ID + } + } + if enableGraphRAG { + relationChunkSize := 5 + indirectRelationChunkSize := 5 + graphBuilder := NewGraphBuilder(s.config, chatModel) + err = graphBuilder.BuildGraph(ctx, textChunks) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks build graph failed") + span.RecordError(err) + } else { + for _, chunk := range textChunks { + chunk.RelationChunks, _ = json.Marshal(graphBuilder.GetRelationChunks(chunk.ID, relationChunkSize)) + chunk.IndirectRelationChunks, _ = json.Marshal(graphBuilder.GetIndirectRelationChunks(chunk.ID, indirectRelationChunkSize)) + } + for i, entity := range graphBuilder.GetAllEntities() { + relationChunks, _ := json.Marshal(entity.ChunkIDs) + entityChunk := &types.Chunk{ + ID: entity.ID, + TenantID: knowledge.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + Content: entity.Description, + ChunkIndex: maxSeq + i*100 + 3, + IsEnabled: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + ChunkType: types.ChunkTypeEntity, + RelationChunks: types.JSON(relationChunks), + } + insertChunks = append(insertChunks, entityChunk) + } + for i, relationship := range graphBuilder.GetAllRelationships() { + relationChunks, _ := json.Marshal(relationship.ChunkIDs) + relationshipChunk := &types.Chunk{ + ID: relationship.ID, + TenantID: knowledge.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + Content: relationship.Description, + ChunkIndex: maxSeq + i*100 + 4, + IsEnabled: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + ChunkType: types.ChunkTypeRelationship, + RelationChunks: types.JSON(relationChunks), + } + insertChunks = append(insertChunks, relationshipChunk) + } + } + } + + span.AddEvent("extract summary") + summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks) + if err != nil { + logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). + WithField("error", err).Errorf("processChunks get summary failed, use first chunk as description") + if len(textChunks) > 0 { + knowledge.Description = textChunks[0].Content + } + } else { + knowledge.Description = summary + } + span.SetAttributes(attribute.String("summary", knowledge.Description)) + + // 批量索引 + if strings.TrimSpace(knowledge.Description) != "" && len(textChunks) > 0 { + sChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: knowledge.TenantID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + Content: fmt.Sprintf("# 文档名称\n%s\n\n# 摘要\n%s", knowledge.FileName, knowledge.Description), + ChunkIndex: maxSeq + 3, // 使用不冲突的索引方式 + IsEnabled: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + StartAt: 0, + EndAt: 0, + ChunkType: types.ChunkTypeSummary, + ParentChunkID: textChunks[0].ID, + } + logger.GetLogger(ctx).Infof("Created summary chunk for %s with index %d", + sChunk.ParentChunkID, sChunk.ChunkIndex) + insertChunks = append(insertChunks, sChunk) + } + + // Create index information for each chunk + indexInfoList := utils.MapSlice(insertChunks, func(chunk *types.Chunk) *types.IndexInfo { + return &types.IndexInfo{ + Content: chunk.Content, + SourceID: chunk.ID, + SourceType: types.ChunkSourceType, + ChunkID: chunk.ID, + KnowledgeID: knowledge.ID, + KnowledgeBaseID: knowledge.KnowledgeBaseID, + } + }) + + // Initialize retrieval engine + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(tenantInfo.RetrieverEngines.Engines) + if err != nil { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(err) + return + } + + // Calculate storage size required for embeddings + span.AddEvent("estimate storage size") + totalStorageSize := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfoList) + if tenantInfo.StorageQuota > 0 { + // Re-fetch tenant storage information + tenantInfo, err = s.tenantRepo.GetTenantByID(ctx, tenantInfo.ID) + if err != nil { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(err) + return + } + // Check if there's enough storage quota available + if tenantInfo.StorageUsed+totalStorageSize > tenantInfo.StorageQuota { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = "存储空间不足" + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(errors.New("storage quota exceeded")) + return + } + } + + // Save chunks to database + span.AddEvent("create chunks") + if err := s.chunkService.CreateChunks(ctx, insertChunks); err != nil { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + span.RecordError(err) + return + } + + span.AddEvent("batch index") + err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList) + if err != nil { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + + // delete failed chunks + if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + logger.Errorf(ctx, "Delete chunks failed: %v", err) + } + + // delete index + if err := retrieveEngine.DeleteByKnowledgeIDList( + ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), + ); err != nil { + logger.Errorf(ctx, "Delete index failed: %v", err) + } + span.RecordError(err) + return + } + logger.GetLogger(ctx).Infof("processChunks batch index successfully, with %d index", len(indexInfoList)) + + logger.Infof(ctx, "processChunks create relationship rag task") + for _, chunk := range textChunks { + err := NewChunkExtractTask(ctx, s.task, chunk.TenantID, chunk.ID, kb.SummaryModelID) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks create chunk extract task failed") + span.RecordError(err) + } + } + + // Update knowledge status to completed + knowledge.ParseStatus = "completed" + knowledge.EnableStatus = "enabled" + knowledge.StorageSize = totalStorageSize + now := time.Now() + knowledge.ProcessedAt = &now + knowledge.UpdatedAt = now + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update knowledge failed") + } + + // Update tenant's storage usage + tenantInfo.StorageUsed += totalStorageSize + if err := s.tenantRepo.AdjustStorageUsed(ctx, tenantInfo.ID, totalStorageSize); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("processChunks update tenant storage used failed") + } + logger.GetLogger(ctx).Infof("processChunks successfully") +} + +// GetSummary generates a summary for knowledge content using an AI model +func (s *knowledgeService) getSummary(ctx context.Context, + summaryModel chat.Chat, knowledge *types.Knowledge, chunks []*types.Chunk, +) (string, error) { + // Get knowledge info from the first chunk + if len(chunks) == 0 { + return "", fmt.Errorf("no chunks provided for summary generation") + } + + // concat chunk contents + chunkContents := "" + allImageInfos := make([]*types.ImageInfo, 0) + + // then, sort chunks by StartAt + sortedChunks := make([]*types.Chunk, len(chunks)) + copy(sortedChunks, chunks) + sort.Slice(sortedChunks, func(i, j int) bool { + return sortedChunks[i].StartAt < sortedChunks[j].StartAt + }) + + // concat chunk contents and collect image infos + for _, chunk := range sortedChunks { + if chunk.EndAt > 4096 { + break + } + chunkContents = string([]rune(chunkContents)[:chunk.StartAt]) + chunk.Content + if chunk.ImageInfo != "" { + var images []*types.ImageInfo + if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err == nil { + allImageInfos = append(allImageInfos, images...) + } + } + } + // remove markdown image syntax + re := regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`) + chunkContents = re.ReplaceAllString(chunkContents, "") + // collect all image infos + if len(allImageInfos) > 0 { + // add image infos to chunk contents + var imageAnnotations string + for _, img := range allImageInfos { + if img.Caption != "" { + imageAnnotations += fmt.Sprintf("\n[图片描述: %s]", img.Caption) + } + if img.OCRText != "" { + imageAnnotations += fmt.Sprintf("\n[图片文字: %s]", img.OCRText) + } + } + + // concat chunk contents and image annotations + chunkContents = chunkContents + imageAnnotations + } + + if len(chunkContents) < 300 { + return chunkContents, nil + } + + // Prepare content with metadata for summary generation + contentWithMetadata := chunkContents + + // Add knowledge metadata if available + if knowledge != nil { + metadataIntro := fmt.Sprintf("文档类型: %s\n文件名称: %s\n", knowledge.FileType, knowledge.FileName) + + // Add additional metadata if available + if knowledge.Type != "" { + metadataIntro += fmt.Sprintf("知识类型: %s\n", knowledge.Type) + } + + // Prepend metadata to content + contentWithMetadata = metadataIntro + "\n内容:\n" + contentWithMetadata + } + + // Generate summary using AI model + thinking := false + summary, err := summaryModel.Chat(ctx, []chat.Message{ + { + Role: "system", + Content: s.config.Conversation.GenerateSummaryPrompt, + }, + { + Role: "user", + Content: contentWithMetadata, + }, + }, &chat.ChatOptions{ + Temperature: 0.3, + MaxTokens: 1024, + Thinking: &thinking, + }) + if err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("GetSummary failed") + return "", err + } + logger.GetLogger(ctx).WithField("summary", summary.Content).Infof("GetSummary success") + return summary.Content, nil +} + +// GetKnowledgeFile retrieves the physical file associated with a knowledge entry +func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) { + // Get knowledge record + knowledge, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), id) + if err != nil { + return nil, "", err + } + + // Get the file from storage + file, err := s.fileSvc.GetFile(ctx, knowledge.FilePath) + if err != nil { + return nil, "", err + } + + return file, knowledge.FileName, nil +} + +func (s *knowledgeService) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { + record, err := s.repo.GetKnowledgeByID(ctx, ctx.Value(types.TenantIDContextKey).(uint), knowledge.ID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge record: %v", err) + return err + } + // if need other fields update, please add here + if knowledge.Title != "" { + record.Title = knowledge.Title + } + + // Update knowledge record in the repository + if err := s.repo.UpdateKnowledge(ctx, record); err != nil { + logger.Errorf(ctx, "Failed to update knowledge: %v", err) + return err + } + logger.Infof(ctx, "Knowledge updated successfully, ID: %s", knowledge.ID) + return nil +} + +// isValidFileType checks if a file type is supported +func isValidFileType(filename string) bool { + switch strings.ToLower(getFileType(filename)) { + case "pdf", "txt", "docx", "doc", "md", "markdown", "png", "jpg", "jpeg", "gif": + return true + default: + return false + } +} + +// getFileType extracts the file extension from a filename +func getFileType(filename string) string { + ext := strings.Split(filename, ".") + if len(ext) < 2 { + return "unknown" + } + return ext[len(ext)-1] +} + +// isValidURL verifies if a URL is valid +// isValidURL 检查URL是否有效 +func isValidURL(url string) bool { + if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + return true + } + return false +} + +// GetKnowledgeBatch retrieves multiple knowledge entries by their IDs +func (s *knowledgeService) GetKnowledgeBatch(ctx context.Context, + tenantID uint, ids []string, +) ([]*types.Knowledge, error) { + if len(ids) == 0 { + return nil, nil + } + return s.repo.GetKnowledgeBatch(ctx, tenantID, ids) +} + +// calculateFileHash calculates MD5 hash of a file +func calculateFileHash(file *multipart.FileHeader) (string, error) { + f, err := file.Open() + if err != nil { + return "", err + } + defer f.Close() + + h := md5.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + // Reset file pointer for subsequent operations + if _, err := f.Seek(0, 0); err != nil { + return "", err + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +func calculateStr(strList ...string) string { + h := md5.New() + input := strings.Join(strList, "") + h.Write([]byte(input)) + return hex.EncodeToString(h.Sum(nil)) +} + +func (s *knowledgeService) CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error { + srcKB, dstKB, err := s.kbService.CopyKnowledgeBase(ctx, srcID, dstID) + if err != nil { + logger.Errorf(ctx, "Failed to copy knowledge base: %v", err) + return err + } + + addKnowledge, err := s.repo.AminusB(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge: %v", err) + return err + } + + delKnowledge, err := s.repo.AminusB(ctx, dstKB.TenantID, dstKB.ID, srcKB.TenantID, srcKB.ID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge: %v", err) + return err + } + logger.Infof(ctx, "Knowledge after update to add: %d, delete: %d", len(addKnowledge), len(delKnowledge)) + + batch := 10 + g, gctx := errgroup.WithContext(ctx) + for ids := range slices.Chunk(delKnowledge, batch) { + ids = ids + g.Go(func() error { + err := s.DeleteKnowledgeList(gctx, ids) + if err != nil { + logger.Errorf(gctx, "delete partial knowledge %v: %w", ids, err) + return err + } + return nil + }) + } + err = g.Wait() + if err != nil { + logger.Errorf(ctx, "delete total knowledge %d: %v", len(delKnowledge), err) + return err + } + + // Copy context out of auto-stop task + g, gctx = errgroup.WithContext(ctx) + g.SetLimit(batch) + for _, knowledge := range addKnowledge { + knowledge = knowledge + g.Go(func() error { + srcKn, err := s.repo.GetKnowledgeByID(gctx, srcKB.TenantID, knowledge) + if err != nil { + logger.Errorf(gctx, "get knowledge %s: %w", knowledge, err) + return err + } + err = s.cloneKnowledge(gctx, srcKn, dstKB) + if err != nil { + logger.Errorf(gctx, "clone knowledge %s: %w", knowledge, err) + return err + } + return nil + }) + } + err = g.Wait() + if err != nil { + logger.Errorf(ctx, "add total knowledge %d: %v", len(addKnowledge), err) + return err + } + return nil +} + +func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, chunks []*types.Chunk) error { + // Get embedding model from knowledge base + sourceKB, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + return err + } + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, sourceKB.EmbeddingModelID) + if err != nil { + return err + } + + // Initialize composite retrieve engine from tenant configuration + indexInfo := make([]*types.IndexInfo, 0, len(chunks)) + ids := make([]string, 0, len(chunks)) + for _, chunk := range chunks { + if chunk.KnowledgeBaseID != kbID { + logger.Warnf(ctx, "Knowledge base ID mismatch: %s != %s", chunk.KnowledgeBaseID, kbID) + continue + } + indexInfo = append(indexInfo, &types.IndexInfo{ + Content: chunk.Content, + SourceID: chunk.ID, + SourceType: types.ChunkSourceType, + ChunkID: chunk.ID, + KnowledgeID: chunk.KnowledgeID, + KnowledgeBaseID: chunk.KnowledgeBaseID, + }) + ids = append(ids, chunk.ID) + } + + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(tenantInfo.RetrieverEngines.Engines) + if err != nil { + return err + } + + // Delete old vector representation of the chunk + err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions()) + if err != nil { + return err + } + + // Index updated chunk content with new vector representation + err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo) + if err != nil { + return err + } + return nil +} + +func (s *knowledgeService) UpdateImageInfo(ctx context.Context, knowledgeID string, chunkID string, imageInfo string) error { + var images []*types.ImageInfo + if err := json.Unmarshal([]byte(imageInfo), &images); err != nil { + logger.Errorf(ctx, "Failed to unmarshal image info: %v", err) + return err + } + if len(images) != 1 { + logger.Warnf(ctx, "Expected exactly one image info, got %d", len(images)) + return nil + } + image := images[0] + + // Retrieve all chunks with the given parent chunk ID + chunk, err := s.chunkService.GetChunkByID(ctx, knowledgeID, chunkID) + if err != nil { + logger.Errorf(ctx, "Failed to get chunk: %v", err) + return err + } + chunk.ImageInfo = imageInfo + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + chunkChildren, err := s.chunkService.ListChunkByParentID(ctx, tenantID, chunkID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "parent_chunk_id": chunkID, + "tenant_id": tenantID, + }) + return err + } + logger.Infof(ctx, "Found %d chunks with parent chunk ID: %s", len(chunkChildren), chunkID) + + // Iterate through each chunk and update its content based on the image information + updateChunk := []*types.Chunk{chunk} + var addChunk []*types.Chunk + + // Track whether we've found OCR and caption child chunks for this image + hasOCRChunk := false + hasCaptionChunk := false + + for i, child := range chunkChildren { + // Skip chunks that are not image types + var cImageInfo []*types.ImageInfo + err = json.Unmarshal([]byte(child.ImageInfo), &cImageInfo) + if err != nil { + logger.Warnf(ctx, "Failed to unmarshal image %s info: %v", child.ID, err) + continue + } + if len(cImageInfo) == 0 { + continue + } + if cImageInfo[0].OriginalURL != image.OriginalURL { + logger.Warnf(ctx, "Skipping chunk ID: %s, image URL mismatch: %s != %s", + child.ID, cImageInfo[0].OriginalURL, image.OriginalURL) + continue + } + + // Mark that we've found chunks for this image + if child.ChunkType == types.ChunkTypeImageCaption { + hasCaptionChunk = true + // Update caption if it has changed + if image.Caption != cImageInfo[0].Caption { + child.Content = image.Caption + child.ImageInfo = imageInfo + updateChunk = append(updateChunk, chunkChildren[i]) + } + } else if child.ChunkType == types.ChunkTypeImageOCR { + hasOCRChunk = true + // Update OCR if it has changed + if image.OCRText != cImageInfo[0].OCRText { + child.Content = image.OCRText + child.ImageInfo = imageInfo + updateChunk = append(updateChunk, chunkChildren[i]) + } + } + } + + // Create a new caption chunk if it doesn't exist and we have caption data + if !hasCaptionChunk && image.Caption != "" { + captionChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: tenantID, + KnowledgeID: chunk.KnowledgeID, + KnowledgeBaseID: chunk.KnowledgeBaseID, + Content: image.Caption, + ChunkType: types.ChunkTypeImageCaption, + ParentChunkID: chunk.ID, + ImageInfo: imageInfo, + } + addChunk = append(addChunk, captionChunk) + logger.Infof(ctx, "Created new caption chunk ID: %s for image URL: %s", captionChunk.ID, image.OriginalURL) + } + + // Create a new OCR chunk if it doesn't exist and we have OCR data + if !hasOCRChunk && image.OCRText != "" { + ocrChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: tenantID, + KnowledgeID: chunk.KnowledgeID, + KnowledgeBaseID: chunk.KnowledgeBaseID, + Content: image.OCRText, + ChunkType: types.ChunkTypeImageOCR, + ParentChunkID: chunk.ID, + ImageInfo: imageInfo, + } + addChunk = append(addChunk, ocrChunk) + logger.Infof(ctx, "Created new OCR chunk ID: %s for image URL: %s", ocrChunk.ID, image.OriginalURL) + } + logger.Infof(ctx, "Updated %d chunks out of %d total chunks", len(updateChunk), len(chunkChildren)+1) + + if len(addChunk) > 0 { + err := s.chunkService.CreateChunks(ctx, addChunk) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "add_chunk_size": len(addChunk), + }) + return err + } + } + + // Update the chunks + for _, c := range updateChunk { + err := s.chunkService.UpdateChunk(ctx, c) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "chunk_id": c.ID, + "knowledge_id": c.KnowledgeID, + }) + return err + } + } + + // Update the chunk vector + err = s.updateChunkVector(ctx, chunk.KnowledgeBaseID, append(updateChunk, addChunk...)) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "chunk_id": chunk.ID, + "knowledge_id": chunk.KnowledgeID, + }) + return err + } + + // Update the knowledge file hash + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge: %v", err) + return err + } + fileHash := calculateStr(knowledgeID, knowledge.FileHash, imageInfo) + knowledge.FileHash = fileHash + err = s.repo.UpdateKnowledge(ctx, knowledge) + if err != nil { + logger.Warnf(ctx, "Failed to update knowledge file hash: %v", err) + } + + logger.Infof(ctx, "Updated chunk successfully, chunk ID: %s, knowledge ID: %s", chunk.ID, chunk.KnowledgeID) + return nil +} + +// CloneChunk clone chunks from one knowledge to another +// This method transfers a chunk from a source knowledge document to a target knowledge document +// It handles the creation of new chunks in the target knowledge and updates the vector database accordingly +// Parameters: +// - ctx: Context with authentication and request information +// - src: Source knowledge document containing the chunk to move +// - dst: Target knowledge document where the chunk will be moved +// +// Returns: +// - error: Any error encountered during the move operation +// +// This method handles the chunk transfer logic, including creating new chunks in the target knowledge +// and updating the vector database representation of the moved chunks. +// It also ensures that the chunk's relationships (like pre and next chunk IDs) are maintained +// by mapping the source chunk IDs to the new target chunk IDs. +func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowledge) error { + chunkPage := 1 + chunkPageSize := 100 + srcTodst := map[string]string{} + targetChunks := make([]*types.Chunk, 0, 10) + chunkType := []types.ChunkType{ + types.ChunkTypeText, types.ChunkTypeSummary, + types.ChunkTypeImageCaption, types.ChunkTypeImageOCR, + } + for { + sourceChunks, _, err := s.chunkRepo.ListPagedChunksByKnowledgeID(ctx, + src.TenantID, + src.ID, + &types.Pagination{ + Page: chunkPage, + PageSize: chunkPageSize, + }, + chunkType, + ) + chunkPage++ + if err != nil { + return err + } + if len(sourceChunks) == 0 { + break + } + for _, sourceChunk := range sourceChunks { + targetChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: dst.TenantID, + KnowledgeID: dst.ID, + KnowledgeBaseID: dst.KnowledgeBaseID, + Content: sourceChunk.Content, + ChunkIndex: sourceChunk.ChunkIndex, + IsEnabled: sourceChunk.IsEnabled, + StartAt: sourceChunk.StartAt, + EndAt: sourceChunk.EndAt, + PreChunkID: sourceChunk.PreChunkID, + NextChunkID: sourceChunk.NextChunkID, + ChunkType: sourceChunk.ChunkType, + ParentChunkID: sourceChunk.ParentChunkID, + ImageInfo: sourceChunk.ImageInfo, + } + targetChunks = append(targetChunks, targetChunk) + srcTodst[sourceChunk.ID] = targetChunk.ID + } + } + for _, targetChunk := range targetChunks { + if val, ok := srcTodst[targetChunk.PreChunkID]; ok { + targetChunk.PreChunkID = val + } else { + targetChunk.PreChunkID = "" + } + if val, ok := srcTodst[targetChunk.NextChunkID]; ok { + targetChunk.NextChunkID = val + } else { + targetChunk.NextChunkID = "" + } + if val, ok := srcTodst[targetChunk.ParentChunkID]; ok { + targetChunk.ParentChunkID = val + } else { + targetChunk.ParentChunkID = "" + } + } + for chunks := range slices.Chunk(targetChunks, chunkPageSize) { + err := s.chunkRepo.CreateChunks(ctx, chunks) + if err != nil { + return err + } + } + + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(tenantInfo.RetrieverEngines.Engines) + if err != nil { + return err + } + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, dst.EmbeddingModelID) + if err != nil { + return err + } + if err := retrieveEngine.CopyIndices(ctx, src.KnowledgeBaseID, dst.KnowledgeBaseID, + map[string]string{src.ID: dst.ID}, + srcTodst, + embeddingModel.GetDimensions(), + ); err != nil { + return err + } + return nil +} + +func IsImageType(fileType string) bool { + switch fileType { + case "jpg", "jpeg", "png", "gif", "webp", "bmp", "svg", "tiff": + return true + default: + return false + } +} diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go new file mode 100644 index 0000000..cd991e3 --- /dev/null +++ b/internal/application/service/knowledgebase.go @@ -0,0 +1,578 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "slices" + "time" + + "github.com/Tencent/WeKnora/internal/application/service/retriever" + "github.com/Tencent/WeKnora/internal/common" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" +) + +// ErrInvalidTenantID represents an error for invalid tenant ID +var ErrInvalidTenantID = errors.New("invalid tenant ID") + +// knowledgeBaseService implements the knowledge base service interface +type knowledgeBaseService struct { + repo interfaces.KnowledgeBaseRepository + kgRepo interfaces.KnowledgeRepository + chunkRepo interfaces.ChunkRepository + modelService interfaces.ModelService +} + +// NewKnowledgeBaseService creates a new knowledge base service +func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository, + kgRepo interfaces.KnowledgeRepository, + chunkRepo interfaces.ChunkRepository, + modelService interfaces.ModelService, +) interfaces.KnowledgeBaseService { + return &knowledgeBaseService{ + repo: repo, + kgRepo: kgRepo, + chunkRepo: chunkRepo, + modelService: modelService, + } +} + +// CreateKnowledgeBase creates a new knowledge base +func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context, + kb *types.KnowledgeBase, +) (*types.KnowledgeBase, error) { + // Generate UUID and set creation timestamps + if kb.ID == "" { + kb.ID = uuid.New().String() + } + kb.CreatedAt = time.Now() + kb.TenantID = ctx.Value(types.TenantIDContextKey).(uint) + kb.UpdatedAt = time.Now() + + logger.Infof(ctx, "Creating knowledge base, ID: %s, tenant ID: %d, name: %s", kb.ID, kb.TenantID, kb.Name) + + if err := s.repo.CreateKnowledgeBase(ctx, kb); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": kb.ID, + "tenant_id": kb.TenantID, + }) + return nil, err + } + + logger.Infof(ctx, "Knowledge base created successfully, ID: %s, name: %s", kb.ID, kb.Name) + return kb, nil +} + +// GetKnowledgeBaseByID retrieves a knowledge base by its ID +func (s *knowledgeBaseService) GetKnowledgeBaseByID(ctx context.Context, id string) (*types.KnowledgeBase, error) { + if id == "" { + logger.Error(ctx, "Knowledge base ID is empty") + return nil, errors.New("knowledge base ID cannot be empty") + } + + logger.Infof(ctx, "Retrieving knowledge base, ID: %s", id) + + kb, err := s.repo.GetKnowledgeBaseByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + }) + return nil, err + } + + logger.Infof(ctx, "Knowledge base retrieved successfully, ID: %s, name: %s", kb.ID, kb.Name) + return kb, nil +} + +// ListKnowledgeBases returns all knowledge bases for a tenant +func (s *knowledgeBaseService) ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) { + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Retrieving knowledge base list for tenant, tenant ID: %d", tenantID) + + kbs, err := s.repo.ListKnowledgeBasesByTenantID(ctx, tenantID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Infof( + ctx, + "Knowledge base list retrieved successfully, tenant ID: %d, knowledge base count: %d", + tenantID, + len(kbs), + ) + return kbs, nil +} + +// UpdateKnowledgeBase updates a knowledge base's properties +func (s *knowledgeBaseService) UpdateKnowledgeBase(ctx context.Context, + id string, + name string, + description string, + config *types.KnowledgeBaseConfig, +) (*types.KnowledgeBase, error) { + if id == "" { + logger.Error(ctx, "Knowledge base ID is empty") + return nil, errors.New("knowledge base ID cannot be empty") + } + + logger.Infof(ctx, "Updating knowledge base, ID: %s, name: %s", id, name) + + // Get existing knowledge base + kb, err := s.repo.GetKnowledgeBaseByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + }) + return nil, err + } + + // Update the knowledge base properties + kb.Name = name + kb.Description = description + kb.ChunkingConfig = config.ChunkingConfig + kb.ImageProcessingConfig = config.ImageProcessingConfig + kb.UpdatedAt = time.Now() + + logger.Info(ctx, "Saving knowledge base update") + if err := s.repo.UpdateKnowledgeBase(ctx, kb); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + }) + return nil, err + } + + logger.Infof(ctx, "Knowledge base updated successfully, ID: %s, name: %s", kb.ID, kb.Name) + return kb, nil +} + +// DeleteKnowledgeBase deletes a knowledge base by its ID +func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id string) error { + if id == "" { + logger.Error(ctx, "Knowledge base ID is empty") + return errors.New("knowledge base ID cannot be empty") + } + + logger.Infof(ctx, "Deleting knowledge base, ID: %s", id) + + err := s.repo.DeleteKnowledgeBase(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + }) + return err + } + + logger.Infof(ctx, "Knowledge base deleted successfully, ID: %s", id) + return nil +} + +// SetEmbeddingModel sets the embedding model for a knowledge base +func (s *knowledgeBaseService) SetEmbeddingModel(ctx context.Context, id string, modelID string) error { + if id == "" { + logger.Error(ctx, "Knowledge base ID is empty") + return errors.New("knowledge base ID cannot be empty") + } + + if modelID == "" { + logger.Error(ctx, "Model ID is empty") + return errors.New("model ID cannot be empty") + } + + logger.Infof(ctx, "Setting embedding model for knowledge base, knowledge base ID: %s, model ID: %s", id, modelID) + + // Get the knowledge base + kb, err := s.repo.GetKnowledgeBaseByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + }) + return err + } + + // Update the knowledge base's embedding model + kb.EmbeddingModelID = modelID + kb.UpdatedAt = time.Now() + + logger.Info(ctx, "Saving knowledge base embedding model update") + err = s.repo.UpdateKnowledgeBase(ctx, kb) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + "embedding_model_id": modelID, + }) + return err + } + + logger.Infof( + ctx, + "Knowledge base embedding model set successfully, knowledge base ID: %s, model ID: %s", + id, + modelID, + ) + return nil +} + +// CopyKnowledgeBase copies a knowledge base to a new knowledge base +// 浅拷贝 +func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context, + srcKB string, dstKB string, +) (*types.KnowledgeBase, *types.KnowledgeBase, error) { + sourceKB, err := s.repo.GetKnowledgeBaseByID(ctx, srcKB) + if err != nil { + logger.Errorf(ctx, "Get source knowledge base failed: %v", err) + return nil, nil, err + } + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + var targetKB *types.KnowledgeBase + if dstKB != "" { + targetKB, err = s.repo.GetKnowledgeBaseByID(ctx, dstKB) + if err != nil { + return nil, nil, err + } + } else { + targetKB = &types.KnowledgeBase{ + ID: uuid.New().String(), + Name: sourceKB.Name, + Description: sourceKB.Description, + TenantID: tenantID, + ChunkingConfig: sourceKB.ChunkingConfig, + ImageProcessingConfig: sourceKB.ImageProcessingConfig, + EmbeddingModelID: sourceKB.EmbeddingModelID, + SummaryModelID: sourceKB.SummaryModelID, + RerankModelID: sourceKB.RerankModelID, + VLMModelID: sourceKB.VLMModelID, + StorageConfig: sourceKB.StorageConfig, + } + if err := s.repo.CreateKnowledgeBase(ctx, targetKB); err != nil { + return nil, nil, err + } + } + return sourceKB, targetKB, nil +} + +// HybridSearch performs hybrid search, including vector retrieval and keyword retrieval +func (s *knowledgeBaseService) HybridSearch(ctx context.Context, + id string, + params types.SearchParams, +) ([]*types.SearchResult, error) { + logger.Infof(ctx, "Hybrid search parameters, knowledge base ID: %s, query text: %s", id, params.QueryText) + + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + logger.Infof(ctx, "Creating composite retrieval engine, tenant ID: %d", tenantInfo.ID) + + // Create a composite retrieval engine with tenant's configured retrievers + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(tenantInfo.RetrieverEngines.Engines) + if err != nil { + logger.Errorf(ctx, "Failed to create retrieval engine: %v", err) + return nil, err + } + + var retrieveParams []types.RetrieveParams + var embeddingModel embedding.Embedder + var kb *types.KnowledgeBase + + // Add vector retrieval params if supported + if retrieveEngine.SupportRetriever(types.VectorRetrieverType) { + logger.Info(ctx, "Vector retrieval supported, preparing vector retrieval parameters") + + kb, err = s.repo.GetKnowledgeBaseByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + }) + return nil, err + } + + logger.Infof(ctx, "Getting embedding model, model ID: %s", kb.EmbeddingModelID) + embeddingModel, err = s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) + if err != nil { + logger.Errorf(ctx, "Failed to get embedding model, model ID: %s, error: %v", kb.EmbeddingModelID, err) + return nil, err + } + logger.Infof(ctx, "Embedding model retrieved: %v", embeddingModel) + + // Generate embedding vector for the query text + logger.Info(ctx, "Starting to generate query embedding") + queryEmbedding, err := embeddingModel.Embed(ctx, params.QueryText) + if err != nil { + logger.Errorf(ctx, "Failed to embed query text, query text: %s, error: %v", params.QueryText, err) + return nil, err + } + logger.Infof(ctx, "Query embedding generated successfully, embedding vector length: %d", len(queryEmbedding)) + + retrieveParams = append(retrieveParams, types.RetrieveParams{ + Query: params.QueryText, + Embedding: queryEmbedding, + KnowledgeBaseIDs: []string{id}, + TopK: params.MatchCount, + Threshold: params.VectorThreshold, + RetrieverType: types.VectorRetrieverType, + }) + logger.Info(ctx, "Vector retrieval parameters setup completed") + } + + // Add keyword retrieval params if supported + if retrieveEngine.SupportRetriever(types.KeywordsRetrieverType) { + logger.Info(ctx, "Keyword retrieval supported, preparing keyword retrieval parameters") + retrieveParams = append(retrieveParams, types.RetrieveParams{ + Query: params.QueryText, + KnowledgeBaseIDs: []string{id}, + TopK: params.MatchCount, + Threshold: params.KeywordThreshold, + RetrieverType: types.KeywordsRetrieverType, + }) + logger.Info(ctx, "Keyword retrieval parameters setup completed") + } + + if len(retrieveParams) == 0 { + logger.Error(ctx, "No retrieval parameters available") + return nil, errors.New("no retrieve params") + } + + // Execute retrieval using the configured engines + logger.Infof(ctx, "Starting retrieval, parameter count: %d", len(retrieveParams)) + retrieveResults, err := retrieveEngine.Retrieve(ctx, retrieveParams) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "knowledge_base_id": id, + "query_text": params.QueryText, + }) + return nil, err + } + + // Collect all results from different retrievers and deduplicate by chunk ID + logger.Infof(ctx, "Processing retrieval results") + matchResults := []*types.IndexWithScore{} + for _, retrieveResult := range retrieveResults { + logger.Infof(ctx, "Retrieval results, engine: %v, retriever: %v, count: %v", + retrieveResult.RetrieverEngineType, + retrieveResult.RetrieverType, + len(retrieveResult.Results), + ) + matchResults = append(matchResults, retrieveResult.Results...) + } + + // Early return if no results + if len(matchResults) == 0 { + logger.Info(ctx, "No search results found") + return nil, nil + } + + // Deduplicate results by chunk ID + logger.Infof(ctx, "Result count before deduplication: %d", len(matchResults)) + deduplicatedChunks := common.Deduplicate(func(r *types.IndexWithScore) string { return r.ChunkID }, matchResults...) + logger.Infof(ctx, "Result count after deduplication: %d", len(deduplicatedChunks)) + + return s.processSearchResults(ctx, deduplicatedChunks) +} + +// processSearchResults handles the processing of search results, optimizing database queries +func (s *knowledgeBaseService) processSearchResults(ctx context.Context, + chunks []*types.IndexWithScore, +) ([]*types.SearchResult, error) { + if len(chunks) == 0 { + return nil, nil + } + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + + // Prepare data structures for efficient processing + var knowledgeIDs []string + var chunkIDs []string + chunkScores := make(map[string]float64) + chunkMatchTypes := make(map[string]types.MatchType) + processedKnowledgeIDs := make(map[string]bool) + + // Collect all knowledge and chunk IDs + for _, chunk := range chunks { + if !processedKnowledgeIDs[chunk.KnowledgeID] { + knowledgeIDs = append(knowledgeIDs, chunk.KnowledgeID) + processedKnowledgeIDs[chunk.KnowledgeID] = true + } + + chunkIDs = append(chunkIDs, chunk.ChunkID) + chunkScores[chunk.ChunkID] = chunk.Score + chunkMatchTypes[chunk.ChunkID] = chunk.MatchType + } + + // Batch fetch knowledge data + logger.Infof(ctx, "Fetching knowledge data for %d IDs", len(knowledgeIDs)) + knowledgeMap, err := s.fetchKnowledgeData(ctx, tenantID, knowledgeIDs) + if err != nil { + return nil, err + } + + // Batch fetch all chunks in one go + logger.Infof(ctx, "Fetching chunk data for %d IDs", len(chunkIDs)) + allChunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, chunkIDs) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenantID, + "chunk_ids": chunkIDs, + }) + return nil, err + } + + // Build chunk map and collect additional IDs to fetch + chunkMap := make(map[string]*types.Chunk, len(allChunks)) + var additionalChunkIDs []string + processedChunkIDs := make(map[string]bool) + + // First pass: Build chunk map and collect parent IDs + for _, chunk := range allChunks { + chunkMap[chunk.ID] = chunk + processedChunkIDs[chunk.ID] = true + + // Collect parent chunks + if chunk.ParentChunkID != "" && !processedChunkIDs[chunk.ParentChunkID] { + additionalChunkIDs = append(additionalChunkIDs, chunk.ParentChunkID) + processedChunkIDs[chunk.ParentChunkID] = true + + // Pass score to parent + chunkScores[chunk.ParentChunkID] = chunkScores[chunk.ID] + chunkMatchTypes[chunk.ParentChunkID] = types.MatchTypeParentChunk + } + + // Collect related chunks + relationChunkIDs := s.collectRelatedChunkIDs(chunk, processedChunkIDs) + for _, chunkID := range relationChunkIDs { + additionalChunkIDs = append(additionalChunkIDs, chunkID) + chunkMatchTypes[chunkID] = types.MatchTypeRelationChunk + } + + // Add nearby chunks (prev and next) + if chunk.ChunkType == types.ChunkTypeText { + if chunk.NextChunkID != "" && !processedChunkIDs[chunk.NextChunkID] { + additionalChunkIDs = append(additionalChunkIDs, chunk.NextChunkID) + processedChunkIDs[chunk.NextChunkID] = true + chunkMatchTypes[chunk.NextChunkID] = types.MatchTypeNearByChunk + } + if chunk.PreChunkID != "" && !processedChunkIDs[chunk.PreChunkID] { + additionalChunkIDs = append(additionalChunkIDs, chunk.PreChunkID) + processedChunkIDs[chunk.PreChunkID] = true + chunkMatchTypes[chunk.PreChunkID] = types.MatchTypeNearByChunk + } + } + } + + // Fetch all additional chunks in one go if needed + if len(additionalChunkIDs) > 0 { + logger.Infof(ctx, "Fetching %d additional chunks", len(additionalChunkIDs)) + additionalChunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, additionalChunkIDs) + if err != nil { + logger.Warnf(ctx, "Failed to fetch some additional chunks: %v", err) + // Continue with what we have + } else { + // Add to chunk map + for _, chunk := range additionalChunks { + chunkMap[chunk.ID] = chunk + } + } + } + + // Build final search results + var searchResults []*types.SearchResult + for chunkID, chunk := range chunkMap { + if !s.isValidTextChunk(chunk) { + continue + } + + score, hasScore := chunkScores[chunkID] + if !hasScore || score <= 0 { + score = 0.0 + } + + if knowledge, ok := knowledgeMap[chunk.KnowledgeID]; ok { + matchType := types.MatchTypeParentChunk + if specificType, exists := chunkMatchTypes[chunkID]; exists { + matchType = specificType + } else { + logger.Warnf(ctx, "Unkonwn match type for chunk: %s", chunkID) + continue + } + searchResults = append(searchResults, s.buildSearchResult(chunk, knowledge, score, matchType)) + } + } + logger.Infof(ctx, "Search results processed, total: %d", len(searchResults)) + return searchResults, nil +} + +// collectRelatedChunkIDs extracts related chunk IDs from a chunk +func (s *knowledgeBaseService) collectRelatedChunkIDs(chunk *types.Chunk, processedIDs map[string]bool) []string { + var relatedIDs []string + // Process direct relations + if len(chunk.RelationChunks) > 0 { + var relations []string + if err := json.Unmarshal(chunk.RelationChunks, &relations); err == nil { + for _, id := range relations { + if !processedIDs[id] { + relatedIDs = append(relatedIDs, id) + processedIDs[id] = true + } + } + } + } + return relatedIDs +} + +// buildSearchResult creates a search result from chunk and knowledge +func (s *knowledgeBaseService) buildSearchResult(chunk *types.Chunk, + knowledge *types.Knowledge, + score float64, + matchType types.MatchType, +) *types.SearchResult { + return &types.SearchResult{ + ID: chunk.ID, + Content: chunk.Content, + KnowledgeID: chunk.KnowledgeID, + ChunkIndex: chunk.ChunkIndex, + KnowledgeTitle: knowledge.Title, + StartAt: chunk.StartAt, + EndAt: chunk.EndAt, + Seq: chunk.ChunkIndex, + Score: score, + MatchType: matchType, + Metadata: knowledge.GetMetadata(), + ChunkType: string(chunk.ChunkType), + ParentChunkID: chunk.ParentChunkID, + ImageInfo: chunk.ImageInfo, + KnowledgeFilename: knowledge.FileName, + KnowledgeSource: knowledge.Source, + } +} + +// isValidTextChunk checks if a chunk is a valid text chunk +func (s *knowledgeBaseService) isValidTextChunk(chunk *types.Chunk) bool { + return slices.Contains([]types.ChunkType{ + types.ChunkTypeText, types.ChunkTypeSummary, + }, chunk.ChunkType) +} + +// fetchKnowledgeData gets knowledge data in batch +func (s *knowledgeBaseService) fetchKnowledgeData(ctx context.Context, + tenantID uint, + knowledgeIDs []string, +) (map[string]*types.Knowledge, error) { + knowledges, err := s.kgRepo.GetKnowledgeBatch(ctx, tenantID, knowledgeIDs) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenantID, + "knowledge_ids": knowledgeIDs, + }) + return nil, err + } + + knowledgeMap := make(map[string]*types.Knowledge, len(knowledges)) + for _, knowledge := range knowledges { + knowledgeMap[knowledge.ID] = knowledge + } + + return knowledgeMap, nil +} diff --git a/internal/application/service/message.go b/internal/application/service/message.go new file mode 100644 index 0000000..5304de9 --- /dev/null +++ b/internal/application/service/message.go @@ -0,0 +1,287 @@ +package service + +import ( + "context" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// messageService implements the MessageService interface for managing messaging operations +// It handles creating, retrieving, updating, and deleting messages within sessions +type messageService struct { + messageRepo interfaces.MessageRepository // Repository for message storage operations + sessionRepo interfaces.SessionRepository // Repository for session validation +} + +// NewMessageService creates a new message service instance with the required repositories +// Parameters: +// - messageRepo: Repository for persisting and retrieving messages +// - sessionRepo: Repository for validating session existence +// +// Returns an implementation of the MessageService interface +func NewMessageService(messageRepo interfaces.MessageRepository, + sessionRepo interfaces.SessionRepository, +) interfaces.MessageService { + return &messageService{ + messageRepo: messageRepo, + sessionRepo: sessionRepo, + } +} + +// CreateMessage creates a new message within an existing session +// It validates that the session exists before creating the message +// Parameters: +// - ctx: Context containing tenant information +// - message: The message to be created +// +// Returns the created message or an error if creation fails +func (s *messageService) CreateMessage(ctx context.Context, message *types.Message) (*types.Message, error) { + logger.Info(ctx, "Start creating message") + logger.Infof(ctx, "Creating message for session ID: %s", message.SessionID) + + // Check if the session exists to validate the message belongs to a valid session + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if session exists, tenant ID: %d, session ID: %s", tenantID, message.SessionID) + _, err := s.sessionRepo.Get(ctx, tenantID, message.SessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session: %v", err) + return nil, err + } + + // Create the message in the repository + logger.Info(ctx, "Session exists, creating message") + createdMessage, err := s.messageRepo.CreateMessage(ctx, message) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": message.SessionID, + }) + return nil, err + } + + logger.Infof(ctx, "Message created successfully, ID: %s", createdMessage.ID) + return createdMessage, nil +} + +// GetMessage retrieves a specific message by its ID within a session +// Parameters: +// - ctx: Context containing tenant information +// - sessionID: The ID of the session containing the message +// - messageID: The ID of the message to retrieve +// +// Returns the requested message or an error if retrieval fails +func (s *messageService) GetMessage(ctx context.Context, sessionID string, messageID string) (*types.Message, error) { + logger.Info(ctx, "Start getting message") + logger.Infof(ctx, "Getting message, session ID: %s, message ID: %s", sessionID, messageID) + + // Verify the session exists before attempting to retrieve the message + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if session exists, tenant ID: %d", tenantID) + _, err := s.sessionRepo.Get(ctx, tenantID, sessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session: %v", err) + return nil, err + } + + // Retrieve the message from the repository + logger.Info(ctx, "Session exists, getting message") + message, err := s.messageRepo.GetMessage(ctx, sessionID, messageID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + "message_id": messageID, + }) + return nil, err + } + + logger.Info(ctx, "Message retrieved successfully") + return message, nil +} + +// GetMessagesBySession retrieves paginated messages for a specific session +// Parameters: +// - ctx: Context containing tenant information +// - sessionID: The ID of the session to get messages from +// - page: The page number for pagination (0-based) +// - pageSize: The number of messages per page +// +// Returns a slice of messages or an error if retrieval fails +func (s *messageService) GetMessagesBySession(ctx context.Context, + sessionID string, page int, pageSize int, +) ([]*types.Message, error) { + logger.Info(ctx, "Start getting messages by session") + logger.Infof(ctx, "Getting messages for session ID: %s, page: %d, pageSize: %d", sessionID, page, pageSize) + + // Verify the session exists before retrieving messages + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if session exists, tenant ID: %d", tenantID) + _, err := s.sessionRepo.Get(ctx, tenantID, sessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session: %v", err) + return nil, err + } + + // Retrieve paginated messages + logger.Info(ctx, "Session exists, getting messages") + messages, err := s.messageRepo.GetMessagesBySession(ctx, sessionID, page, pageSize) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + "page": page, + "page_size": pageSize, + }) + return nil, err + } + + logger.Infof(ctx, "Retrieved %d messages successfully", len(messages)) + return messages, nil +} + +// GetRecentMessagesBySession retrieves the most recent messages from a session +// This is typically used for loading the initial conversation history +// Parameters: +// - ctx: Context containing tenant information +// - sessionID: The ID of the session to get messages from +// - limit: Maximum number of messages to retrieve +// +// Returns a slice of recent messages or an error if retrieval fails +func (s *messageService) GetRecentMessagesBySession(ctx context.Context, + sessionID string, limit int, +) ([]*types.Message, error) { + logger.Info(ctx, "Start getting recent messages by session") + logger.Infof(ctx, "Getting recent messages for session ID: %s, limit: %d", sessionID, limit) + + // Verify the session exists before retrieving messages + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if session exists, tenant ID: %d", tenantID) + _, err := s.sessionRepo.Get(ctx, tenantID, sessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session: %v", err) + return nil, err + } + + // Retrieve the most recent messages + logger.Info(ctx, "Session exists, getting recent messages") + messages, err := s.messageRepo.GetRecentMessagesBySession(ctx, sessionID, limit) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + "limit": limit, + }) + return nil, err + } + + logger.Infof(ctx, "Retrieved %d recent messages successfully", len(messages)) + return messages, nil +} + +// GetMessagesBySessionBeforeTime retrieves messages sent before a specific time +// This is typically used for pagination when scrolling through conversation history +// Parameters: +// - ctx: Context containing tenant information +// - sessionID: The ID of the session to get messages from +// - beforeTime: Timestamp to retrieve messages before +// - limit: Maximum number of messages to retrieve +// +// Returns a slice of messages or an error if retrieval fails +func (s *messageService) GetMessagesBySessionBeforeTime(ctx context.Context, + sessionID string, beforeTime time.Time, limit int, +) ([]*types.Message, error) { + logger.Info(ctx, "Start getting messages before time") + logger.Infof(ctx, "Getting messages before %v for session ID: %s, limit: %d", beforeTime, sessionID, limit) + + // Verify the session exists before retrieving messages + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if session exists, tenant ID: %d", tenantID) + _, err := s.sessionRepo.Get(ctx, tenantID, sessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session: %v", err) + return nil, err + } + + // Retrieve messages before the specified time + logger.Info(ctx, "Session exists, getting messages before time") + messages, err := s.messageRepo.GetMessagesBySessionBeforeTime(ctx, sessionID, beforeTime, limit) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + "before_time": beforeTime, + "limit": limit, + }) + return nil, err + } + + logger.Infof(ctx, "Retrieved %d messages before time successfully", len(messages)) + return messages, nil +} + +// UpdateMessage updates an existing message's content or metadata +// Parameters: +// - ctx: Context containing tenant information +// - message: The message with updated fields +// +// Returns an error if the update fails +func (s *messageService) UpdateMessage(ctx context.Context, message *types.Message) error { + logger.Info(ctx, "Start updating message") + logger.Infof(ctx, "Updating message, ID: %s, session ID: %s", message.ID, message.SessionID) + + // Verify the session exists before updating the message + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if session exists, tenant ID: %d", tenantID) + _, err := s.sessionRepo.Get(ctx, tenantID, message.SessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session: %v", err) + return err + } + + // Update the message in the repository + logger.Info(ctx, "Session exists, updating message") + err = s.messageRepo.UpdateMessage(ctx, message) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": message.SessionID, + "message_id": message.ID, + }) + return err + } + + logger.Info(ctx, "Message updated successfully") + return nil +} + +// DeleteMessage removes a message from a session +// Parameters: +// - ctx: Context containing tenant information +// - sessionID: The ID of the session containing the message +// - messageID: The ID of the message to delete +// +// Returns an error if deletion fails +func (s *messageService) DeleteMessage(ctx context.Context, sessionID string, messageID string) error { + logger.Info(ctx, "Start deleting message") + logger.Infof(ctx, "Deleting message, session ID: %s, message ID: %s", sessionID, messageID) + + // Verify the session exists before deleting the message + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Checking if session exists, tenant ID: %d", tenantID) + _, err := s.sessionRepo.Get(ctx, tenantID, sessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session: %v", err) + return err + } + + // Delete the message from the repository + logger.Info(ctx, "Session exists, deleting message") + err = s.messageRepo.DeleteMessage(ctx, sessionID, messageID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + "message_id": messageID, + }) + return err + } + + logger.Info(ctx, "Message deleted successfully") + return nil +} diff --git a/internal/application/service/metric/bleu.go b/internal/application/service/metric/bleu.go new file mode 100644 index 0000000..c1f6dec --- /dev/null +++ b/internal/application/service/metric/bleu.go @@ -0,0 +1,165 @@ +package metric + +// references: https://github.com/waygo/bleu + +// Package bleu implements the BLEU method, which is used to evaluate +// the quality of machine translation. [1] +// +// The code in this package was largely ported from the corresponding package +// in Python NLTK. [2] +// +// [1] Papineni, Kishore, et al. "BLEU: a method for automatic evaluation of +// machine translation." Proceedings of the 40th annual meeting on +// association for computational linguistics. Association for Computational +// Linguistics, 2002. +// +// [2] http://www.nltk.org/_modules/nltk/align/bleu.html + +import ( + "encoding/json" + "log" + "math" + "strings" + + "github.com/Tencent/WeKnora/internal/types" +) + +type BLEUMetric struct { + smoothing bool + weights BLEUWeight +} + +func NewBLEUMetric(smoothing bool, weights BLEUWeight) *BLEUMetric { + return &BLEUMetric{smoothing: smoothing, weights: weights} +} + +type Sentence []string + +type BLEUWeight []float64 + +var ( + BLEU1Gram BLEUWeight = []float64{1.0, 0.0, 0.0, 0.0} + BLEU2Gram BLEUWeight = []float64{0.5, 0.5, 0.0, 0.0} + BLEU3Gram BLEUWeight = []float64{0.33, 0.33, 0.33, 0.0} + BLEU4Gram BLEUWeight = []float64{0.25, 0.25, 0.25, 0.25} +) + +func (b *BLEUMetric) Compute(metricInput *types.MetricInput) float64 { + candidate := splitIntoWords(splitSentences(metricInput.GeneratedTexts)) + references := []Sentence{splitIntoWords(splitSentences(metricInput.GeneratedGT))} + + for i := range candidate { + candidate[i] = strings.ToLower(candidate[i]) + } + + for i := range references { + for u := range references[i] { + references[i][u] = strings.ToLower(references[i][u]) + } + } + + ps := make([]float64, len(b.weights)) + for i := range b.weights { + ps[i] = b.modifiedPrecision(candidate, references, i+1) + } + + s := 0.0 + overlap := 0 + for i := range b.weights { + w := b.weights[i] + pn := ps[i] + if pn > 0.0 { + overlap++ + s += w * math.Log(pn) + } + } + + if overlap == 0 { + return 0 + } + + bp := b.brevityPenalty(candidate, references) + return bp * math.Exp(s) +} + +type phrase []string + +func (p phrase) String() string { + b, err := json.Marshal(p) + if err != nil { + log.Fatal("encode error:", err) + } + return string(b) +} + +func (b *BLEUMetric) getNphrase(s Sentence, n int) []phrase { + nphrase := []phrase{} + for i := 0; i < len(s)-n+1; i++ { + nphrase = append(nphrase, phrase(s[i:i+n])) + } + return nphrase +} + +func (b *BLEUMetric) countNphrase(nphrase []phrase) map[string]int { + counts := map[string]int{} + for _, gram := range nphrase { + counts[gram.String()]++ + } + return counts +} + +func (b *BLEUMetric) modifiedPrecision(candidate Sentence, references []Sentence, n int) float64 { + nphrase := b.getNphrase(candidate, n) + if len(nphrase) == 0 { + return 0.0 + } + + counts := b.countNphrase(nphrase) + + if len(counts) == 0 { + return 0.0 + } + + maxCounts := map[string]int{} + for i := range references { + referenceCounts := b.countNphrase(b.getNphrase(references[i], n)) + for ngram := range counts { + if v, ok := maxCounts[ngram]; !ok { + maxCounts[ngram] = referenceCounts[ngram] + } else if v < referenceCounts[ngram] { + maxCounts[ngram] = referenceCounts[ngram] + } + } + } + + clippedCounts := map[string]int{} + for ngram, count := range counts { + clippedCounts[ngram] = min(count, maxCounts[ngram]) + } + + smoothingFactor := 0.0 + if b.smoothing { + smoothingFactor = 1.0 + } + return (float64(sum(clippedCounts)) + smoothingFactor) / (float64(sum(counts)) + smoothingFactor) +} + +func (b *BLEUMetric) brevityPenalty(candidate Sentence, references []Sentence) float64 { + c := len(candidate) + refLens := []int{} + for i := range references { + refLens = append(refLens, len(references[i])) + } + minDiffInd, minDiff := 0, -1 + for i := range refLens { + if minDiff == -1 || abs(refLens[i]-c) < minDiff { + minDiffInd = i + minDiff = abs(refLens[i] - c) + } + } + r := refLens[minDiffInd] + if c > r { + return 1 + } + return math.Exp(float64(1 - float64(r)/float64(c))) +} diff --git a/internal/application/service/metric/common.go b/internal/application/service/metric/common.go new file mode 100644 index 0000000..8f48a74 --- /dev/null +++ b/internal/application/service/metric/common.go @@ -0,0 +1,133 @@ +package metric + +import ( + "regexp" + "strings" + + "github.com/Tencent/WeKnora/internal/types" +) + +func sum(m map[string]int) int { + s := 0 + for _, v := range m { + s += v + } + return s +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func abs(a int) int { + if a < 0 { + return -a + } + return a +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func splitSentences(text string) []string { + // 编译正则表达式（匹配中文句号或英文句号） + re := regexp.MustCompile(`([。.])`) + + // 分割文本并保留分隔符用于定位 + split := re.Split(text, -1) + + var sentences []string + current := strings.Builder{} + + for i, s := range split { + // 交替获取文本段和分隔符（奇数为分隔符） + if i%2 == 0 { + current.WriteString(s) + } else { + // 当遇到分隔符时，完成当前句子 + if current.Len() > 0 { + sentence := strings.TrimSpace(current.String()) + if sentence != "" { + sentences = append(sentences, sentence) + } + current.Reset() + } + } + } + + // 处理最后一个无分隔符的文本段 + if remaining := strings.TrimSpace(current.String()); remaining != "" { + sentences = append(sentences, remaining) + } + + return sentences +} + +func splitIntoWords(sentences []string) []string { + // 正则匹配中英文段落（中文块、英文块、其他字符） + re := regexp.MustCompile(`([\p{Han}]+)|([a-zA-Z0-9_.,!?]+)|(\p{P})`) + + var tokens []string + for _, text := range sentences { + matches := re.FindAllStringSubmatch(text, -1) + + for _, groups := range matches { + chineseBlock := groups[1] + englishBlock := groups[2] + punctuation := groups[3] + + switch { + case chineseBlock != "": // 处理中文部分 + words := types.Jieba.Cut(chineseBlock, true) + tokens = append(tokens, words...) + case englishBlock != "": // 处理英文部分 + engTokens := strings.Fields(englishBlock) + tokens = append(tokens, engTokens...) + case punctuation != "": // 保留标点符号 + tokens = append(tokens, punctuation) + } + } + } + return tokens +} + +func ToSet[T comparable](li []T) map[T]struct{} { + res := make(map[T]struct{}, len(li)) + for _, v := range li { + res[v] = struct{}{} + } + return res +} + +func SliceMap[T any, Y any](li []T, fn func(T) Y) []Y { + res := make([]Y, len(li)) + for i, v := range li { + res[i] = fn(v) + } + return res +} + +func Hit[T comparable](li []T, set map[T]struct{}) int { + count := 0 + for _, v := range li { + if _, exist := set[v]; exist { + count++ + } + } + return count +} + +func Fold[T any, Y any](slice []T, initial Y, f func(Y, T) Y) Y { + accumulator := initial + for _, item := range slice { + accumulator = f(accumulator, item) + } + return accumulator +} diff --git a/internal/application/service/metric/map.go b/internal/application/service/metric/map.go new file mode 100644 index 0000000..10f463b --- /dev/null +++ b/internal/application/service/metric/map.go @@ -0,0 +1,70 @@ +package metric + +import ( + "github.com/Tencent/WeKnora/internal/types" +) + +// MAPMetric calculates Mean Average Precision for retrieval evaluation +type MAPMetric struct{} + +// NewMAPMetric creates a new MAPMetric instance +func NewMAPMetric() *MAPMetric { + return &MAPMetric{} +} + +// Compute calculates the Mean Average Precision score +func (m *MAPMetric) Compute(metricInput *types.MetricInput) float64 { + // Convert ground truth to sets for efficient lookup + gts := metricInput.RetrievalGT + ids := metricInput.RetrievalIDs + + // Create sets of relevant document IDs for each query + gtSets := make([]map[int]struct{}, len(gts)) + for i, gt := range gts { + gtSets[i] = make(map[int]struct{}) + for _, docID := range gt { + gtSets[i][docID] = struct{}{} + } + } + + var apSum float64 // Sum of average precision for all queries + + // Calculate average precision for each query + for _, gtSet := range gtSets { + // Mark which predicted documents are relevant + predHits := make([]bool, len(ids)) + for i, predID := range ids { + if _, ok := gtSet[predID]; ok { + predHits[i] = true + } else { + predHits[i] = false + } + } + + var ( + ap float64 // Average precision for current query + hitCount int // Number of relevant documents found + ) + + // Calculate precision at each rank position + for k := 0; k < len(predHits); k++ { + if predHits[k] { + hitCount++ + // Precision at k: relevant docs found up to k / k + ap += float64(hitCount) / float64(k+1) + } + } + // Normalize by number of relevant documents + if hitCount > 0 { + ap /= float64(hitCount) + } + apSum += ap + } + + // Handle case with no ground truth + if len(gtSets) == 0 { + return 0 + } + // Return mean of average precision across all queries + return apSum / float64(len(gtSets)) +} diff --git a/internal/application/service/metric/map_test.go b/internal/application/service/metric/map_test.go new file mode 100644 index 0000000..7bcaf0e --- /dev/null +++ b/internal/application/service/metric/map_test.go @@ -0,0 +1,85 @@ +package metric + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestMAPMetric_Compute(t *testing.T) { + tests := []struct { + name string + input *types.MetricInput + expected float64 + }{ + { + name: "total match", + input: &types.MetricInput{ + RetrievalGT: [][]int{{2, 4, 6}}, + RetrievalIDs: []int{2, 4, 6}, + }, + expected: 1.0, + }, + { + name: "no match", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2}}, + RetrievalIDs: []int{3, 4}, + }, + expected: 0.0, + }, + { + name: "partial match", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}}, + RetrievalIDs: []int{2, 5, 1, 3}, + }, + // AP = (1/1 + 2/3 + 3/4)/3 ≈ 0.80555555 + expected: 0.8055555555555555, + }, + { + name: "empty ground truth", + input: &types.MetricInput{ + RetrievalGT: [][]int{}, + RetrievalIDs: []int{1, 2}, + }, + expected: 0.0, + }, + { + name: "multiple queries", + input: &types.MetricInput{ + RetrievalGT: [][]int{ + {1, 2}, + {3, 4}, + }, + RetrievalIDs: []int{1, 3, 2, 4}, + }, + // Query1 AP: (1/1 + 2/3)/2 ≈ 0.8333 + // Query2 AP: (1/2 + 2/4)/2 = 0.5 + // MAP: (0.8333 + 0.5)/2 ≈ 0.6667 + expected: 0.6666666666666666, + }, + } + + metric := NewMAPMetric() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := metric.Compute(tt.input) + if !almostEqual(got, tt.expected, 1e-6) { + t.Errorf("Compute() = %v, want %v", got, tt.expected) + } + }) + } +} + +// Helper function to compare floating point numbers with tolerance +func almostEqual(a, b, tolerance float64) bool { + if a == b { + return true + } + diff := a - b + if diff < 0 { + diff = -diff + } + return diff < tolerance +} diff --git a/internal/application/service/metric/mrr.go b/internal/application/service/metric/mrr.go new file mode 100644 index 0000000..4d89712 --- /dev/null +++ b/internal/application/service/metric/mrr.go @@ -0,0 +1,48 @@ +package metric + +import ( + "github.com/Tencent/WeKnora/internal/types" +) + +// MRRMetric calculates Mean Reciprocal Rank for retrieval evaluation +type MRRMetric struct{} + +// NewMRRMetric creates a new MRRMetric instance +func NewMRRMetric() *MRRMetric { + return &MRRMetric{} +} + +// Compute calculates the Mean Reciprocal Rank score +func (m *MRRMetric) Compute(metricInput *types.MetricInput) float64 { + // Get ground truth and predicted IDs + gts := metricInput.RetrievalGT + ids := metricInput.RetrievalIDs + + // Convert ground truth to sets for efficient lookup + gtSets := make([]map[int]struct{}, len(gts)) + for i, gt := range gts { + gtSets[i] = make(map[int]struct{}) + for _, docID := range gt { + gtSets[i][docID] = struct{}{} + } + } + + var sumRR float64 // Sum of reciprocal ranks + // Calculate reciprocal rank for each query + for _, gtSet := range gtSets { + // Find first relevant document in results + for i, predID := range ids { + if _, ok := gtSet[predID]; ok { + // Reciprocal rank is 1/position (1-based) + sumRR += 1.0 / float64(i+1) + break // Only consider first relevant document + } + } + } + // Handle case with no ground truth + if len(gtSets) == 0 { + return 0 + } + // Return mean of reciprocal ranks + return sumRR / float64(len(gtSets)) +} diff --git a/internal/application/service/metric/mrr_test.go b/internal/application/service/metric/mrr_test.go new file mode 100644 index 0000000..4d4531d --- /dev/null +++ b/internal/application/service/metric/mrr_test.go @@ -0,0 +1,72 @@ +package metric + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestMRRMetric_Compute(t *testing.T) { + tests := []struct { + name string + input *types.MetricInput + expected float64 + }{ + { + name: "perfect match - first position", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2}}, + RetrievalIDs: []int{1, 2, 3}, + }, + // RR = 1/1 = 1.0 + expected: 1.0, + }, + { + name: "match at second position", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2}}, + RetrievalIDs: []int{3, 1, 2}, + }, + // RR = 1/2 = 0.5 + expected: 0.5, + }, + { + name: "no match", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2}}, + RetrievalIDs: []int{3, 4}, + }, + expected: 0.0, + }, + { + name: "multiple queries", + input: &types.MetricInput{ + RetrievalGT: [][]int{ + {1, 2}, // RR = 1/1 = 1.0 + {3, 4}, // RR = 1/2 = 0.5 + }, + RetrievalIDs: []int{1, 3, 2, 4}, + }, + // MRR = (1.0 + 0.5)/2 = 0.75 + expected: 0.75, + }, + { + name: "empty ground truth", + input: &types.MetricInput{ + RetrievalGT: [][]int{}, + RetrievalIDs: []int{1, 2}, + }, + expected: 0.0, + }, + } + + metric := NewMRRMetric() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := metric.Compute(tt.input) + if !almostEqual(got, tt.expected, 1e-6) { + t.Errorf("Compute() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/internal/application/service/metric/ndcg.go b/internal/application/service/metric/ndcg.go new file mode 100644 index 0000000..07fd2e1 --- /dev/null +++ b/internal/application/service/metric/ndcg.go @@ -0,0 +1,78 @@ +package metric + +import ( + "math" + + "github.com/Tencent/WeKnora/internal/types" +) + +// NDCGMetric calculates Normalized Discounted Cumulative Gain +type NDCGMetric struct { + k int // Top k results to consider +} + +// NewNDCGMetric creates a new NDCGMetric instance with given k value +func NewNDCGMetric(k int) *NDCGMetric { + return &NDCGMetric{k: k} +} + +// Compute calculates the NDCG score +func (n *NDCGMetric) Compute(metricInput *types.MetricInput) float64 { + gts := metricInput.RetrievalGT + ids := metricInput.RetrievalIDs + + // Limit results to top k + if len(ids) > n.k { + ids = ids[:n.k] + } + + // Create set of relevant documents and count total relevant + gtSets := make(map[int]struct{}, len(gts)) + countGt := 0 + for _, gt := range gts { + countGt += len(gt) + for _, g := range gt { + gtSets[g] = struct{}{} + } + } + + // Assign relevance scores (1 for relevant, 0 otherwise) + relevanceScores := make(map[int]int) + for _, docID := range ids { + if _, exist := gtSets[docID]; exist { + relevanceScores[docID] = 1 + } else { + relevanceScores[docID] = 0 + } + } + + // Calculate DCG (Discounted Cumulative Gain) + var dcg float64 + for i, docID := range ids { + dcg += (math.Pow(2, float64(relevanceScores[docID])) - 1) / math.Log2(float64(i+2)) + } + + // Create ideal ranking (all relevant docs first) + idealLen := min(countGt, len(ids)) + idealPred := make([]int, len(ids)) + for i := 0; i < len(ids); i++ { + if i < idealLen { + idealPred[i] = 1 + } else { + idealPred[i] = 0 + } + } + + // Calculate IDCG (Ideal DCG) + var idcg float64 + for i, relevance := range idealPred { + idcg += float64(relevance) / math.Log2(float64(i+2)) + } + + // Handle division by zero case + if idcg == 0 { + return 0 + } + // NDCG = DCG / IDCG + return dcg / idcg +} diff --git a/internal/application/service/metric/precision.go b/internal/application/service/metric/precision.go new file mode 100644 index 0000000..9121120 --- /dev/null +++ b/internal/application/service/metric/precision.go @@ -0,0 +1,33 @@ +package metric + +import ( + "github.com/Tencent/WeKnora/internal/types" +) + +// PrecisionMetric calculates precision for retrieval evaluation +type PrecisionMetric struct{} + +// NewPrecisionMetric creates a new PrecisionMetric instance +func NewPrecisionMetric() *PrecisionMetric { + return &PrecisionMetric{} +} + +// Compute calculates the precision score +func (r *PrecisionMetric) Compute(metricInput *types.MetricInput) float64 { + // Get ground truth and predicted IDs + gts := metricInput.RetrievalGT + ids := metricInput.RetrievalIDs + + // Convert ground truth to sets for efficient lookup + gtSets := SliceMap(gts, ToSet) + // Count total hits across all queries + ahit := Fold(gtSets, 0, func(a int, b map[int]struct{}) int { return a + Hit(ids, b) }) + + // Handle case with no ground truth + if len(gts) == 0 { + return 0.0 + } + + // Precision = total hits / number of queries + return float64(ahit) / float64(len(gts)) +} diff --git a/internal/application/service/metric/precision_test.go b/internal/application/service/metric/precision_test.go new file mode 100644 index 0000000..4b947f1 --- /dev/null +++ b/internal/application/service/metric/precision_test.go @@ -0,0 +1,67 @@ +package metric + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestPrecisionMetric_Compute(t *testing.T) { + tests := []struct { + name string + input *types.MetricInput + expected float64 + }{ + { + name: "perfect match", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 3, 5}}, + RetrievalIDs: []int{1, 3, 5}, + }, + expected: 1.0, + }, + { + name: "half match", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}}, + RetrievalIDs: []int{1, 4, 2}, + }, + expected: 0.6666666666666666, + }, + { + name: "no match", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}}, + RetrievalIDs: []int{4, 5, 6}, + }, + expected: 0.0, + }, + { + name: "empty retrieval", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}}, + RetrievalIDs: []int{}, + }, + expected: 0.0, + }, + { + name: "multiple ground truths", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2}, {3, 4}}, + RetrievalIDs: []int{1, 3, 5}, + }, + expected: 0.3333333333333333, + }, + } + + pm := NewPrecisionMetric() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := pm.Compute(tt.input) + if got != tt.expected { + t.Errorf("Compute() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/internal/application/service/metric/recall.go b/internal/application/service/metric/recall.go new file mode 100644 index 0000000..807ad00 --- /dev/null +++ b/internal/application/service/metric/recall.go @@ -0,0 +1,33 @@ +package metric + +import ( + "github.com/Tencent/WeKnora/internal/types" +) + +// RecallMetric calculates recall for retrieval evaluation +type RecallMetric struct{} + +// NewRecallMetric creates a new RecallMetric instance +func NewRecallMetric() *RecallMetric { + return &RecallMetric{} +} + +// Compute calculates the recall score +func (r *RecallMetric) Compute(metricInput *types.MetricInput) float64 { + // Get ground truth and predicted IDs + gts := metricInput.RetrievalGT + ids := metricInput.RetrievalIDs + + // Convert ground truth to sets for efficient lookup + gtSets := SliceMap(gts, ToSet) + // Count total hits across all relevant documents + ahit := Fold(gtSets, 0, func(a int, b map[int]struct{}) int { return a + Hit(ids, b) }) + + // Handle case with no ground truth + if len(gtSets) == 0 { + return 0.0 + } + + // Recall = total hits / total relevant documents + return float64(ahit) / float64(len(gtSets)) +} diff --git a/internal/application/service/metric/recall_test.go b/internal/application/service/metric/recall_test.go new file mode 100644 index 0000000..e83a54d --- /dev/null +++ b/internal/application/service/metric/recall_test.go @@ -0,0 +1,70 @@ +package metric + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestRecallMetric_Compute(t *testing.T) { + tests := []struct { + name string + input *types.MetricInput + expected float64 + }{ + { + name: "perfect recall - all ground truth retrieved", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}}, + RetrievalIDs: []int{1, 2, 3, 4}, + }, + expected: 1.0, + }, + { + name: "partial recall - some ground truth retrieved", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}, {4, 5}}, + RetrievalIDs: []int{1, 4, 6}, + }, + // 命中2个ground truth集合中的元素(a和d) + expected: 0.41666666666666663, // (1/3 + 1/2) / 2 = 0.41666 (每个ground truth集合只要命中一个就算召回) + + }, + { + name: "no recall - no ground truth retrieved", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}}, + RetrievalIDs: []int{4, 5, 6}, + }, + expected: 0.0, + }, + { + name: "empty retrieval list", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2, 3}}, + RetrievalIDs: []int{}, + }, + expected: 0.0, + }, + { + name: "multiple ground truth sets", + input: &types.MetricInput{ + RetrievalGT: [][]int{{1, 2}, {3, 4}, {5, 6}}, + RetrievalIDs: []int{1, 3, 7}, + }, + // 命中了前两个ground truth集合(a和c) + expected: 0.3333333333333333, // 1/3≈0.333... + }, + } + + rm := NewRecallMetric() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := rm.Compute(tt.input) + if got != tt.expected { + t.Errorf("Compute() = %v, want %v", got, tt.expected) + } + }) + } +} diff --git a/internal/application/service/metric/rouge.go b/internal/application/service/metric/rouge.go new file mode 100644 index 0000000..22ebe68 --- /dev/null +++ b/internal/application/service/metric/rouge.go @@ -0,0 +1,72 @@ +package metric + +import "github.com/Tencent/WeKnora/internal/types" + +// reference: https://github.com/dd-Rebecca/rouge + +// RougeMetric implements ROUGE (Recall-Oriented Understudy for Gisting Evaluation) metrics +// for evaluating text summarization quality by comparing generated text to reference text +type RougeMetric struct { + exclusive bool // Whether to use exclusive matching mode + metric string // ROUGE metric type (e.g. "rouge-1", "rouge-l") + stats string // Statistic to return (e.g. "f", "p", "r") +} + +// AvailableMetrics defines all supported ROUGE variants and their calculation functions +var AvailableMetrics = map[string]func([]string, []string, bool) map[string]float64{ + "rouge-1": func(hyp, ref []string, exclusive bool) map[string]float64 { + return rougeN(hyp, ref, 1, false, exclusive) // Unigram-based ROUGE + }, + "rouge-2": func(hyp, ref []string, exclusive bool) map[string]float64 { + return rougeN(hyp, ref, 2, false, exclusive) // Bigram-based ROUGE + }, + "rouge-3": func(hyp, ref []string, exclusive bool) map[string]float64 { + return rougeN(hyp, ref, 3, false, exclusive) // Trigram-based ROUGE + }, + "rouge-4": func(hyp, ref []string, exclusive bool) map[string]float64 { + return rougeN(hyp, ref, 4, false, exclusive) // 4-gram based ROUGE + }, + "rouge-5": func(hyp, ref []string, exclusive bool) map[string]float64 { + return rougeN(hyp, ref, 5, false, exclusive) // 5-gram based ROUGE + }, + "rouge-l": func(hyp, ref []string, exclusive bool) map[string]float64 { + return rougeLSummaryLevel(hyp, ref, false, exclusive) // Longest common subsequence based ROUGE + }, +} + +// NewRougeMetric creates a new ROUGE metric calculator +func NewRougeMetric(exclusive bool, metrics, stats string) *RougeMetric { + r := &RougeMetric{ + exclusive: exclusive, + metric: metrics, + stats: stats, + } + return r +} + +// Compute calculates the ROUGE score between generated text and reference text +func (r *RougeMetric) Compute(metricInput *types.MetricInput) float64 { + hyps := []string{metricInput.GeneratedTexts} // Generated/hypothesis text + refs := []string{metricInput.GeneratedGT} // Reference/ground truth text + + scores := 0.0 + count := 0 + + // Calculate scores for each hypothesis-reference pair + for i := 0; i < len(hyps); i++ { + hyp := splitSentences(hyps[i]) // Split into sentences + ref := splitSentences(refs[i]) + + // Get appropriate ROUGE calculation function + fn := AvailableMetrics[r.metric] + sc := fn(hyp, ref, r.exclusive) + scores += sc[r.stats] // Accumulate specified statistic (f1/precision/recall) + + count++ + } + + if count == 0 { + return 0 // Avoid division by zero + } + return scores / float64(count) // Return average score +} diff --git a/internal/application/service/metric/rouge_score.go b/internal/application/service/metric/rouge_score.go new file mode 100644 index 0000000..dd51e14 --- /dev/null +++ b/internal/application/service/metric/rouge_score.go @@ -0,0 +1,252 @@ +package metric + +/* +# -*- coding: utf-8 -*- +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ROUGE Metric Implementation + +This is a very slightly version of: +https://github.com/pltrdy/seq2seq/blob/master/seq2seq/metrics/rouge.py + +--- + +ROUGe metric implementation. + +This is a modified and slightly extended verison of +https://github.com/miso-belica/sumy/blob/dev/sumy/evaluation/rouge.py. +*/ + +import ( + "strings" +) + +type Ngrams struct { + ngrams map[string]int + exclusive bool +} + +func NewNgrams(exclusive bool) *Ngrams { + return &Ngrams{ngrams: make(map[string]int), exclusive: exclusive} +} + +func (n *Ngrams) Add(o string) { + if n.exclusive { + n.ngrams[o] = 1 + } else { + n.ngrams[o]++ + } +} + +func (n *Ngrams) Len() int { + return len(n.ngrams) +} + +func (n *Ngrams) Intersection(o *Ngrams) *Ngrams { + intersection := NewNgrams(n.exclusive) + for k := range n.ngrams { + if _, ok := o.ngrams[k]; ok { + intersection.Add(k) + } + } + return intersection +} + +func (n *Ngrams) BatchAdd(o []string) { + for _, v := range o { + n.Add(v) + } +} + +func (n *Ngrams) Union(others ...*Ngrams) *Ngrams { + union := NewNgrams(n.exclusive) + for k := range n.ngrams { + union.Add(k) + } + for _, other := range others { + for k := range other.ngrams { + union.Add(k) + } + } + return union +} + +func getNgrams(n int, text []string, exclusive bool) *Ngrams { + ngramSet := NewNgrams(exclusive) + for i := 0; i <= len(text)-n; i++ { + ngramSet.Add(strings.Join(text[i:i+n], " ")) + } + return ngramSet +} + +func getWordNgrams(n int, sentences []string, exclusive bool) *Ngrams { + words := splitIntoWords(sentences) + return getNgrams(n, words, exclusive) +} + +func lenLcs(x, y []string) int { + table := lcs(x, y) + return table[len(x)][len(y)] +} + +func lcs(x, y []string) [][]int { + n, m := len(x), len(y) + table := make([][]int, n+1) + for i := range table { + table[i] = make([]int, m+1) + } + for i := 1; i <= n; i++ { + for j := 1; j <= m; j++ { + if x[i-1] == y[j-1] { + table[i][j] = table[i-1][j-1] + 1 + } else { + table[i][j] = max(table[i-1][j], table[i][j-1]) + } + } + } + return table +} + +func reconLcs(x, y []string, exclusive bool) *Ngrams { + i, j := len(x), len(y) + table := lcs(x, y) + + var reconFunc func(int, int) []string + reconFunc = func(i, j int) []string { + if i == 0 || j == 0 { + return []string{} + } else if x[i-1] == y[j-1] { + return append(reconFunc(i-1, j-1), x[i-1]) + } else if table[i-1][j] > table[i][j-1] { + return reconFunc(i-1, j) + } else { + return reconFunc(i, j-1) + } + } + + reconList := reconFunc(i, j) + ngramList := NewNgrams(exclusive) + for _, word := range reconList { + ngramList.Add(word) + } + return ngramList +} + +func rougeN(evaluatedSentences, referenceSentences []string, n int, rawResults, exclusive bool) map[string]float64 { + evaluatedNgrams := getWordNgrams(n, evaluatedSentences, exclusive) + referenceNgrams := getWordNgrams(n, referenceSentences, exclusive) + referenceCount := referenceNgrams.Len() + evaluatedCount := evaluatedNgrams.Len() + + overlappingNgrams := evaluatedNgrams.Intersection(referenceNgrams) + overlappingCount := overlappingNgrams.Len() + + results := make(map[string]float64) + if rawResults { + results["hyp"] = float64(evaluatedCount) + results["ref"] = float64(referenceCount) + results["overlap"] = float64(overlappingCount) + return results + } else { + return calculateRougeN(evaluatedCount, referenceCount, overlappingCount) + } +} + +func calculateRougeN(evaluatedCount, referenceCount, overlappingCount int) map[string]float64 { + results := make(map[string]float64) + if evaluatedCount == 0 { + results["p"] = 0.0 + } else { + results["p"] = float64(overlappingCount) / float64(evaluatedCount) + } + + if referenceCount == 0 { + results["r"] = 0.0 + } else { + results["r"] = float64(overlappingCount) / float64(referenceCount) + } + + results["f"] = 2.0 * ((results["p"] * results["r"]) / (results["p"] + results["r"] + 1e-8)) + return results +} + +func unionLcs(evaluatedSentences []string, referenceSentence string, prevUnion *Ngrams, exclusive bool) (int, *Ngrams) { + if prevUnion == nil { + prevUnion = NewNgrams(exclusive) + } + + lcsUnion := prevUnion + prevCount := len(prevUnion.ngrams) + referenceWords := splitIntoWords([]string{referenceSentence}) + + combinedLcsLength := 0 + for _, evalS := range evaluatedSentences { + evaluatedWords := splitIntoWords([]string{evalS}) + lcs := reconLcs(referenceWords, evaluatedWords, exclusive) + combinedLcsLength += lcs.Len() + lcsUnion = lcsUnion.Union(lcs) + } + + newLcsCount := lcsUnion.Len() - prevCount + return newLcsCount, lcsUnion +} + +func rougeLSummaryLevel(evaluatedSentences, referenceSentences []string, rawResults, exclusive bool) map[string]float64 { + referenceNgrams := NewNgrams(exclusive) + referenceNgrams.BatchAdd(splitIntoWords(referenceSentences)) + m := referenceNgrams.Len() + + evaluatedNgrams := NewNgrams(exclusive) + evaluatedNgrams.BatchAdd(splitIntoWords(evaluatedSentences)) + n := evaluatedNgrams.Len() + + unionLcsSumAcrossAllReferences := 0 + union := NewNgrams(exclusive) + for _, refS := range referenceSentences { + lcsCount, newUnion := unionLcs(evaluatedSentences, refS, union, exclusive) + union = newUnion + unionLcsSumAcrossAllReferences += lcsCount + } + + llcs := unionLcsSumAcrossAllReferences + + var rLcs float64 + if m == 0 { + rLcs = 0.0 + } else { + rLcs = float64(llcs) / float64(m) + } + var pLcs float64 + if n == 0 { + pLcs = 0.0 + } else { + pLcs = float64(llcs) / float64(n) + } + + fLcs := 2.0 * ((pLcs * rLcs) / (pLcs + rLcs + 1e-8)) + + results := make(map[string]float64) + if rawResults { + results["hyp"] = float64(n) + results["ref"] = float64(m) + results["overlap"] = float64(llcs) + return results + } else { + results["f"] = fLcs + results["p"] = pLcs + results["r"] = rLcs + return results + } +} diff --git a/internal/application/service/metric_hook.go b/internal/application/service/metric_hook.go new file mode 100644 index 0000000..ec3e80b --- /dev/null +++ b/internal/application/service/metric_hook.go @@ -0,0 +1,167 @@ +package service + +import ( + "context" + "sync" + + "github.com/Tencent/WeKnora/internal/application/service/metric" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// MetricList stores and aggregates metric results +type MetricList struct { + results []*types.MetricResult +} + +// metricCalculators defines all metrics to be calculated +var metricCalculators = []struct { + calc interfaces.Metrics // Metric calculator implementation + getField func(*types.MetricResult) *float64 // Field accessor for result +}{ + // Retrieval Metrics + {metric.NewPrecisionMetric(), func(r *types.MetricResult) *float64 { return &r.RetrievalMetrics.Precision }}, + {metric.NewRecallMetric(), func(r *types.MetricResult) *float64 { return &r.RetrievalMetrics.Recall }}, + {metric.NewNDCGMetric(3), func(r *types.MetricResult) *float64 { return &r.RetrievalMetrics.NDCG3 }}, + {metric.NewNDCGMetric(10), func(r *types.MetricResult) *float64 { return &r.RetrievalMetrics.NDCG10 }}, + {metric.NewMRRMetric(), func(r *types.MetricResult) *float64 { return &r.RetrievalMetrics.MRR }}, + {metric.NewMAPMetric(), func(r *types.MetricResult) *float64 { return &r.RetrievalMetrics.MAP }}, + + // Generation Metrics + {metric.NewBLEUMetric(true, metric.BLEU1Gram), func(r *types.MetricResult) *float64 { + return &r.GenerationMetrics.BLEU1 + }}, + {metric.NewBLEUMetric(true, metric.BLEU2Gram), func(r *types.MetricResult) *float64 { + return &r.GenerationMetrics.BLEU2 + }}, + {metric.NewBLEUMetric(true, metric.BLEU4Gram), func(r *types.MetricResult) *float64 { + return &r.GenerationMetrics.BLEU4 + }}, + {metric.NewRougeMetric(true, "rouge-1", "f"), func(r *types.MetricResult) *float64 { + return &r.GenerationMetrics.ROUGE1 + }}, + {metric.NewRougeMetric(true, "rouge-2", "f"), func(r *types.MetricResult) *float64 { + return &r.GenerationMetrics.ROUGE2 + }}, + {metric.NewRougeMetric(true, "rouge-l", "f"), func(r *types.MetricResult) *float64 { + return &r.GenerationMetrics.ROUGEL + }}, +} + +// Append calculates and stores metrics for given input +func (m *MetricList) Append(metricInput *types.MetricInput) { + result := &types.MetricResult{} + // Calculate all configured metrics + for _, c := range metricCalculators { + score := c.calc.Compute(metricInput) + *c.getField(result) = score + } + logger.Infof(context.Background(), "metric: %v", result) + m.results = append(m.results, result) +} + +// Avg calculates average of all stored metric results +func (m *MetricList) Avg() *types.MetricResult { + if len(m.results) == 0 { + return &types.MetricResult{} + } + + avgResult := &types.MetricResult{} + count := float64(len(m.results)) + + // Calculate average for each metric + for _, config := range metricCalculators { + sum := 0.0 + for _, r := range m.results { + sum += *config.getField(r) + } + *config.getField(avgResult) = sum / count + } + return avgResult +} + +// HookMetric tracks evaluation metrics for QA pairs +type HookMetric struct { + task *types.EvaluationTask + qaPairMetricList []*qaPairMetric // Per-QA pair metrics + metricResults *MetricList // Aggregated results + mu *sync.RWMutex // Thread safety +} + +// qaPairMetric stores metrics for a single QA pair +type qaPairMetric struct { + qaPair *types.QAPair + searchResult []*types.SearchResult + rerankResult []*types.SearchResult + chatResponse *types.ChatResponse +} + +// NewHookMetric creates a new HookMetric with given capacity +func NewHookMetric(capacity int) *HookMetric { + return &HookMetric{ + metricResults: &MetricList{}, + qaPairMetricList: make([]*qaPairMetric, capacity), + mu: &sync.RWMutex{}, + } +} + +// recordInit initializes metric tracking for a QA pair +func (h *HookMetric) recordInit(index int) { + h.qaPairMetricList[index] = &qaPairMetric{} +} + +// recordQaPair records the QA pair data +func (h *HookMetric) recordQaPair(index int, qaPair *types.QAPair) { + h.qaPairMetricList[index].qaPair = qaPair +} + +// recordSearchResult records search results +func (h *HookMetric) recordSearchResult(index int, searchResult []*types.SearchResult) { + h.qaPairMetricList[index].searchResult = searchResult +} + +// recordRerankResult records reranked results +func (h *HookMetric) recordRerankResult(index int, rerankResult []*types.SearchResult) { + h.qaPairMetricList[index].rerankResult = rerankResult +} + +// recordChatResponse records the generated chat response +func (h *HookMetric) recordChatResponse(index int, chatResponse *types.ChatResponse) { + h.qaPairMetricList[index].chatResponse = chatResponse +} + +// recordFinish finalizes metrics for a QA pair +func (h *HookMetric) recordFinish(index int) { + // Prepare retrieval IDs from rerank results + retrievalIDs := make([]int, len(h.qaPairMetricList[index].rerankResult)) + for i, r := range h.qaPairMetricList[index].rerankResult { + retrievalIDs[i] = r.ChunkIndex + } + + // Get generated text if available + generatedTexts := "" + if h.qaPairMetricList[index].chatResponse != nil { + generatedTexts = h.qaPairMetricList[index].chatResponse.Content + } + + // Prepare metric input data + metricInput := &types.MetricInput{ + RetrievalGT: [][]int{h.qaPairMetricList[index].qaPair.PIDs}, + RetrievalIDs: retrievalIDs, + GeneratedTexts: generatedTexts, + GeneratedGT: h.qaPairMetricList[index].qaPair.Answer, + } + + // Thread-safe append of metrics + h.mu.Lock() + defer h.mu.Unlock() + h.metricResults.Append(metricInput) +} + +// MetricResult returns the averaged metric results +func (h *HookMetric) MetricResult() *types.MetricResult { + h.mu.RLock() + defer h.mu.RUnlock() + return h.metricResults.Avg() +} diff --git a/internal/application/service/model.go b/internal/application/service/model.go new file mode 100644 index 0000000..48cb789 --- /dev/null +++ b/internal/application/service/model.go @@ -0,0 +1,327 @@ +package service + +import ( + "context" + "errors" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/models/utils/ollama" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// ErrModelNotFound is returned when a model cannot be found in the repository +var ErrModelNotFound = errors.New("model not found") + +// modelService implements the model service interface +type modelService struct { + repo interfaces.ModelRepository + ollamaService *ollama.OllamaService +} + +// NewModelService creates a new model service instance +func NewModelService(repo interfaces.ModelRepository, ollamaService *ollama.OllamaService) interfaces.ModelService { + return &modelService{ + repo: repo, + ollamaService: ollamaService, + } +} + +// CreateModel creates a new model in the repository +// For local models, it initiates an asynchronous download process +// Remote models are immediately set to active status +func (s *modelService) CreateModel(ctx context.Context, model *types.Model) error { + logger.Info(ctx, "Start creating model") + logger.Infof(ctx, "Creating model: %s, type: %s, source: %s", model.Name, model.Type, model.Source) + + // Handle remote models (e.g., OpenAI, Azure) + if model.Source == types.ModelSourceRemote { + logger.Info(ctx, "Remote model detected, setting status to active") + model.Status = types.ModelStatusActive + + logger.Info(ctx, "Saving remote model to repository") + err := s.repo.Create(ctx, model) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_name": model.Name, + "model_type": model.Type, + }) + return err + } + + logger.Infof(ctx, "Remote model created successfully: %s", model.ID) + return nil + } + + // Handle local models (e.g., Ollama) + logger.Info(ctx, "Local model detected, setting status to downloading") + model.Status = types.ModelStatusDownloading + + logger.Info(ctx, "Saving local model to repository") + err := s.repo.Create(ctx, model) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_name": model.Name, + "model_type": model.Type, + }) + return err + } + + // Start asynchronous model download + logger.Infof(ctx, "Starting background download for model: %s", model.Name) + newCtx := logger.CloneContext(ctx) + go func() { + logger.Info(newCtx, "Background download started") + err := s.ollamaService.PullModel(newCtx, model.Name) + if err != nil { + logger.ErrorWithFields(newCtx, err, map[string]interface{}{ + "model_name": model.Name, + }) + model.Status = types.ModelStatusDownloadFailed + } else { + logger.Infof(newCtx, "Model download completed successfully: %s", model.Name) + model.Status = types.ModelStatusActive + } + logger.Infof(newCtx, "Updating model status to: %s", model.Status) + s.repo.Update(newCtx, model) + }() + + logger.Infof(ctx, "Model creation initiated successfully: %s", model.ID) + return nil +} + +// GetModelByID retrieves a model by its ID +// Returns an error if the model is not found or is in a non-active state +func (s *modelService) GetModelByID(ctx context.Context, id string) (*types.Model, error) { + logger.Info(ctx, "Start getting model by ID") + logger.Infof(ctx, "Getting model with ID: %s", id) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + // Fetch model from repository + model, err := s.repo.GetByID(ctx, tenantID, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": id, + "tenant_id": tenantID, + }) + return nil, err + } + + // Check if model exists + if model == nil { + logger.Error(ctx, "Model not found") + return nil, ErrModelNotFound + } + + logger.Infof(ctx, "Model found, name: %s, status: %s", model.Name, model.Status) + + // Check model status + if model.Status == types.ModelStatusActive { + logger.Info(ctx, "Model is active and ready to use") + return model, nil + } + + if model.Status == types.ModelStatusDownloading { + logger.Warn(ctx, "Model is currently downloading") + return nil, errors.New("model is currently downloading") + } + + if model.Status == types.ModelStatusDownloadFailed { + logger.Error(ctx, "Model download failed") + return nil, errors.New("model download failed") + } + + logger.Error(ctx, "Model status is abnormal") + return nil, errors.New("abnormal model status") +} + +// ListModels returns all models belonging to the tenant +func (s *modelService) ListModels(ctx context.Context) ([]*types.Model, error) { + logger.Info(ctx, "Start listing models") + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Listing models for tenant ID: %d", tenantID) + + // List models from repository with no additional filters + models, err := s.repo.List(ctx, tenantID, "", "") + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Infof(ctx, "Retrieved %d models successfully", len(models)) + return models, nil +} + +// UpdateModel updates an existing model in the repository +func (s *modelService) UpdateModel(ctx context.Context, model *types.Model) error { + logger.Info(ctx, "Start updating model") + logger.Infof(ctx, "Updating model ID: %s, name: %s", model.ID, model.Name) + + // Update model in repository + err := s.repo.Update(ctx, model) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "model_name": model.Name, + }) + return err + } + + logger.Infof(ctx, "Model updated successfully: %s", model.ID) + return nil +} + +// DeleteModel removes a model from the repository +func (s *modelService) DeleteModel(ctx context.Context, id string) error { + logger.Info(ctx, "Start deleting model") + logger.Infof(ctx, "Deleting model ID: %s", id) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + // Delete model from repository + err := s.repo.Delete(ctx, tenantID, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": id, + "tenant_id": tenantID, + }) + return err + } + + logger.Infof(ctx, "Model deleted successfully: %s", id) + return nil +} + +// GetEmbeddingModel retrieves and initializes an embedding model instance +// Takes a model ID and returns an Embedder interface implementation +func (s *modelService) GetEmbeddingModel(ctx context.Context, modelId string) (embedding.Embedder, error) { + logger.Info(ctx, "Start getting embedding model") + logger.Infof(ctx, "Getting embedding model with ID: %s", modelId) + + // Get the model details + model, err := s.GetModelByID(ctx, modelId) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": modelId, + }) + return nil, err + } + + logger.Info(ctx, "Creating embedder instance") + logger.Infof(ctx, "Model name: %s, source: %s", model.Name, model.Source) + + // Initialize the embedder with model configuration + embedder, err := embedding.NewEmbedder(embedding.Config{ + Source: model.Source, + BaseURL: model.Parameters.BaseURL, + APIKey: model.Parameters.APIKey, + ModelID: model.ID, + ModelName: model.Name, + Dimensions: model.Parameters.EmbeddingParameters.Dimension, + TruncatePromptTokens: model.Parameters.EmbeddingParameters.TruncatePromptTokens, + }) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "model_name": model.Name, + }) + return nil, err + } + + logger.Info(ctx, "Embedding model initialized successfully") + return embedder, nil +} + +// GetRerankModel retrieves and initializes a reranking model instance +// Takes a model ID and returns a Reranker interface implementation +func (s *modelService) GetRerankModel(ctx context.Context, modelId string) (rerank.Reranker, error) { + logger.Info(ctx, "Start getting rerank model") + logger.Infof(ctx, "Getting rerank model with ID: %s", modelId) + + // Get the model details + model, err := s.GetModelByID(ctx, modelId) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": modelId, + }) + return nil, err + } + + logger.Info(ctx, "Creating reranker instance") + logger.Infof(ctx, "Model name: %s, source: %s", model.Name, model.Source) + + // Initialize the reranker with model configuration + reranker, err := rerank.NewReranker(&rerank.RerankerConfig{ + ModelID: model.ID, + APIKey: model.Parameters.APIKey, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + Source: model.Source, + }) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "model_name": model.Name, + }) + return nil, err + } + + logger.Info(ctx, "Rerank model initialized successfully") + return reranker, nil +} + +// GetChatModel retrieves and initializes a chat model instance +// Takes a model ID and returns a Chat interface implementation +func (s *modelService) GetChatModel(ctx context.Context, modelId string) (chat.Chat, error) { + logger.Info(ctx, "Start getting chat model") + logger.Infof(ctx, "Getting chat model with ID: %s", modelId) + + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Tenant ID: %d", tenantID) + + // Get the model directly from repository to avoid status checks + model, err := s.repo.GetByID(ctx, tenantID, modelId) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": modelId, + "tenant_id": tenantID, + }) + return nil, err + } + + if model == nil { + logger.Error(ctx, "Chat model not found") + return nil, ErrModelNotFound + } + + logger.Info(ctx, "Creating chat model instance") + logger.Infof(ctx, "Model name: %s, source: %s", model.Name, model.Source) + + // Initialize the chat model with model configuration + chatModel, err := chat.NewChat(&chat.ChatConfig{ + ModelID: model.ID, + APIKey: model.Parameters.APIKey, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + Source: model.Source, + }) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "model_name": model.Name, + }) + return nil, err + } + + logger.Info(ctx, "Chat model initialized successfully") + return chatModel, nil +} diff --git a/internal/application/service/retriever/composite.go b/internal/application/service/retriever/composite.go new file mode 100644 index 0000000..701a658 --- /dev/null +++ b/internal/application/service/retriever/composite.go @@ -0,0 +1,289 @@ +package retriever + +import ( + "context" + "fmt" + "maps" + "slices" + "sync" + "sync/atomic" + + "github.com/Tencent/WeKnora/internal/common" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/runtime" + "github.com/Tencent/WeKnora/internal/tracing" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "go.opentelemetry.io/otel/attribute" +) + +// engineInfo holds information about a retrieve engine and its supported retriever types +type engineInfo struct { + retrieveEngine interfaces.RetrieveEngineService + retrieverType []types.RetrieverType +} + +// CompositeRetrieveEngine implements a composite pattern for retrieval engines, +// delegating operations to all registered engines +type CompositeRetrieveEngine struct { + engineInfos []*engineInfo +} + +// Retrieve performs retrieval operations by delegating to the appropriate engine +// based on the retriever type specified in the parameters +func (c *CompositeRetrieveEngine) Retrieve(ctx context.Context, + retrieveParams []types.RetrieveParams, +) ([]*types.RetrieveResult, error) { + return concurrentRetrieve(ctx, retrieveParams, + func(ctx context.Context, param types.RetrieveParams, results *[]*types.RetrieveResult, mu *sync.Mutex) error { + found := false + for _, engineInfo := range c.engineInfos { + if slices.Contains(engineInfo.retrieverType, param.RetrieverType) { + result, err := engineInfo.retrieveEngine.Retrieve(ctx, param) + if err != nil { + return err + } + mu.Lock() + *results = append(*results, result...) + mu.Unlock() + found = true + break + } + } + if !found { + return fmt.Errorf("retriever type %s not found", param.RetrieverType) + } + return nil + }, + ) +} + +// NewCompositeRetrieveEngine creates a new composite retrieve engine with the given parameters +func NewCompositeRetrieveEngine(engineParams []types.RetrieverEngineParams) (*CompositeRetrieveEngine, error) { + var registry interfaces.RetrieveEngineRegistry + runtime.GetContainer().Invoke(func(r interfaces.RetrieveEngineRegistry) { + registry = r + }) + engineInfos := make(map[types.RetrieverEngineType]*engineInfo) + for _, engineParam := range engineParams { + repo, err := registry.GetRetrieveEngineService(engineParam.RetrieverEngineType) + if err != nil { + return nil, err + } + if !slices.Contains(repo.Support(), engineParam.RetrieverType) { + return nil, fmt.Errorf("retrieval engine %s does not support retriever type: %s", + repo.EngineType(), engineParam.RetrieverType) + } + if _, exists := engineInfos[repo.EngineType()]; exists { + engineInfos[repo.EngineType()].retrieverType = append(engineInfos[repo.EngineType()].retrieverType, + engineParam.RetrieverType) + continue + } + engineInfos[repo.EngineType()] = &engineInfo{ + retrieveEngine: repo, + retrieverType: []types.RetrieverType{engineParam.RetrieverType}, + } + } + return &CompositeRetrieveEngine{engineInfos: slices.Collect(maps.Values(engineInfos))}, nil +} + +// SupportRetriever checks if a retriever type is supported by any of the registered engines +func (c *CompositeRetrieveEngine) SupportRetriever(r types.RetrieverType) bool { + for _, engineInfo := range c.engineInfos { + if slices.Contains(engineInfo.retrieverType, r) { + return true + } + } + return false +} + +// concurrentRetrieve is a helper function for concurrent processing of retrieval parameters +// and collecting results +func concurrentRetrieve( + ctx context.Context, + retrieveParams []types.RetrieveParams, + fn func(ctx context.Context, param types.RetrieveParams, results *[]*types.RetrieveResult, mu *sync.Mutex) error, +) ([]*types.RetrieveResult, error) { + var results []*types.RetrieveResult + var mu sync.Mutex + var wg sync.WaitGroup + errCh := make(chan error, len(retrieveParams)) + + for _, param := range retrieveParams { + wg.Add(1) + p := param // Create local copy for safe use in closure + go func() { + defer wg.Done() + if err := fn(ctx, p, &results, &mu); err != nil { + errCh <- err + } + }() + } + + wg.Wait() + close(errCh) + + // Check for errors + for err := range errCh { + if err != nil { + return nil, err + } + } + + return results, nil +} + +// concurrentExecWithError is a generic function for concurrent execution of operations +// and handling errors +func (c *CompositeRetrieveEngine) concurrentExecWithError( + ctx context.Context, + fn func(ctx context.Context, engineInfo *engineInfo) error, +) error { + var wg sync.WaitGroup + errCh := make(chan error, len(c.engineInfos)) + + for _, engineInfo := range c.engineInfos { + wg.Add(1) + eng := engineInfo // Create local copy for safe use in closure + go func() { + defer wg.Done() + if err := fn(ctx, eng); err != nil { + errCh <- err + } + }() + } + + wg.Wait() + close(errCh) + + // Return the first error (if any) + for err := range errCh { + if err != nil { + return err + } + } + return nil +} + +// Index saves vector embeddings to all registered repositories +func (c *CompositeRetrieveEngine) Index(ctx context.Context, + embedder embedding.Embedder, indexInfo *types.IndexInfo, +) error { + ctx, span := tracing.ContextWithSpan(ctx, "CompositeRetrieveEngine.Index") + defer span.End() + err := c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { + if err := engineInfo.retrieveEngine.Index(ctx, embedder, indexInfo, engineInfo.retrieverType); err != nil { + logger.Errorf(ctx, "Repository %s failed to save: %v", engineInfo.retrieveEngine.EngineType(), err) + return err + } + return nil + }) + span.RecordError(err) + span.SetAttributes( + attribute.String("embedder", embedder.GetModelName()), + attribute.String("source_id", indexInfo.SourceID), + ) + return err +} + +// BatchIndex batch saves vector embeddings to all registered repositories +func (c *CompositeRetrieveEngine) BatchIndex(ctx context.Context, + embedder embedding.Embedder, indexInfoList []*types.IndexInfo, +) error { + ctx, span := tracing.ContextWithSpan(ctx, "CompositeRetrieveEngine.BatchIndex") + defer span.End() + // Deduplicate sourceIDs + indexInfoList = common.Deduplicate(func(info *types.IndexInfo) string { return info.SourceID }, indexInfoList...) + err := c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { + if err := engineInfo.retrieveEngine.BatchIndex( + ctx, + embedder, + indexInfoList, + engineInfo.retrieverType, + ); err != nil { + logger.Errorf(ctx, "Repository %s failed to batch save: %v", engineInfo.retrieveEngine.EngineType(), err) + return err + } + return nil + }) + span.RecordError(err) + span.SetAttributes( + attribute.String("embedder", embedder.GetModelName()), + attribute.Int("index_info_count", len(indexInfoList)), + ) + return err +} + +// DeleteByChunkIDList deletes vector embeddings by chunk ID list from all registered repositories +func (c *CompositeRetrieveEngine) DeleteByChunkIDList(ctx context.Context, + chunkIDList []string, dimension int, +) error { + return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { + if err := engineInfo.retrieveEngine.DeleteByChunkIDList(ctx, chunkIDList, dimension); err != nil { + logger.GetLogger(ctx).Errorf("Repository %s failed to delete chunk ID list: %v", + engineInfo.retrieveEngine.EngineType(), err) + return err + } + return nil + }) +} + +// CopyIndices copies indices from a source knowledge base to a target knowledge base +func (c *CompositeRetrieveEngine) CopyIndices( + ctx context.Context, + sourceKnowledgeBaseID string, + targetKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + dimension int, +) error { + return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { + if err := engineInfo.retrieveEngine.CopyIndices( + ctx, + sourceKnowledgeBaseID, + sourceToTargetKBIDMap, + sourceToTargetChunkIDMap, + targetKnowledgeBaseID, + dimension, + ); err != nil { + logger.Errorf(ctx, "Repository %s failed to copy indices: %v", engineInfo.retrieveEngine.EngineType(), err) + return err + } + return nil + }) +} + +// DeleteByKnowledgeIDList deletes vector embeddings by knowledge ID list from all registered repositories +func (c *CompositeRetrieveEngine) DeleteByKnowledgeIDList(ctx context.Context, + knowledgeIDList []string, dimension int, +) error { + return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { + if err := engineInfo.retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension); err != nil { + logger.GetLogger(ctx).Errorf("Repository %s failed to delete knowledge ID list: %v", + engineInfo.retrieveEngine.EngineType(), err) + return err + } + return nil + }) +} + +// EstimateStorageSize estimates the storage size required for the provided index information +func (c *CompositeRetrieveEngine) EstimateStorageSize(ctx context.Context, + embedder embedding.Embedder, indexInfoList []*types.IndexInfo, +) int64 { + ctx, span := tracing.ContextWithSpan(ctx, "CompositeRetrieveEngine.EstimateStorageSize") + defer span.End() + sum := atomic.Int64{} + err := c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { + sum.Add(engineInfo.retrieveEngine.EstimateStorageSize(ctx, embedder, indexInfoList, engineInfo.retrieverType)) + return nil + }) + span.RecordError(err) + span.SetAttributes( + attribute.String("embedder", embedder.GetModelName()), + attribute.Int("index_info_count", len(indexInfoList)), + attribute.Int64("storage_size", sum.Load()), + ) + return sum.Load() +} diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go new file mode 100644 index 0000000..0345994 --- /dev/null +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go @@ -0,0 +1,163 @@ +package retriever + +import ( + "context" + "slices" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine +// that supports both keyword-based and vector-based retrieval +type KeywordsVectorHybridRetrieveEngineService struct { + indexRepository interfaces.RetrieveEngineRepository + engineType types.RetrieverEngineType +} + +// NewKVHybridRetrieveEngine creates a new instance of the hybrid retrieval engine +// KV stands for KeywordsVector +func NewKVHybridRetrieveEngine(indexRepository interfaces.RetrieveEngineRepository, + engineType types.RetrieverEngineType, +) interfaces.RetrieveEngineService { + return &KeywordsVectorHybridRetrieveEngineService{indexRepository: indexRepository, engineType: engineType} +} + +// EngineType returns the type of the retrieval engine +func (v *KeywordsVectorHybridRetrieveEngineService) EngineType() types.RetrieverEngineType { + return v.engineType +} + +// Retrieve performs retrieval based on the provided parameters +func (v *KeywordsVectorHybridRetrieveEngineService) Retrieve(ctx context.Context, + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { + return v.indexRepository.Retrieve(ctx, params) +} + +// Index creates embeddings for the content and saves it to the repository +// if vector retrieval is enabled in the retriever types +func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context, + embedder embedding.Embedder, indexInfo *types.IndexInfo, retrieverTypes []types.RetrieverType, +) error { + params := make(map[string]any) + embeddingMap := make(map[string][]float32) + if slices.Contains(retrieverTypes, types.VectorRetrieverType) { + embedding, err := embedder.Embed(ctx, indexInfo.Content) + if err != nil { + return err + } + embeddingMap[indexInfo.ChunkID] = embedding + } + params["embedding"] = embeddingMap + return v.indexRepository.Save(ctx, indexInfo, params) +} + +// BatchIndex creates embeddings for multiple content items and saves them to the repository +// in batches for efficiency +func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Context, + embedder embedding.Embedder, indexInfoList []*types.IndexInfo, retrieverTypes []types.RetrieverType, +) error { + if len(indexInfoList) == 0 { + return nil + } + params := make(map[string]any) + if slices.Contains(retrieverTypes, types.VectorRetrieverType) { + var contentList []string + for _, indexInfo := range indexInfoList { + contentList = append(contentList, indexInfo.Content) + } + var embeddings [][]float32 + var err error + for range 5 { + embeddings, err = embedder.BatchEmbedWithPool(ctx, embedder, contentList) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + if err != nil { + return err + } + batchSize := 20 + for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) { + embeddingMap := make(map[string][]float32) + for j, indexInfo := range indexChunk { + embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j] + } + params["embedding"] = embeddingMap + err = v.indexRepository.BatchSave(ctx, indexChunk, params) + if err != nil { + return err + } + } + return nil + } + var err error + for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) { + err = v.indexRepository.BatchSave(ctx, indexChunk, params) + if err != nil { + return err + } + } + return nil +} + +// DeleteByChunkIDList deletes vectors by their chunk IDs +func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByChunkIDList(ctx context.Context, + indexIDList []string, dimension int, +) error { + return v.indexRepository.DeleteByChunkIDList(ctx, indexIDList, dimension) +} + +// DeleteByKnowledgeIDList deletes vectors by their knowledge IDs +func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByKnowledgeIDList(ctx context.Context, + knowledgeIDList []string, dimension int, +) error { + return v.indexRepository.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension) +} + +// Support returns the retriever types supported by this engine +func (v *KeywordsVectorHybridRetrieveEngineService) Support() []types.RetrieverType { + return v.indexRepository.Support() +} + +// EstimateStorageSize estimates the storage space needed for the provided index information +func (v *KeywordsVectorHybridRetrieveEngineService) EstimateStorageSize( + ctx context.Context, + embedder embedding.Embedder, + indexInfoList []*types.IndexInfo, + retrieverTypes []types.RetrieverType, +) int64 { + params := make(map[string]any) + if slices.Contains(retrieverTypes, types.VectorRetrieverType) { + embeddingMap := make(map[string][]float32) + // just for estimate storage size + for _, indexInfo := range indexInfoList { + embeddingMap[indexInfo.ChunkID] = make([]float32, embedder.GetDimensions()) + } + params["embedding"] = embeddingMap + } + return v.indexRepository.EstimateStorageSize(ctx, indexInfoList, params) +} + +// CopyIndices copies indices from a source knowledge base to a target knowledge base +func (v *KeywordsVectorHybridRetrieveEngineService) CopyIndices( + ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, +) error { + logger.Infof(ctx, "Copy indices from knowledge base %s to %s, mapping relation count: %d", + sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), + ) + return v.indexRepository.CopyIndices( + ctx, sourceKnowledgeBaseID, sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension, + ) +} diff --git a/internal/application/service/retriever/registry.go b/internal/application/service/retriever/registry.go new file mode 100644 index 0000000..f489c58 --- /dev/null +++ b/internal/application/service/retriever/registry.go @@ -0,0 +1,64 @@ +package retriever + +import ( + "fmt" + "sync" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// RetrieveEngineRegistry implements the retrieval engine registry +type RetrieveEngineRegistry struct { + repositories map[types.RetrieverEngineType]interfaces.RetrieveEngineService + mu sync.RWMutex +} + +// NewRetrieveEngineRegistry creates a new retrieval engine registry +func NewRetrieveEngineRegistry() interfaces.RetrieveEngineRegistry { + return &RetrieveEngineRegistry{ + repositories: make(map[types.RetrieverEngineType]interfaces.RetrieveEngineService), + } +} + +// Register registers a retrieval engine service +func (r *RetrieveEngineRegistry) Register(repo interfaces.RetrieveEngineService) error { + r.mu.Lock() + defer r.mu.Unlock() + + if _, exists := r.repositories[repo.EngineType()]; exists { + return fmt.Errorf("Repository type %s already registered", repo.EngineType()) + } + + r.repositories[repo.EngineType()] = repo + return nil +} + +// GetRetrieveEngineService retrieves a retrieval engine service by type +func (r *RetrieveEngineRegistry) GetRetrieveEngineService(repoType types.RetrieverEngineType) ( + interfaces.RetrieveEngineService, error, +) { + r.mu.RLock() + defer r.mu.RUnlock() + + repo, exists := r.repositories[repoType] + if !exists { + return nil, fmt.Errorf("Repository of type %s not found", repoType) + } + + return repo, nil +} + +// GetAllRetrieveEngineServices retrieves all registered retrieval engine services +func (r *RetrieveEngineRegistry) GetAllRetrieveEngineServices() []interfaces.RetrieveEngineService { + r.mu.RLock() + defer r.mu.RUnlock() + + // Create a copy to avoid modifying the original map + result := make([]interfaces.RetrieveEngineService, 0, len(r.repositories)) + for _, v := range r.repositories { + result = append(result, v) + } + + return result +} diff --git a/internal/application/service/session.go b/internal/application/service/session.go new file mode 100644 index 0000000..a229ba3 --- /dev/null +++ b/internal/application/service/session.go @@ -0,0 +1,517 @@ +package service + +import ( + "context" + "errors" + "strings" + + chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/tracing" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" +) + +// sessionService implements the SessionService interface for managing conversation sessions +type sessionService struct { + cfg *config.Config // Application configuration + sessionRepo interfaces.SessionRepository // Repository for session data + messageRepo interfaces.MessageRepository // Repository for message data + knowledgeBaseService interfaces.KnowledgeBaseService // Service for knowledge base operations + modelService interfaces.ModelService // Service for model operations + eventManager *chatpipline.EventManager // Event manager for chat pipeline +} + +// NewSessionService creates a new session service instance with all required dependencies +func NewSessionService(cfg *config.Config, + sessionRepo interfaces.SessionRepository, + messageRepo interfaces.MessageRepository, + knowledgeBaseService interfaces.KnowledgeBaseService, + modelService interfaces.ModelService, + eventManager *chatpipline.EventManager, +) interfaces.SessionService { + return &sessionService{ + cfg: cfg, + sessionRepo: sessionRepo, + messageRepo: messageRepo, + knowledgeBaseService: knowledgeBaseService, + modelService: modelService, + eventManager: eventManager, + } +} + +// CreateSession creates a new conversation session +func (s *sessionService) CreateSession(ctx context.Context, session *types.Session) (*types.Session, error) { + logger.Info(ctx, "Start creating session") + + // Validate tenant ID + if session.TenantID == 0 { + logger.Error(ctx, "Failed to create session: tenant ID cannot be empty") + return nil, errors.New("tenant ID is required") + } + + logger.Infof(ctx, "Creating session, tenant ID: %d, model ID: %s, knowledge base ID: %s", + session.TenantID, session.SummaryModelID, session.KnowledgeBaseID) + + // Create session in repository + createdSession, err := s.sessionRepo.Create(ctx, session) + if err != nil { + return nil, err + } + + logger.Infof(ctx, "Session created successfully, ID: %s, tenant ID: %d", createdSession.ID, createdSession.TenantID) + return createdSession, nil +} + +// GetSession retrieves a session by its ID +func (s *sessionService) GetSession(ctx context.Context, id string) (*types.Session, error) { + logger.Info(ctx, "Start retrieving session") + + // Validate session ID + if id == "" { + logger.Error(ctx, "Failed to get session: session ID cannot be empty") + return nil, errors.New("session id is required") + } + + // Get tenant ID from context + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Retrieving session, ID: %s, tenant ID: %d", id, tenantID) + + // Get session from repository + session, err := s.sessionRepo.Get(ctx, tenantID, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": id, + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Infof(ctx, "Session retrieved successfully, ID: %s, tenant ID: %d", session.ID, session.TenantID) + return session, nil +} + +// GetSessionsByTenant retrieves all sessions for the current tenant +func (s *sessionService) GetSessionsByTenant(ctx context.Context) ([]*types.Session, error) { + logger.Info(ctx, "Start retrieving all sessions for tenant") + + // Get tenant ID from context + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Retrieving all sessions for tenant, tenant ID: %d", tenantID) + + // Get sessions from repository + sessions, err := s.sessionRepo.GetByTenantID(ctx, tenantID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenantID, + }) + return nil, err + } + + logger.Infof( + ctx, "Tenant sessions retrieved successfully, tenant ID: %d, session count: %d", tenantID, len(sessions), + ) + return sessions, nil +} + +// GetPagedSessionsByTenant retrieves sessions for the current tenant with pagination +func (s *sessionService) GetPagedSessionsByTenant(ctx context.Context, + pagination *types.Pagination, +) (*types.PageResult, error) { + logger.Info(ctx, "Start retrieving paged sessions for tenant") + + // Get tenant ID from context + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Retrieving paged sessions for tenant, tenant ID: %d, page: %d, page size: %d", + tenantID, pagination.Page, pagination.PageSize) + + // Get paged sessions from repository + sessions, total, err := s.sessionRepo.GetPagedByTenantID(ctx, tenantID, pagination) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenantID, + "page": pagination.Page, + "page_size": pagination.PageSize, + }) + return nil, err + } + + logger.Infof(ctx, "Tenant paged sessions retrieved successfully, tenant ID: %d, total: %d", tenantID, total) + return types.NewPageResult(total, pagination, sessions), nil +} + +// UpdateSession updates an existing session's properties +func (s *sessionService) UpdateSession(ctx context.Context, session *types.Session) error { + logger.Info(ctx, "Start updating session") + + // Validate session ID + if session.ID == "" { + logger.Error(ctx, "Failed to update session: session ID cannot be empty") + return errors.New("session id is required") + } + + logger.Infof(ctx, "Updating session, ID: %s, tenant ID: %d", session.ID, session.TenantID) + + // Update session in repository + err := s.sessionRepo.Update(ctx, session) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": session.ID, + "tenant_id": session.TenantID, + }) + return err + } + + logger.Infof(ctx, "Session updated successfully, ID: %s", session.ID) + return nil +} + +// DeleteSession removes a session by its ID +func (s *sessionService) DeleteSession(ctx context.Context, id string) error { + logger.Info(ctx, "Start deleting session") + + // Validate session ID + if id == "" { + logger.Error(ctx, "Failed to delete session: session ID cannot be empty") + return errors.New("session id is required") + } + + // Get tenant ID from context + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Deleting session, ID: %s, tenant ID: %d", id, tenantID) + + // Delete session from repository + err := s.sessionRepo.Delete(ctx, tenantID, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": id, + "tenant_id": tenantID, + }) + return err + } + + logger.Infof(ctx, "Session deleted successfully, ID: %s", id) + return nil +} + +// GenerateTitle generates a title for the current conversation content +func (s *sessionService) GenerateTitle(ctx context.Context, + sessionID string, messages []types.Message, +) (string, error) { + logger.Info(ctx, "Start generating session title") + + // Validate session ID + if sessionID == "" { + logger.Error(ctx, "Failed to generate title: session ID cannot be empty") + return "", errors.New("session id is required") + } + + // Get tenant ID from context + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Getting session info, session ID: %s, tenant ID: %d", sessionID, tenantID) + + // Get session from repository + session, err := s.sessionRepo.Get(ctx, tenantID, sessionID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + "tenant_id": tenantID, + }) + return "", err + } + + // Skip if title already exists + if session.Title != "" { + logger.Infof(ctx, "Session already has a title, session ID: %s, title: %s", sessionID, session.Title) + return session.Title, nil + } + + // Get the first user message, either from provided messages or repository + var message *types.Message + if len(messages) == 0 { + logger.Info(ctx, "Message list is empty, getting the first user message") + message, err = s.messageRepo.GetFirstMessageOfUser(ctx, sessionID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + }) + return "", err + } + } else { + logger.Info(ctx, "Searching for user message in message list") + for _, m := range messages { + if m.Role == "user" { + message = &m + break + } + } + } + + // Ensure a user message was found + if message == nil { + logger.Error(ctx, "No user message found, cannot generate title") + return "", errors.New("no user message found") + } + + // Get chat model + logger.Infof(ctx, "Getting chat model, model ID: %s", session.SummaryModelID) + chatModel, err := s.modelService.GetChatModel(ctx, session.SummaryModelID) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": session.SummaryModelID, + }) + return "", err + } + + // Prepare messages for title generation + logger.Info(ctx, "Preparing to generate session title") + var chatMessages []chat.Message + chatMessages = append(chatMessages, + chat.Message{Role: "system", Content: s.cfg.Conversation.GenerateSessionTitlePrompt}, + ) + chatMessages = append(chatMessages, + chat.Message{Role: "user", Content: message.Content + " /no_think"}, + ) + + // Call model to generate title + thinking := false + logger.Info(ctx, "Calling model to generate title") + response, err := chatModel.Chat(ctx, chatMessages, &chat.ChatOptions{ + Temperature: 0.3, + Thinking: &thinking, + }) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + return "", err + } + + // Process and store the generated title + session.Title = strings.TrimPrefix(response.Content, "\n\n") + logger.Infof(ctx, "Title generated successfully: %s", session.Title) + + // Update session with new title + logger.Info(ctx, "Updating session title") + err = s.sessionRepo.Update(ctx, session) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + return "", err + } + + logger.Infof(ctx, "Session title updated successfully, ID: %s, title: %s", sessionID, session.Title) + return session.Title, nil +} + +// KnowledgeQA performs knowledge base question answering with LLM summarization +func (s *sessionService) KnowledgeQA(ctx context.Context, sessionID, query string) ( + []*types.SearchResult, <-chan types.StreamResponse, error, +) { + logger.Info(ctx, "Start knowledge base question answering") + logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, query: %s", sessionID, query) + + // Get tenant ID from context + tenantID := ctx.Value(types.TenantIDContextKey).(uint) + logger.Infof(ctx, "Getting session info, session ID: %s, tenant ID: %d", sessionID, tenantID) + + // Get session information + session, err := s.sessionRepo.Get(ctx, tenantID, sessionID) + if err != nil { + logger.Errorf(ctx, "Failed to get session, session ID: %s, error: %v", sessionID, err) + return nil, nil, err + } + + // Validate knowledge base association + if session.KnowledgeBaseID == "" { + logger.Warnf(ctx, "Session has no associated knowledge base, session ID: %s", sessionID) + return nil, nil, errors.New("session has no knowledge base") + } + + // Create chat management object with session settings + logger.Infof(ctx, "Creating chat manage object, knowledge base ID: %s", session.KnowledgeBaseID) + chatManage := &types.ChatManage{ + Query: query, + RewriteQuery: query, + SessionID: sessionID, + KnowledgeBaseID: session.KnowledgeBaseID, + VectorThreshold: session.VectorThreshold, + KeywordThreshold: session.KeywordThreshold, + EmbeddingTopK: session.EmbeddingTopK, + RerankModelID: session.RerankModelID, + RerankTopK: session.RerankTopK, + RerankThreshold: session.RerankThreshold, + ChatModelID: session.SummaryModelID, + SummaryConfig: types.SummaryConfig{ + MaxTokens: session.SummaryParameters.MaxTokens, + RepeatPenalty: session.SummaryParameters.RepeatPenalty, + TopK: session.SummaryParameters.TopK, + TopP: session.SummaryParameters.TopP, + FrequencyPenalty: session.SummaryParameters.FrequencyPenalty, + PresencePenalty: session.SummaryParameters.PresencePenalty, + Prompt: session.SummaryParameters.Prompt, + ContextTemplate: session.SummaryParameters.ContextTemplate, + Temperature: session.SummaryParameters.Temperature, + Seed: session.SummaryParameters.Seed, + NoMatchPrefix: session.SummaryParameters.NoMatchPrefix, + MaxCompletionTokens: session.SummaryParameters.MaxCompletionTokens, + }, + FallbackResponse: session.FallbackResponse, + } + + // Start knowledge QA event processing + logger.Info(ctx, "Triggering knowledge base question answering event") + err = s.KnowledgeQAByEvent(ctx, chatManage, types.Pipline["rag_stream"]) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "session_id": sessionID, + "knowledge_base_id": session.KnowledgeBaseID, + }) + return nil, nil, err + } + + logger.Info(ctx, "Knowledge base question answering completed") + return chatManage.MergeResult, chatManage.ResponseChan, nil +} + +// KnowledgeQAByEvent processes knowledge QA through a series of events in the pipeline +func (s *sessionService) KnowledgeQAByEvent(ctx context.Context, + chatManage *types.ChatManage, eventList []types.EventType, +) error { + ctx, span := tracing.ContextWithSpan(ctx, "SessionService.KnowledgeQAByEvent") + defer span.End() + + logger.Info(ctx, "Start processing knowledge base question answering through events") + logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, knowledge base ID: %s, query: %s", + chatManage.SessionID, chatManage.KnowledgeBaseID, chatManage.Query) + + // Prepare method list for logging and tracing + methods := []string{} + for _, event := range eventList { + methods = append(methods, string(event)) + } + + // Set up tracing attributes + logger.Infof(ctx, "Trigger event list: %v", methods) + span.SetAttributes( + attribute.String("request_id", ctx.Value(types.RequestIDContextKey).(string)), + attribute.String("query", chatManage.Query), + attribute.String("method", strings.Join(methods, ",")), + ) + + // Process each event in sequence + for _, event := range eventList { + logger.Infof(ctx, "Starting to trigger event: %v", event) + err := s.eventManager.Trigger(ctx, event, chatManage) + + // Handle case where search returns no results + if err == chatpipline.ErrSearchNothing { + logger.Warnf(ctx, "Event %v triggered, search result is empty, using fallback response", event) + chatManage.ResponseChan = chatpipline.NewFallbackChan(ctx, chatManage.FallbackResponse) + chatManage.ChatResponse = &types.ChatResponse{Content: chatManage.FallbackResponse} + return nil + } + + // Handle other errors + if err != nil { + logger.Errorf(ctx, "Event triggering failed, event: %v, error type: %s, description: %s, error: %v", + event, err.ErrorType, err.Description, err.Err) + span.RecordError(err.Err) + span.SetStatus(codes.Error, err.Description) + span.SetAttributes(attribute.String("error_type", err.ErrorType)) + return err.Err + } + logger.Infof(ctx, "Event %v triggered successfully", event) + } + + logger.Info(ctx, "All events triggered successfully") + return nil +} + +// SearchKnowledge performs knowledge base search without LLM summarization +func (s *sessionService) SearchKnowledge(ctx context.Context, + knowledgeBaseID, query string, +) ([]*types.SearchResult, error) { + logger.Info(ctx, "Start knowledge base search without LLM summary") + logger.Infof(ctx, "Knowledge base search parameters, knowledge base ID: %s, query: %s", knowledgeBaseID, query) + + // Create default retrieval parameters + chatManage := &types.ChatManage{ + Query: query, + RewriteQuery: query, + KnowledgeBaseID: knowledgeBaseID, + VectorThreshold: s.cfg.Conversation.VectorThreshold, // Use default configuration + KeywordThreshold: s.cfg.Conversation.KeywordThreshold, // Use default configuration + EmbeddingTopK: s.cfg.Conversation.EmbeddingTopK, // Use default configuration + RerankTopK: s.cfg.Conversation.RerankTopK, // Use default configuration + RerankThreshold: s.cfg.Conversation.RerankThreshold, // Use default configuration + } + + // Get default models + models, err := s.modelService.ListModels(ctx) + if err != nil { + logger.Errorf(ctx, "Failed to get models: %v", err) + return nil, err + } + + // Find the first available rerank model + for _, model := range models { + if model.Type == types.ModelTypeRerank { + chatManage.RerankModelID = model.ID + break + } + } + + // Use specific event list, only including retrieval-related events, not LLM summarization + searchEvents := []types.EventType{ + types.PREPROCESS_QUERY, // Preprocess query + types.CHUNK_SEARCH, // Vector search + types.CHUNK_RERANK, // Rerank search results + types.CHUNK_MERGE, // Merge search results + types.FILTER_TOP_K, // Filter top K results + } + + ctx, span := tracing.ContextWithSpan(ctx, "SessionService.SearchKnowledge") + defer span.End() + + // Prepare method list for logging and tracing + methods := []string{} + for _, event := range searchEvents { + methods = append(methods, string(event)) + } + + // Set up tracing attributes + logger.Infof(ctx, "Trigger search event list: %v", methods) + span.SetAttributes( + attribute.String("query", query), + attribute.String("knowledge_base_id", knowledgeBaseID), + attribute.String("method", strings.Join(methods, ",")), + ) + + // Process each search event in sequence + for _, event := range searchEvents { + logger.Infof(ctx, "Starting to trigger search event: %v", event) + err := s.eventManager.Trigger(ctx, event, chatManage) + + // Handle case where search returns no results + if err == chatpipline.ErrSearchNothing { + logger.Warnf(ctx, "Event %v triggered, search result is empty", event) + return []*types.SearchResult{}, nil + } + + // Handle other errors + if err != nil { + logger.Errorf(ctx, "Event triggering failed, event: %v, error type: %s, description: %s, error: %v", + event, err.ErrorType, err.Description, err.Err) + span.RecordError(err.Err) + span.SetStatus(codes.Error, err.Description) + span.SetAttributes(attribute.String("error_type", err.ErrorType)) + return nil, err.Err + } + logger.Infof(ctx, "Event %v triggered successfully", event) + } + + logger.Infof(ctx, "Knowledge base search completed, found %d results", len(chatManage.MergeResult)) + return chatManage.MergeResult, nil +} diff --git a/internal/application/service/tenant.go b/internal/application/service/tenant.go new file mode 100644 index 0000000..4c3ea73 --- /dev/null +++ b/internal/application/service/tenant.go @@ -0,0 +1,291 @@ +package service + +import ( + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/binary" + "errors" + "io" + "os" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +var apiKeySecret = func() []byte { + return []byte(os.Getenv("TENANT_AES_KEY")) +} + +// ListTenantsParams defines parameters for listing tenants with filtering and pagination +type ListTenantsParams struct { + Page int // Page number for pagination + PageSize int // Number of items per page + Status string // Filter by tenant status + Name string // Filter by tenant name +} + +// tenantService implements the TenantService interface +type tenantService struct { + repo interfaces.TenantRepository // Repository for tenant data operations +} + +// NewTenantService creates a new tenant service instance +func NewTenantService(repo interfaces.TenantRepository) interfaces.TenantService { + return &tenantService{repo: repo} +} + +// CreateTenant creates a new tenant +func (s *tenantService) CreateTenant(ctx context.Context, tenant *types.Tenant) (*types.Tenant, error) { + logger.Info(ctx, "Start creating tenant") + + if tenant.Name == "" { + logger.Error(ctx, "Tenant name cannot be empty") + return nil, errors.New("tenant name cannot be empty") + } + + logger.Infof(ctx, "Creating tenant, name: %s", tenant.Name) + + // Create tenant with initial values + tenant.APIKey = s.generateApiKey(0) + tenant.Status = "active" + tenant.CreatedAt = time.Now() + tenant.UpdatedAt = time.Now() + + logger.Info(ctx, "Saving tenant information to database") + if err := s.repo.CreateTenant(ctx, tenant); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_name": tenant.Name, + }) + return nil, err + } + + logger.Infof(ctx, "Tenant created successfully, ID: %d, generating official API Key", tenant.ID) + tenant.APIKey = s.generateApiKey(tenant.ID) + if err := s.repo.UpdateTenant(ctx, tenant); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenant.ID, + "tenant_name": tenant.Name, + }) + return nil, err + } + + logger.Infof(ctx, "Tenant creation and update completed, ID: %d, name: %s", tenant.ID, tenant.Name) + return tenant, nil +} + +// GetTenantByID retrieves a tenant by their ID +func (s *tenantService) GetTenantByID(ctx context.Context, id uint) (*types.Tenant, error) { + logger.Info(ctx, "Start retrieving tenant") + + if id == 0 { + logger.Error(ctx, "Tenant ID cannot be 0") + return nil, errors.New("tenant ID cannot be 0") + } + + logger.Infof(ctx, "Retrieving tenant, ID: %d", id) + + tenant, err := s.repo.GetTenantByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": id, + }) + return nil, err + } + + logger.Infof(ctx, "Tenant retrieved successfully, ID: %d, name: %s", tenant.ID, tenant.Name) + return tenant, nil +} + +// ListTenants retrieves a list of all tenants +func (s *tenantService) ListTenants(ctx context.Context) ([]*types.Tenant, error) { + logger.Info(ctx, "Start retrieving tenant list") + + tenants, err := s.repo.ListTenants(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + return nil, err + } + + logger.Infof(ctx, "Tenant list retrieved successfully, total: %d", len(tenants)) + return tenants, nil +} + +// UpdateTenant updates an existing tenant's information +func (s *tenantService) UpdateTenant(ctx context.Context, tenant *types.Tenant) (*types.Tenant, error) { + if tenant.ID == 0 { + logger.Error(ctx, "Tenant ID cannot be 0") + return nil, errors.New("tenant ID cannot be 0") + } + + logger.Infof(ctx, "Updating tenant, ID: %d, name: %s", tenant.ID, tenant.Name) + + // Generate new API key if empty + if tenant.APIKey == "" { + logger.Info(ctx, "API Key is empty, generating new API Key") + tenant.APIKey = s.generateApiKey(tenant.ID) + } + + tenant.UpdatedAt = time.Now() + logger.Info(ctx, "Saving tenant information to database") + + if err := s.repo.UpdateTenant(ctx, tenant); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": tenant.ID, + }) + return nil, err + } + + logger.Infof(ctx, "Tenant updated successfully, ID: %d", tenant.ID) + return tenant, nil +} + +// DeleteTenant removes a tenant by their ID +func (s *tenantService) DeleteTenant(ctx context.Context, id uint) error { + logger.Info(ctx, "Start deleting tenant") + + if id == 0 { + logger.Error(ctx, "Tenant ID cannot be 0") + return errors.New("tenant ID cannot be 0") + } + + logger.Infof(ctx, "Deleting tenant, ID: %d", id) + + // Get tenant information for logging + tenant, err := s.repo.GetTenantByID(ctx, id) + if err != nil { + if err.Error() == "record not found" { + logger.Warnf(ctx, "Tenant to be deleted does not exist, ID: %d", id) + } else { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": id, + }) + return err + } + } else { + logger.Infof(ctx, "Deleting tenant, ID: %d, name: %s", id, tenant.Name) + } + + err = s.repo.DeleteTenant(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": id, + }) + return err + } + + logger.Infof(ctx, "Tenant deleted successfully, ID: %d", id) + return nil +} + +// UpdateAPIKey updates the API key for a specific tenant +func (s *tenantService) UpdateAPIKey(ctx context.Context, id uint) (string, error) { + logger.Info(ctx, "Start updating tenant API Key") + + if id == 0 { + logger.Error(ctx, "Tenant ID cannot be 0") + return "", errors.New("tenant ID cannot be 0") + } + + logger.Infof(ctx, "Retrieving tenant information, ID: %d", id) + + tenant, err := s.repo.GetTenantByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": id, + }) + return "", err + } + + logger.Infof(ctx, "Generating new API Key for tenant, ID: %d", id) + tenant.APIKey = s.generateApiKey(tenant.ID) + + if err := s.repo.UpdateTenant(ctx, tenant); err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "tenant_id": id, + }) + return "", err + } + + logger.Infof(ctx, "Tenant API Key updated successfully, ID: %d", id) + return tenant.APIKey, nil +} + +// generateApiKey generates a secure API key for tenant authentication +func (r *tenantService) generateApiKey(tenantID uint) string { + // 1. Convert tenant_id to bytes + idBytes := make([]byte, 8) + binary.LittleEndian.PutUint64(idBytes, uint64(tenantID)) + + // 2. Encrypt tenant_id using AES-GCM + block, err := aes.NewCipher(apiKeySecret()) + if err != nil { + panic("Failed to create AES cipher: " + err.Error()) + } + + nonce := make([]byte, 12) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + panic(err.Error()) + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + panic("Failed to create GCM cipher: " + err.Error()) + } + + ciphertext := aesgcm.Seal(nil, nonce, idBytes, nil) + + // 3. Combine nonce and ciphertext, then encode with base64 + combined := append(nonce, ciphertext...) + encoded := base64.RawURLEncoding.EncodeToString(combined) + + // Create final API Key in format: sk-{encrypted_part} + return "sk-" + encoded +} + +// ExtractTenantIDFromAPIKey extracts the tenant ID from an API key +func (r *tenantService) ExtractTenantIDFromAPIKey(apiKey string) (uint, error) { + // 1. Validate format and extract encrypted part + parts := strings.SplitN(apiKey, "-", 2) + if len(parts) != 2 || parts[0] != "sk" { + return 0, errors.New("invalid API key format") + } + + // 2. Decode the base64 part + encryptedData, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return 0, errors.New("invalid API key encoding") + } + + // 3. Separate nonce and ciphertext + if len(encryptedData) < 12 { + return 0, errors.New("invalid API key length") + } + nonce, ciphertext := encryptedData[:12], encryptedData[12:] + + // 4. Decrypt + block, err := aes.NewCipher(apiKeySecret()) + if err != nil { + return 0, errors.New("decryption error") + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return 0, errors.New("decryption error") + } + + plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return 0, errors.New("API key is invalid or has been tampered with") + } + + // 5. Convert back to tenant_id + tenantID := binary.LittleEndian.Uint64(plaintext) + + return uint(tenantID), nil +} diff --git a/internal/application/service/user.go b/internal/application/service/user.go new file mode 100644 index 0000000..671d12e --- /dev/null +++ b/internal/application/service/user.go @@ -0,0 +1,449 @@ +package service + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/uuid" + "golang.org/x/crypto/bcrypt" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// JWT secret key - in production this should be from environment variable +var jwtSecret = []byte("your-secret-key") + +// userService implements the UserService interface +type userService struct { + userRepo interfaces.UserRepository + tokenRepo interfaces.AuthTokenRepository + tenantService interfaces.TenantService +} + +// NewUserService creates a new user service instance +func NewUserService(userRepo interfaces.UserRepository, tokenRepo interfaces.AuthTokenRepository, tenantService interfaces.TenantService) interfaces.UserService { + return &userService{ + userRepo: userRepo, + tokenRepo: tokenRepo, + tenantService: tenantService, + } +} + +var engine = map[string][]types.RetrieverEngineParams{ + "postgres": { + { + RetrieverType: types.KeywordsRetrieverType, + RetrieverEngineType: types.PostgresRetrieverEngineType, + }, + { + RetrieverType: types.VectorRetrieverType, + RetrieverEngineType: types.PostgresRetrieverEngineType, + }, + }, + "elasticsearch_v7": { + { + RetrieverType: types.KeywordsRetrieverType, + RetrieverEngineType: types.ElasticsearchRetrieverEngineType, + }, + }, + "elasticsearch_v8": { + { + RetrieverType: types.KeywordsRetrieverType, + RetrieverEngineType: types.ElasticsearchRetrieverEngineType, + }, + { + RetrieverType: types.VectorRetrieverType, + RetrieverEngineType: types.ElasticsearchRetrieverEngineType, + }, + }, +} + +// Register creates a new user account +func (s *userService) Register(ctx context.Context, req *types.RegisterRequest) (*types.User, error) { + logger.Info(ctx, "Start user registration") + + // Validate input + if req.Username == "" || req.Email == "" || req.Password == "" { + return nil, errors.New("username, email and password are required") + } + + // Check if user already exists + existingUser, _ := s.userRepo.GetUserByEmail(ctx, req.Email) + if existingUser != nil { + return nil, errors.New("user with this email already exists") + } + + existingUser, _ = s.userRepo.GetUserByUsername(ctx, req.Username) + if existingUser != nil { + return nil, errors.New("user with this username already exists") + } + + // Hash password + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(req.Password), bcrypt.DefaultCost) + if err != nil { + logger.Errorf(ctx, "Failed to hash password: %v", err) + return nil, errors.New("failed to process password") + } + + egs := []types.RetrieverEngineParams{} + for _, driver := range strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") { + if val, ok := engine[driver]; ok { + egs = append(egs, val...) + } + } + egs = uniqueRetrieverEngine(egs) + logger.Debugf(ctx, "user register retriever engines: %v", egs) + + // Create default tenant for the user + tenant := &types.Tenant{ + Name: fmt.Sprintf("%s's Workspace", req.Username), + Description: "Default workspace", + Status: "active", + RetrieverEngines: types.RetrieverEngines{Engines: egs}, + } + + createdTenant, err := s.tenantService.CreateTenant(ctx, tenant) + if err != nil { + logger.Errorf(ctx, "Failed to create tenant: %v", err) + return nil, errors.New("failed to create workspace") + } + + // Create user + user := &types.User{ + ID: uuid.New().String(), + Username: req.Username, + Email: req.Email, + PasswordHash: string(hashedPassword), + TenantID: createdTenant.ID, + IsActive: true, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + err = s.userRepo.CreateUser(ctx, user) + if err != nil { + logger.Errorf(ctx, "Failed to create user: %v", err) + return nil, errors.New("failed to create user") + } + + logger.Infof(ctx, "User registered successfully: %s", user.Email) + return user, nil +} + +// Login authenticates a user and returns tokens +func (s *userService) Login(ctx context.Context, req *types.LoginRequest) (*types.LoginResponse, error) { + logger.Infof(ctx, "Start user login for email: %s", req.Email) + + // Get user by email + user, err := s.userRepo.GetUserByEmail(ctx, req.Email) + if err != nil { + logger.Errorf(ctx, "Failed to get user by email %s: %v", req.Email, err) + return &types.LoginResponse{ + Success: false, + Message: "Invalid email or password", + }, nil + } + if user == nil { + logger.Warnf(ctx, "User not found for email: %s", req.Email) + return &types.LoginResponse{ + Success: false, + Message: "Invalid email or password", + }, nil + } + + logger.Infof(ctx, "Found user: ID=%s, Email=%s, IsActive=%t", user.ID, user.Email, user.IsActive) + + // Check if user is active + if !user.IsActive { + logger.Warnf(ctx, "User account is disabled for email: %s", req.Email) + return &types.LoginResponse{ + Success: false, + Message: "Account is disabled", + }, nil + } + + // Verify password + logger.Infof(ctx, "Verifying password for user: %s", user.Email) + err = bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(req.Password)) + if err != nil { + logger.Warnf(ctx, "Password verification failed for user %s: %v", user.Email, err) + return &types.LoginResponse{ + Success: false, + Message: "Invalid email or password", + }, nil + } + logger.Infof(ctx, "Password verification successful for user: %s", user.Email) + + // Generate tokens + logger.Infof(ctx, "Generating tokens for user: %s", user.Email) + accessToken, refreshToken, err := s.GenerateTokens(ctx, user) + if err != nil { + logger.Errorf(ctx, "Failed to generate tokens for user %s: %v", user.Email, err) + return &types.LoginResponse{ + Success: false, + Message: "Login failed", + }, nil + } + logger.Infof(ctx, "Tokens generated successfully for user: %s", user.Email) + + // Get tenant information + logger.Infof(ctx, "Getting tenant information for user %s, tenant ID: %s", user.Email, user.TenantID) + tenant, err := s.tenantService.GetTenantByID(ctx, user.TenantID) + if err != nil { + logger.Warnf(ctx, "Failed to get tenant info for user %s, tenant ID %s: %v", user.Email, user.TenantID, err) + } else { + logger.Infof(ctx, "Tenant information retrieved successfully for user: %s", user.Email) + } + + logger.Infof(ctx, "User logged in successfully: %s", user.Email) + return &types.LoginResponse{ + Success: true, + Message: "Login successful", + User: user, + Tenant: tenant, + Token: accessToken, + RefreshToken: refreshToken, + }, nil +} + +// GetUserByID gets a user by ID +func (s *userService) GetUserByID(ctx context.Context, id string) (*types.User, error) { + return s.userRepo.GetUserByID(ctx, id) +} + +// GetUserByEmail gets a user by email +func (s *userService) GetUserByEmail(ctx context.Context, email string) (*types.User, error) { + return s.userRepo.GetUserByEmail(ctx, email) +} + +// GetUserByUsername gets a user by username +func (s *userService) GetUserByUsername(ctx context.Context, username string) (*types.User, error) { + return s.userRepo.GetUserByUsername(ctx, username) +} + +// UpdateUser updates user information +func (s *userService) UpdateUser(ctx context.Context, user *types.User) error { + user.UpdatedAt = time.Now() + return s.userRepo.UpdateUser(ctx, user) +} + +// DeleteUser deletes a user +func (s *userService) DeleteUser(ctx context.Context, id string) error { + return s.userRepo.DeleteUser(ctx, id) +} + +// ChangePassword changes user password +func (s *userService) ChangePassword(ctx context.Context, userID string, oldPassword, newPassword string) error { + user, err := s.userRepo.GetUserByID(ctx, userID) + if err != nil { + return err + } + + // Verify old password + err = bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(oldPassword)) + if err != nil { + return errors.New("invalid old password") + } + + // Hash new password + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(newPassword), bcrypt.DefaultCost) + if err != nil { + return err + } + + user.PasswordHash = string(hashedPassword) + user.UpdatedAt = time.Now() + + return s.userRepo.UpdateUser(ctx, user) +} + +// ValidatePassword validates user password +func (s *userService) ValidatePassword(ctx context.Context, userID string, password string) error { + user, err := s.userRepo.GetUserByID(ctx, userID) + if err != nil { + return err + } + + return bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(password)) +} + +// GenerateTokens generates access and refresh tokens for user +func (s *userService) GenerateTokens(ctx context.Context, user *types.User) (accessToken, refreshToken string, err error) { + // Generate access token (expires in 24 hours) + accessClaims := jwt.MapClaims{ + "user_id": user.ID, + "email": user.Email, + "tenant_id": user.TenantID, + "exp": time.Now().Add(24 * time.Hour).Unix(), + "iat": time.Now().Unix(), + "type": "access", + } + + accessTokenObj := jwt.NewWithClaims(jwt.SigningMethodHS256, accessClaims) + accessToken, err = accessTokenObj.SignedString(jwtSecret) + if err != nil { + return "", "", err + } + + // Generate refresh token (expires in 7 days) + refreshClaims := jwt.MapClaims{ + "user_id": user.ID, + "exp": time.Now().Add(7 * 24 * time.Hour).Unix(), + "iat": time.Now().Unix(), + "type": "refresh", + } + + refreshTokenObj := jwt.NewWithClaims(jwt.SigningMethodHS256, refreshClaims) + refreshToken, err = refreshTokenObj.SignedString(jwtSecret) + if err != nil { + return "", "", err + } + + // Store tokens in database + accessTokenRecord := &types.AuthToken{ + ID: uuid.New().String(), + UserID: user.ID, + Token: accessToken, + TokenType: "access_token", + ExpiresAt: time.Now().Add(24 * time.Hour), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + refreshTokenRecord := &types.AuthToken{ + ID: uuid.New().String(), + UserID: user.ID, + Token: refreshToken, + TokenType: "refresh_token", + ExpiresAt: time.Now().Add(7 * 24 * time.Hour), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + } + + _ = s.tokenRepo.CreateToken(ctx, accessTokenRecord) + _ = s.tokenRepo.CreateToken(ctx, refreshTokenRecord) + + return accessToken, refreshToken, nil +} + +// ValidateToken validates an access token +func (s *userService) ValidateToken(ctx context.Context, tokenString string) (*types.User, error) { + token, err := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return jwtSecret, nil + }) + + if err != nil || !token.Valid { + return nil, errors.New("invalid token") + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + return nil, errors.New("invalid token claims") + } + + userID, ok := claims["user_id"].(string) + if !ok { + return nil, errors.New("invalid user ID in token") + } + + // Check if token is revoked + tokenRecord, err := s.tokenRepo.GetTokenByValue(ctx, tokenString) + if err != nil || tokenRecord == nil || tokenRecord.IsRevoked { + return nil, errors.New("token is revoked") + } + + return s.userRepo.GetUserByID(ctx, userID) +} + +// RefreshToken refreshes access token using refresh token +func (s *userService) RefreshToken(ctx context.Context, refreshTokenString string) (accessToken, newRefreshToken string, err error) { + token, err := jwt.Parse(refreshTokenString, func(token *jwt.Token) (interface{}, error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return jwtSecret, nil + }) + + if err != nil || !token.Valid { + return "", "", errors.New("invalid refresh token") + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + return "", "", errors.New("invalid token claims") + } + + tokenType, ok := claims["type"].(string) + if !ok || tokenType != "refresh" { + return "", "", errors.New("not a refresh token") + } + + userID, ok := claims["user_id"].(string) + if !ok { + return "", "", errors.New("invalid user ID in token") + } + + // Check if token is revoked + tokenRecord, err := s.tokenRepo.GetTokenByValue(ctx, refreshTokenString) + if err != nil || tokenRecord == nil || tokenRecord.IsRevoked { + return "", "", errors.New("refresh token is revoked") + } + + // Get user + user, err := s.userRepo.GetUserByID(ctx, userID) + if err != nil { + return "", "", err + } + + // Revoke old refresh token + tokenRecord.IsRevoked = true + _ = s.tokenRepo.UpdateToken(ctx, tokenRecord) + + // Generate new tokens + return s.GenerateTokens(ctx, user) +} + +// RevokeToken revokes a token +func (s *userService) RevokeToken(ctx context.Context, tokenString string) error { + tokenRecord, err := s.tokenRepo.GetTokenByValue(ctx, tokenString) + if err != nil { + return err + } + + tokenRecord.IsRevoked = true + tokenRecord.UpdatedAt = time.Now() + + return s.tokenRepo.UpdateToken(ctx, tokenRecord) +} + +// GetCurrentUser gets current user from context +func (s *userService) GetCurrentUser(ctx context.Context) (*types.User, error) { + user, ok := ctx.Value("user").(*types.User) + if !ok { + return nil, errors.New("user not found in context") + } + + return user, nil +} + +func uniqueRetrieverEngine(engine []types.RetrieverEngineParams) []types.RetrieverEngineParams { + seen := make(map[types.RetrieverEngineParams]bool) + var result []types.RetrieverEngineParams + for _, v := range engine { + if !seen[v] { + seen[v] = true + result = append(result, v) + } + } + return result +} diff --git a/internal/common/tools.go b/internal/common/tools.go new file mode 100644 index 0000000..7811827 --- /dev/null +++ b/internal/common/tools.go @@ -0,0 +1,100 @@ +package common + +import ( + "encoding/json" + "maps" + "regexp" + "slices" + "strings" + "unicode/utf8" +) + +// ToInterfaceSlice converts a slice of strings to a slice of empty interfaces. +func ToInterfaceSlice[T any](slice []T) []interface{} { + interfaceSlice := make([]interface{}, len(slice)) + for i, v := range slice { + interfaceSlice[i] = v + } + return interfaceSlice +} + +// []string -> string, " join, space separated +func StringSliceJoin(slice []string) string { + result := make([]string, len(slice)) + for i, v := range slice { + result[i] = `"` + v + `"` + } + return strings.Join(result, " ") +} + +func GetAttrs[A, B any](extract func(A) B, attrs ...A) []B { + result := make([]B, len(attrs)) + for i, attr := range attrs { + result[i] = extract(attr) + } + return result +} + +// Deduplicate removes duplicates from a slice based on a key function +// T: the type of elements in the slice +// K: the type of key used for deduplication +func Deduplicate[T any, K comparable](keyFunc func(T) K, items ...T) []T { + seen := make(map[K]T) + for _, item := range items { + key := keyFunc(item) + if _, exists := seen[key]; !exists { + seen[key] = item + } + } + return slices.Collect(maps.Values(seen)) +} + +// ParseLLMJsonResponse parses a JSON response from LLM, handling cases where JSON is wrapped in code blocks. +// This is useful when LLMs return responses like: +// ```json +// {"key": "value"} +// ``` +// or regular JSON responses directly. +func ParseLLMJsonResponse(content string, target interface{}) error { + // First, try to parse directly as JSON + err := json.Unmarshal([]byte(content), target) + if err == nil { + return nil + } + + // If direct parsing fails, try to extract JSON from code blocks + re := regexp.MustCompile("```(?:json)?\\s*([\\s\\S]*?)```") + matches := re.FindStringSubmatch(content) + if len(matches) >= 2 { + // Extract the JSON content within the code block + jsonContent := strings.TrimSpace(matches[1]) + return json.Unmarshal([]byte(jsonContent), target) + } + + // If no code block found, return the original error + return err +} + +// CleanInvalidUTF8 移除字符串中的非法 UTF-8 字符和 \x00 +func CleanInvalidUTF8(s string) string { + var b strings.Builder + b.Grow(len(s)) + + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + // 非法 UTF-8 字节，跳过 + i++ + continue + } + if r == 0 { + // NULL 字符 \x00，跳过 + i += size + continue + } + b.WriteRune(r) + i += size + } + + return b.String() +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..8072813 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,190 @@ +package config + +import ( + "fmt" + "os" + "regexp" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/go-viper/mapstructure/v2" + "github.com/spf13/viper" +) + +// Config 应用程序总配置 +type Config struct { + Conversation *ConversationConfig `yaml:"conversation" json:"conversation"` + Server *ServerConfig `yaml:"server" json:"server"` + KnowledgeBase *KnowledgeBaseConfig `yaml:"knowledge_base" json:"knowledge_base"` + Tenant *TenantConfig `yaml:"tenant" json:"tenant"` + Models []ModelConfig `yaml:"models" json:"models"` + VectorDatabase *VectorDatabaseConfig `yaml:"vector_database" json:"vector_database"` + DocReader *DocReaderConfig `yaml:"docreader" json:"docreader"` + StreamManager *StreamManagerConfig `yaml:"stream_manager" json:"stream_manager"` + ExtractManager *ExtractManagerConfig `yaml:"extract" json:"extract"` +} + +type DocReaderConfig struct { + Addr string `yaml:"addr" json:"addr"` +} + +type VectorDatabaseConfig struct { + Driver string `yaml:"driver" json:"driver"` +} + +// ConversationConfig 对话服务配置 +type ConversationConfig struct { + MaxRounds int `yaml:"max_rounds" json:"max_rounds"` + KeywordThreshold float64 `yaml:"keyword_threshold" json:"keyword_threshold"` + EmbeddingTopK int `yaml:"embedding_top_k" json:"embedding_top_k"` + VectorThreshold float64 `yaml:"vector_threshold" json:"vector_threshold"` + RerankTopK int `yaml:"rerank_top_k" json:"rerank_top_k"` + RerankThreshold float64 `yaml:"rerank_threshold" json:"rerank_threshold"` + FallbackStrategy string `yaml:"fallback_strategy" json:"fallback_strategy"` + FallbackResponse string `yaml:"fallback_response" json:"fallback_response"` + FallbackPrompt string `yaml:"fallback_prompt" json:"fallback_prompt"` + EnableRewrite bool `yaml:"enable_rewrite" json:"enable_rewrite"` + EnableRerank bool `yaml:"enable_rerank" json:"enable_rerank"` + Summary *SummaryConfig `yaml:"summary" json:"summary"` + GenerateSessionTitlePrompt string `yaml:"generate_session_title_prompt" json:"generate_session_title_prompt"` + GenerateSummaryPrompt string `yaml:"generate_summary_prompt" json:"generate_summary_prompt"` + RewritePromptSystem string `yaml:"rewrite_prompt_system" json:"rewrite_prompt_system"` + RewritePromptUser string `yaml:"rewrite_prompt_user" json:"rewrite_prompt_user"` + SimplifyQueryPrompt string `yaml:"simplify_query_prompt" json:"simplify_query_prompt"` + SimplifyQueryPromptUser string `yaml:"simplify_query_prompt_user" json:"simplify_query_prompt_user"` + ExtractEntitiesPrompt string `yaml:"extract_entities_prompt" json:"extract_entities_prompt"` + ExtractRelationshipsPrompt string `yaml:"extract_relationships_prompt" json:"extract_relationships_prompt"` +} + +// SummaryConfig 摘要配置 +type SummaryConfig struct { + MaxTokens int `yaml:"max_tokens" json:"max_tokens"` + RepeatPenalty float64 `yaml:"repeat_penalty" json:"repeat_penalty"` + TopK int `yaml:"top_k" json:"top_k"` + TopP float64 `yaml:"top_p" json:"top_p"` + FrequencyPenalty float64 `yaml:"frequency_penalty" json:"frequency_penalty"` + PresencePenalty float64 `yaml:"presence_penalty" json:"presence_penalty"` + Prompt string `yaml:"prompt" json:"prompt"` + ContextTemplate string `yaml:"context_template" json:"context_template"` + Temperature float64 `yaml:"temperature" json:"temperature"` + Seed int `yaml:"seed" json:"seed"` + MaxCompletionTokens int `yaml:"max_completion_tokens" json:"max_completion_tokens"` + NoMatchPrefix string `yaml:"no_match_prefix" json:"no_match_prefix"` +} + +// ServerConfig 服务器配置 +type ServerConfig struct { + Port int `yaml:"port" json:"port"` + Host string `yaml:"host" json:"host"` + LogPath string `yaml:"log_path" json:"log_path"` + ShutdownTimeout time.Duration `yaml:"shutdown_timeout" json:"shutdown_timeout" default:"30s"` +} + +// KnowledgeBaseConfig 知识库配置 +type KnowledgeBaseConfig struct { + ChunkSize int `yaml:"chunk_size" json:"chunk_size"` + ChunkOverlap int `yaml:"chunk_overlap" json:"chunk_overlap"` + SplitMarkers []string `yaml:"split_markers" json:"split_markers"` + KeepSeparator bool `yaml:"keep_separator" json:"keep_separator"` + ImageProcessing *ImageProcessingConfig `yaml:"image_processing" json:"image_processing"` +} + +// ImageProcessingConfig 图像处理配置 +type ImageProcessingConfig struct { + EnableMultimodal bool `yaml:"enable_multimodal" json:"enable_multimodal"` +} + +// TenantConfig 租户配置 +type TenantConfig struct { + DefaultSessionName string `yaml:"default_session_name" json:"default_session_name"` + DefaultSessionTitle string `yaml:"default_session_title" json:"default_session_title"` + DefaultSessionDescription string `yaml:"default_session_description" json:"default_session_description"` +} + +// ModelConfig 模型配置 +type ModelConfig struct { + Type string `yaml:"type" json:"type"` + Source string `yaml:"source" json:"source"` + ModelName string `yaml:"model_name" json:"model_name"` + Parameters map[string]interface{} `yaml:"parameters" json:"parameters"` +} + +// StreamManagerConfig 流管理器配置 +type StreamManagerConfig struct { + Type string `yaml:"type" json:"type"` // 类型: "memory" 或 "redis" + Redis RedisConfig `yaml:"redis" json:"redis"` // Redis配置 + CleanupTimeout time.Duration `yaml:"cleanup_timeout" json:"cleanup_timeout"` // 清理超时，单位秒 +} + +// RedisConfig Redis配置 +type RedisConfig struct { + Address string `yaml:"address" json:"address"` // Redis地址 + Password string `yaml:"password" json:"password"` // Redis密码 + DB int `yaml:"db" json:"db"` // Redis数据库 + Prefix string `yaml:"prefix" json:"prefix"` // 键前缀 + TTL time.Duration `yaml:"ttl" json:"ttl"` // 过期时间(小时) +} + +// ExtractManagerConfig 抽取管理器配置 +type ExtractManagerConfig struct { + ExtractGraph *types.PromptTemplateStructured `yaml:"extract_graph" json:"extract_graph"` + ExtractEntity *types.PromptTemplateStructured `yaml:"extract_entity" json:"extract_entity"` + FabriText *FebriText `yaml:"fabri_text" json:"fabri_text"` +} + +type FebriText struct { + WithTag string `yaml:"with_tag" json:"with_tag"` + WithNoTag string `yaml:"with_no_tag" json:"with_no_tag"` +} + +// LoadConfig 从配置文件加载配置 +func LoadConfig() (*Config, error) { + // 设置配置文件名和路径 + viper.SetConfigName("config") // 配置文件名称(不带扩展名) + viper.SetConfigType("yaml") // 配置文件类型 + viper.AddConfigPath(".") // 当前目录 + viper.AddConfigPath("./config") // config子目录 + viper.AddConfigPath("$HOME/.appname") // 用户目录 + viper.AddConfigPath("/etc/appname/") // etc目录 + + // 启用环境变量替换 + viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + // 读取配置文件 + if err := viper.ReadInConfig(); err != nil { + return nil, fmt.Errorf("error reading config file: %w", err) + } + + // 替换配置中的环境变量引用 + configFileContent, err := os.ReadFile(viper.ConfigFileUsed()) + if err != nil { + return nil, fmt.Errorf("error reading config file content: %w", err) + } + + // 替换${ENV_VAR}格式的环境变量引用 + re := regexp.MustCompile(`\${([^}]+)}`) + result := re.ReplaceAllStringFunc(string(configFileContent), func(match string) string { + // 提取环境变量名称（去掉${}部分） + envVar := match[2 : len(match)-1] + // 获取环境变量值，如果不存在则保持原样 + if value := os.Getenv(envVar); value != "" { + return value + } + return match + }) + + // 使用处理后的配置内容 + viper.ReadConfig(strings.NewReader(result)) + + // 解析配置到结构体 + var cfg Config + if err := viper.Unmarshal(&cfg, func(dc *mapstructure.DecoderConfig) { + dc.TagName = "yaml" + }); err != nil { + return nil, fmt.Errorf("unable to decode config into struct: %w", err) + } + fmt.Printf("Using configuration file: %s\n", viper.ConfigFileUsed()) + return &cfg, nil +} diff --git a/internal/container/cleanup.go b/internal/container/cleanup.go new file mode 100644 index 0000000..fd14357 --- /dev/null +++ b/internal/container/cleanup.go @@ -0,0 +1,86 @@ +package container + +import ( + "context" + "log" + "sync" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// ResourceCleaner is a resource cleaner that can be used to clean up resources +type ResourceCleaner struct { + mu sync.Mutex + cleanups []types.CleanupFunc +} + +// NewResourceCleaner creates a new resource cleaner +func NewResourceCleaner() interfaces.ResourceCleaner { + return &ResourceCleaner{ + cleanups: make([]types.CleanupFunc, 0), + } +} + +// Register registers a cleanup function +// Note: the cleanup function will be executed in reverse order (the last registered will be executed first) +func (c *ResourceCleaner) Register(cleanup types.CleanupFunc) { + if cleanup == nil { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + + c.cleanups = append(c.cleanups, cleanup) +} + +// RegisterWithName registers a cleanup function with a name, for logging tracking +func (c *ResourceCleaner) RegisterWithName(name string, cleanup types.CleanupFunc) { + if cleanup == nil { + return + } + + wrappedCleanup := func() error { + log.Printf("Cleaning up resource: %s", name) + err := cleanup() + if err != nil { + log.Printf("Error cleaning up resource %s: %v", name, err) + } else { + log.Printf("Successfully cleaned up resource: %s", name) + } + return err + } + + c.Register(wrappedCleanup) +} + +// Cleanup executes all cleanup functions +// Even if a cleanup function fails, other cleanup functions will still be executed +func (c *ResourceCleaner) Cleanup(ctx context.Context) (errs []error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Execute cleanup functions in reverse order (the last registered will be executed first) + for i := len(c.cleanups) - 1; i >= 0; i-- { + select { + case <-ctx.Done(): + errs = append(errs, ctx.Err()) + return errs + default: + if err := c.cleanups[i](); err != nil { + errs = append(errs, err) + } + } + } + + return errs +} + +// Reset clears all registered cleanup functions +func (c *ResourceCleaner) Reset() { + c.mu.Lock() + defer c.mu.Unlock() + + c.cleanups = make([]types.CleanupFunc, 0) +} diff --git a/internal/container/container.go b/internal/container/container.go new file mode 100644 index 0000000..4a686e6 --- /dev/null +++ b/internal/container/container.go @@ -0,0 +1,418 @@ +// Package container implements dependency injection container setup +// Provides centralized configuration for services, repositories, and handlers +// This package is responsible for wiring up all dependencies and ensuring proper lifecycle management +package container + +import ( + "context" + "fmt" + "os" + "slices" + "strconv" + "strings" + "time" + + esv7 "github.com/elastic/go-elasticsearch/v7" + "github.com/elastic/go-elasticsearch/v8" + "github.com/neo4j/neo4j-go-driver/v6/neo4j" + "github.com/panjf2000/ants/v2" + "go.uber.org/dig" + "gorm.io/driver/postgres" + "gorm.io/gorm" + + "github.com/Tencent/WeKnora/internal/application/repository" + elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7" + elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8" + neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j" + postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres" + "github.com/Tencent/WeKnora/internal/application/service" + chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline" + "github.com/Tencent/WeKnora/internal/application/service/file" + "github.com/Tencent/WeKnora/internal/application/service/retriever" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/handler" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/models/utils/ollama" + "github.com/Tencent/WeKnora/internal/router" + "github.com/Tencent/WeKnora/internal/stream" + "github.com/Tencent/WeKnora/internal/tracing" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/services/docreader/src/client" +) + +// BuildContainer constructs the dependency injection container +// Registers all components, services, repositories and handlers needed by the application +// Creates a fully configured application container with proper dependency resolution +// Parameters: +// - container: Base dig container to add dependencies to +// +// Returns: +// - Configured container with all application dependencies registered +func BuildContainer(container *dig.Container) *dig.Container { + // Register resource cleaner for proper cleanup of resources + must(container.Provide(NewResourceCleaner, dig.As(new(interfaces.ResourceCleaner)))) + + // Core infrastructure configuration + must(container.Provide(config.LoadConfig)) + must(container.Provide(initTracer)) + must(container.Provide(initDatabase)) + must(container.Provide(initFileService)) + must(container.Provide(initAntsPool)) + + // Register goroutine pool cleanup handler + must(container.Invoke(registerPoolCleanup)) + + // Initialize retrieval engine registry for search capabilities + must(container.Provide(initRetrieveEngineRegistry)) + + // External service clients + must(container.Provide(initDocReaderClient)) + must(container.Provide(initOllamaService)) + must(container.Provide(initNeo4jClient)) + must(container.Provide(stream.NewStreamManager)) + + // Data repositories layer + must(container.Provide(repository.NewTenantRepository)) + must(container.Provide(repository.NewKnowledgeBaseRepository)) + must(container.Provide(repository.NewKnowledgeRepository)) + must(container.Provide(repository.NewChunkRepository)) + must(container.Provide(repository.NewSessionRepository)) + must(container.Provide(repository.NewMessageRepository)) + must(container.Provide(repository.NewModelRepository)) + must(container.Provide(repository.NewUserRepository)) + must(container.Provide(repository.NewAuthTokenRepository)) + must(container.Provide(neo4jRepo.NewNeo4jRepository)) + + // Business service layer + must(container.Provide(service.NewTenantService)) + must(container.Provide(service.NewKnowledgeBaseService)) + must(container.Provide(service.NewKnowledgeService)) + must(container.Provide(service.NewSessionService)) + must(container.Provide(service.NewMessageService)) + must(container.Provide(service.NewChunkService)) + must(container.Provide(embedding.NewBatchEmbedder)) + must(container.Provide(service.NewModelService)) + must(container.Provide(service.NewDatasetService)) + must(container.Provide(service.NewEvaluationService)) + must(container.Provide(service.NewUserService)) + must(container.Provide(service.NewChunkExtractService)) + + // Chat pipeline components for processing chat requests + must(container.Provide(chatpipline.NewEventManager)) + must(container.Invoke(chatpipline.NewPluginTracing)) + must(container.Invoke(chatpipline.NewPluginSearch)) + must(container.Invoke(chatpipline.NewPluginRerank)) + must(container.Invoke(chatpipline.NewPluginMerge)) + must(container.Invoke(chatpipline.NewPluginIntoChatMessage)) + must(container.Invoke(chatpipline.NewPluginChatCompletion)) + must(container.Invoke(chatpipline.NewPluginChatCompletionStream)) + must(container.Invoke(chatpipline.NewPluginStreamFilter)) + must(container.Invoke(chatpipline.NewPluginFilterTopK)) + must(container.Invoke(chatpipline.NewPluginPreprocess)) + must(container.Invoke(chatpipline.NewPluginRewrite)) + must(container.Invoke(chatpipline.NewPluginExtractEntity)) + must(container.Invoke(chatpipline.NewPluginSearchEntity)) + + // HTTP handlers layer + must(container.Provide(handler.NewTenantHandler)) + must(container.Provide(handler.NewKnowledgeBaseHandler)) + must(container.Provide(handler.NewKnowledgeHandler)) + must(container.Provide(handler.NewChunkHandler)) + must(container.Provide(handler.NewSessionHandler)) + must(container.Provide(handler.NewMessageHandler)) + must(container.Provide(handler.NewModelHandler)) + must(container.Provide(handler.NewEvaluationHandler)) + must(container.Provide(handler.NewInitializationHandler)) + must(container.Provide(handler.NewAuthHandler)) + must(container.Provide(handler.NewSystemHandler)) + + // Router configuration + must(container.Provide(router.NewRouter)) + must(container.Provide(router.NewAsyncqClient)) + must(container.Provide(router.NewAsynqServer)) + must(container.Invoke(router.RunAsynqServer)) + + return container +} + +// must is a helper function for error handling +// Panics if the error is not nil, useful for configuration steps that must succeed +// Parameters: +// - err: Error to check +func must(err error) { + if err != nil { + panic(err) + } +} + +// initTracer initializes OpenTelemetry tracer +// Sets up distributed tracing for observability across the application +// Parameters: +// - None +// +// Returns: +// - Configured tracer instance +// - Error if initialization fails +func initTracer() (*tracing.Tracer, error) { + return tracing.InitTracer() +} + +// initDatabase initializes database connection +// Creates and configures database connection based on environment configuration +// Supports multiple database backends (PostgreSQL) +// Parameters: +// - cfg: Application configuration +// +// Returns: +// - Configured database connection +// - Error if connection fails +func initDatabase(cfg *config.Config) (*gorm.DB, error) { + var dialector gorm.Dialector + switch os.Getenv("DB_DRIVER") { + case "postgres": + dsn := fmt.Sprintf( + "host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", + os.Getenv("DB_HOST"), + os.Getenv("DB_PORT"), + os.Getenv("DB_USER"), + os.Getenv("DB_PASSWORD"), + os.Getenv("DB_NAME"), + "disable", + ) + dialector = postgres.Open(dsn) + default: + return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER")) + } + db, err := gorm.Open(dialector, &gorm.Config{}) + if err != nil { + return nil, err + } + + // Auto-migrate database tables + err = db.AutoMigrate( + &types.User{}, + &types.AuthToken{}, + &types.KnowledgeBase{}, + ) + if err != nil { + return nil, fmt.Errorf("failed to auto-migrate database tables: %v", err) + } + + // Get underlying SQL DB object + sqlDB, err := db.DB() + if err != nil { + return nil, err + } + + // Configure connection pool parameters + sqlDB.SetMaxIdleConns(10) + sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute) + + return db, nil +} + +// initFileService initializes file storage service +// Creates the appropriate file storage service based on configuration +// Supports multiple storage backends (MinIO, COS, local filesystem) +// Parameters: +// - cfg: Application configuration +// +// Returns: +// - Configured file service implementation +// - Error if initialization fails +func initFileService(cfg *config.Config) (interfaces.FileService, error) { + switch os.Getenv("STORAGE_TYPE") { + case "minio": + if os.Getenv("MINIO_ENDPOINT") == "" || + os.Getenv("MINIO_ACCESS_KEY_ID") == "" || + os.Getenv("MINIO_SECRET_ACCESS_KEY") == "" || + os.Getenv("MINIO_BUCKET_NAME") == "" { + return nil, fmt.Errorf("missing MinIO configuration") + } + return file.NewMinioFileService( + os.Getenv("MINIO_ENDPOINT"), + os.Getenv("MINIO_ACCESS_KEY_ID"), + os.Getenv("MINIO_SECRET_ACCESS_KEY"), + os.Getenv("MINIO_BUCKET_NAME"), + false, + ) + case "cos": + if os.Getenv("COS_BUCKET_NAME") == "" || + os.Getenv("COS_REGION") == "" || + os.Getenv("COS_SECRET_ID") == "" || + os.Getenv("COS_SECRET_KEY") == "" || + os.Getenv("COS_PATH_PREFIX") == "" { + return nil, fmt.Errorf("missing COS configuration") + } + return file.NewCosFileService( + os.Getenv("COS_BUCKET_NAME"), + os.Getenv("COS_REGION"), + os.Getenv("COS_SECRET_ID"), + os.Getenv("COS_SECRET_KEY"), + os.Getenv("COS_PATH_PREFIX"), + ) + case "local": + return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil + case "dummy": + return file.NewDummyFileService(), nil + default: + return nil, fmt.Errorf("unsupported storage type: %s", os.Getenv("STORAGE_TYPE")) + } +} + +// initRetrieveEngineRegistry initializes the retrieval engine registry +// Sets up and configures various search engine backends based on configuration +// Supports multiple retrieval engines (PostgreSQL, ElasticsearchV7, ElasticsearchV8) +// Parameters: +// - db: Database connection +// - cfg: Application configuration +// +// Returns: +// - Configured retrieval engine registry +// - Error if initialization fails +func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.RetrieveEngineRegistry, error) { + registry := retriever.NewRetrieveEngineRegistry() + retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") + log := logger.GetLogger(context.Background()) + + if slices.Contains(retrieveDriver, "postgres") { + postgresRepo := postgresRepo.NewPostgresRetrieveEngineRepository(db) + if err := registry.Register( + retriever.NewKVHybridRetrieveEngine(postgresRepo, types.PostgresRetrieverEngineType), + ); err != nil { + log.Errorf("Register postgres retrieve engine failed: %v", err) + } else { + log.Infof("Register postgres retrieve engine success") + } + } + if slices.Contains(retrieveDriver, "elasticsearch_v8") { + client, err := elasticsearch.NewTypedClient(elasticsearch.Config{ + Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")}, + Username: os.Getenv("ELASTICSEARCH_USERNAME"), + Password: os.Getenv("ELASTICSEARCH_PASSWORD"), + }) + if err != nil { + log.Errorf("Create elasticsearch_v8 client failed: %v", err) + } else { + elasticsearchRepo := elasticsearchRepoV8.NewElasticsearchEngineRepository(client, cfg) + if err := registry.Register( + retriever.NewKVHybridRetrieveEngine( + elasticsearchRepo, types.ElasticsearchRetrieverEngineType, + ), + ); err != nil { + log.Errorf("Register elasticsearch_v8 retrieve engine failed: %v", err) + } else { + log.Infof("Register elasticsearch_v8 retrieve engine success") + } + } + } + + if slices.Contains(retrieveDriver, "elasticsearch_v7") { + client, err := esv7.NewClient(esv7.Config{ + Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")}, + Username: os.Getenv("ELASTICSEARCH_USERNAME"), + Password: os.Getenv("ELASTICSEARCH_PASSWORD"), + }) + if err != nil { + log.Errorf("Create elasticsearch_v7 client failed: %v", err) + } else { + elasticsearchRepo := elasticsearchRepoV7.NewElasticsearchEngineRepository(client, cfg) + if err := registry.Register( + retriever.NewKVHybridRetrieveEngine( + elasticsearchRepo, types.ElasticsearchRetrieverEngineType, + ), + ); err != nil { + log.Errorf("Register elasticsearch_v7 retrieve engine failed: %v", err) + } else { + log.Infof("Register elasticsearch_v7 retrieve engine success") + } + } + } + return registry, nil +} + +// initAntsPool initializes the goroutine pool +// Creates a managed goroutine pool for concurrent task execution +// Parameters: +// - cfg: Application configuration +// +// Returns: +// - Configured goroutine pool +// - Error if initialization fails +func initAntsPool(cfg *config.Config) (*ants.Pool, error) { + // Default to 5 if not specified in config + poolSize := os.Getenv("CONCURRENCY_POOL_SIZE") + if poolSize == "" { + poolSize = "5" + } + poolSizeInt, err := strconv.Atoi(poolSize) + if err != nil { + return nil, err + } + // Set up the pool with pre-allocation for better performance + return ants.NewPool(poolSizeInt, ants.WithPreAlloc(true)) +} + +// registerPoolCleanup registers the goroutine pool for cleanup +// Ensures proper cleanup of the goroutine pool when application shuts down +// Parameters: +// - pool: Goroutine pool +// - cleaner: Resource cleaner +func registerPoolCleanup(pool *ants.Pool, cleaner interfaces.ResourceCleaner) { + cleaner.RegisterWithName("AntsPool", func() error { + pool.Release() + return nil + }) +} + +// initDocReaderClient initializes the document reader client +// Creates a client for interacting with the document reader service +// Parameters: +// - cfg: Application configuration +// +// Returns: +// - Configured document reader client +// - Error if initialization fails +func initDocReaderClient(cfg *config.Config) (*client.Client, error) { + // Use the DocReader URL from environment or config + docReaderURL := os.Getenv("DOCREADER_ADDR") + if docReaderURL == "" && cfg.DocReader != nil { + docReaderURL = cfg.DocReader.Addr + } + return client.NewClient(docReaderURL) +} + +// initOllamaService initializes the Ollama service client +// Creates a client for interacting with Ollama API for model inference +// Parameters: +// - None +// +// Returns: +// - Configured Ollama service client +// - Error if initialization fails +func initOllamaService() (*ollama.OllamaService, error) { + // Get Ollama service from existing factory function + return ollama.GetOllamaService() +} + +func initNeo4jClient() (neo4j.Driver, error) { + if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" { + logger.Debugf(context.Background(), "NOT SUPPORT RETRIEVE GRAPH") + return nil, nil + } + uri := os.Getenv("NEO4J_URI") + username := os.Getenv("NEO4J_USERNAME") + password := os.Getenv("NEO4J_PASSWORD") + + driver, err := neo4j.NewDriver(uri, neo4j.BasicAuth(username, password, "")) + if err != nil { + return nil, err + } + err = driver.VerifyAuthentication(context.Background(), nil) + if err != nil { + return nil, err + } + return driver, nil +} diff --git a/internal/errors/errors.go b/internal/errors/errors.go new file mode 100644 index 0000000..c682e4a --- /dev/null +++ b/internal/errors/errors.go @@ -0,0 +1,152 @@ +package errors + +import ( + "fmt" + "net/http" +) + +// ErrorCode defines the error code type +type ErrorCode int + +// System error codes +const ( + // Common error codes (1000-1999) + ErrBadRequest ErrorCode = 1000 + ErrUnauthorized ErrorCode = 1001 + ErrForbidden ErrorCode = 1002 + ErrNotFound ErrorCode = 1003 + ErrMethodNotAllowed ErrorCode = 1004 + ErrConflict ErrorCode = 1005 + ErrTooManyRequests ErrorCode = 1006 + ErrInternalServer ErrorCode = 1007 + ErrServiceUnavailable ErrorCode = 1008 + ErrTimeout ErrorCode = 1009 + ErrValidation ErrorCode = 1010 + + // Tenant related error codes (2000-2099) + ErrTenantNotFound ErrorCode = 2000 + ErrTenantAlreadyExists ErrorCode = 2001 + ErrTenantInactive ErrorCode = 2002 + ErrTenantNameRequired ErrorCode = 2003 + ErrTenantInvalidStatus ErrorCode = 2004 + + // Add more error codes here +) + +// AppError defines the application error structure +type AppError struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` + Details any `json:"details,omitempty"` + HTTPCode int `json:"-"` +} + +// Error implements the error interface +func (e *AppError) Error() string { + return fmt.Sprintf("error code: %d, error message: %s", e.Code, e.Message) +} + +// WithDetails adds error details +func (e *AppError) WithDetails(details any) *AppError { + e.Details = details + return e +} + +// NewBadRequestError creates a bad request error +func NewBadRequestError(message string) *AppError { + return &AppError{ + Code: ErrBadRequest, + Message: message, + HTTPCode: http.StatusBadRequest, + } +} + +// NewUnauthorizedError creates an unauthorized error +func NewUnauthorizedError(message string) *AppError { + return &AppError{ + Code: ErrUnauthorized, + Message: message, + HTTPCode: http.StatusUnauthorized, + } +} + +// NewForbiddenError creates a forbidden error +func NewForbiddenError(message string) *AppError { + return &AppError{ + Code: ErrForbidden, + Message: message, + HTTPCode: http.StatusForbidden, + } +} + +// NewNotFoundError creates a not found error +func NewNotFoundError(message string) *AppError { + return &AppError{ + Code: ErrNotFound, + Message: message, + HTTPCode: http.StatusNotFound, + } +} + +// NewConflictError creates a conflict error +func NewConflictError(message string) *AppError { + return &AppError{ + Code: ErrConflict, + Message: message, + HTTPCode: http.StatusConflict, + } +} + +// NewInternalServerError creates an internal server error +func NewInternalServerError(message string) *AppError { + if message == "" { + message = "服务器内部错误" + } + return &AppError{ + Code: ErrInternalServer, + Message: message, + HTTPCode: http.StatusInternalServerError, + } +} + +// NewValidationError creates a validation error +func NewValidationError(message string) *AppError { + return &AppError{ + Code: ErrValidation, + Message: message, + HTTPCode: http.StatusBadRequest, + } +} + +// Tenant related errors +func NewTenantNotFoundError() *AppError { + return &AppError{ + Code: ErrTenantNotFound, + Message: "租户不存在", + HTTPCode: http.StatusNotFound, + } +} + +// NewTenantAlreadyExistsError creates a tenant already exists error +func NewTenantAlreadyExistsError() *AppError { + return &AppError{ + Code: ErrTenantAlreadyExists, + Message: "租户已存在", + HTTPCode: http.StatusConflict, + } +} + +// NewTenantInactiveError creates a tenant inactive error +func NewTenantInactiveError() *AppError { + return &AppError{ + Code: ErrTenantInactive, + Message: "租户已停用", + HTTPCode: http.StatusForbidden, + } +} + +// IsAppError checks if the error is an AppError type +func IsAppError(err error) (*AppError, bool) { + appErr, ok := err.(*AppError) + return appErr, ok +} diff --git a/internal/errors/session.go b/internal/errors/session.go new file mode 100644 index 0000000..8e9d3b1 --- /dev/null +++ b/internal/errors/session.go @@ -0,0 +1,16 @@ +package errors + +import "errors" + +var ( + // ErrSessionNotFound session not found error + ErrSessionNotFound = errors.New("session not found") + // ErrSessionExpired session expired error + ErrSessionExpired = errors.New("session expired") + // ErrSessionLimitExceeded session limit exceeded error + ErrSessionLimitExceeded = errors.New("session limit exceeded") + // ErrInvalidSessionID invalid session ID error + ErrInvalidSessionID = errors.New("invalid session id") + // ErrInvalidTenantID invalid tenant ID error + ErrInvalidTenantID = errors.New("invalid tenant id") +) diff --git a/internal/handler/auth.go b/internal/handler/auth.go new file mode 100644 index 0000000..622e9fb --- /dev/null +++ b/internal/handler/auth.go @@ -0,0 +1,343 @@ +package handler + +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// AuthHandler implements HTTP request handlers for user authentication +// Provides functionality for user registration, login, logout, and token management +// through the REST API endpoints +type AuthHandler struct { + userService interfaces.UserService + tenantService interfaces.TenantService +} + +// NewAuthHandler creates a new auth handler instance with the provided services +// Parameters: +// - userService: An implementation of the UserService interface for business logic +// - tenantService: An implementation of the TenantService interface for tenant management +// +// Returns a pointer to the newly created AuthHandler +func NewAuthHandler(userService interfaces.UserService, tenantService interfaces.TenantService) *AuthHandler { + return &AuthHandler{ + userService: userService, + tenantService: tenantService, + } +} + +// Register handles the HTTP request for user registration +// It deserializes the request body into a registration request object, validates it, +// calls the service to create the user, and returns the result +// Parameters: +// - c: Gin context for the HTTP request +func (h *AuthHandler) Register(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start user registration") + + var req types.RegisterRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse registration request parameters", err) + appErr := errors.NewValidationError("Invalid registration parameters").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Validate required fields + if req.Username == "" || req.Email == "" || req.Password == "" { + logger.Error(ctx, "Missing required registration fields") + appErr := errors.NewValidationError("Username, email and password are required") + c.Error(appErr) + return + } + + // Call service to register user + user, err := h.userService.Register(ctx, &req) + if err != nil { + logger.Errorf(ctx, "Failed to register user: %v", err) + appErr := errors.NewBadRequestError("Registration failed").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Return success response + response := &types.RegisterResponse{ + Success: true, + Message: "Registration successful", + User: user, + } + + logger.Infof(ctx, "User registered successfully: %s", user.Email) + c.JSON(http.StatusCreated, response) +} + +// Login handles the HTTP request for user login +// It deserializes the request body into a login request object, validates it, +// calls the service to authenticate the user, and returns tokens +// Parameters: +// - c: Gin context for the HTTP request +func (h *AuthHandler) Login(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start user login") + + var req types.LoginRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse login request parameters", err) + appErr := errors.NewValidationError("Invalid login parameters").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Validate required fields + if req.Email == "" || req.Password == "" { + logger.Error(ctx, "Missing required login fields") + appErr := errors.NewValidationError("Email and password are required") + c.Error(appErr) + return + } + + // Call service to authenticate user + response, err := h.userService.Login(ctx, &req) + if err != nil { + logger.Errorf(ctx, "Failed to login user: %v", err) + appErr := errors.NewUnauthorizedError("Login failed").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Check if login was successful + if !response.Success { + logger.Warnf(ctx, "Login failed: %s", response.Message) + c.JSON(http.StatusUnauthorized, response) + return + } + + // User is already in the correct format from service + + logger.Infof(ctx, "User logged in successfully: %s", req.Email) + c.JSON(http.StatusOK, response) +} + +// Logout handles the HTTP request for user logout +// It extracts the token from the Authorization header and revokes it +// Parameters: +// - c: Gin context for the HTTP request +func (h *AuthHandler) Logout(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start user logout") + + // Extract token from Authorization header + authHeader := c.GetHeader("Authorization") + if authHeader == "" { + logger.Error(ctx, "Missing Authorization header") + appErr := errors.NewValidationError("Authorization header is required") + c.Error(appErr) + return + } + + // Parse Bearer token + tokenParts := strings.Split(authHeader, " ") + if len(tokenParts) != 2 || tokenParts[0] != "Bearer" { + logger.Error(ctx, "Invalid Authorization header format") + appErr := errors.NewValidationError("Invalid Authorization header format") + c.Error(appErr) + return + } + + token := tokenParts[1] + + // Revoke token + err := h.userService.RevokeToken(ctx, token) + if err != nil { + logger.Errorf(ctx, "Failed to revoke token: %v", err) + appErr := errors.NewInternalServerError("Logout failed").WithDetails(err.Error()) + c.Error(appErr) + return + } + + logger.Info(ctx, "User logged out successfully") + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Logout successful", + }) +} + +// RefreshToken handles the HTTP request for refreshing access tokens +// It extracts the refresh token from the request body and generates new tokens +// Parameters: +// - c: Gin context for the HTTP request +func (h *AuthHandler) RefreshToken(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start token refresh") + + var req struct { + RefreshToken string `json:"refreshToken" binding:"required"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse refresh token request", err) + appErr := errors.NewValidationError("Invalid refresh token request").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Call service to refresh token + accessToken, newRefreshToken, err := h.userService.RefreshToken(ctx, req.RefreshToken) + if err != nil { + logger.Errorf(ctx, "Failed to refresh token: %v", err) + appErr := errors.NewUnauthorizedError("Token refresh failed").WithDetails(err.Error()) + c.Error(appErr) + return + } + + logger.Info(ctx, "Token refreshed successfully") + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Token refreshed successfully", + "access_token": accessToken, + "refresh_token": newRefreshToken, + }) +} + +// GetCurrentUser handles the HTTP request for getting current user information +// It extracts the user from the context (set by auth middleware) and returns user info +// Parameters: +// - c: Gin context for the HTTP request +func (h *AuthHandler) GetCurrentUser(c *gin.Context) { + ctx := c.Request.Context() + + logger.Debugf(ctx, "Get current user info") + + // Get current user from service (which extracts from context) + user, err := h.userService.GetCurrentUser(ctx) + if err != nil { + logger.Errorf(ctx, "Failed to get current user: %v", err) + appErr := errors.NewUnauthorizedError("Failed to get user information").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Get tenant information + var tenant *types.Tenant + if user.TenantID > 0 { + tenant, err = h.tenantService.GetTenantByID(ctx, user.TenantID) + if err != nil { + logger.Warnf(ctx, "Failed to get tenant info for user %s, tenant ID %d: %v", user.Email, user.TenantID, err) + // Don't fail the request if tenant info is not available + } else { + logger.Debugf(ctx, "Retrieved tenant info for user %s: %s", user.Email, tenant.Name) + } + } + + logger.Debugf(ctx, "Retrieved current user info: %s", user.Email) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "user": user.ToUserInfo(), + "tenant": tenant, + }, + }) +} + +// ChangePassword handles the HTTP request for changing user password +// It extracts the current user and validates the old password before setting new one +// Parameters: +// - c: Gin context for the HTTP request +func (h *AuthHandler) ChangePassword(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start password change") + + var req struct { + OldPassword string `json:"old_password" binding:"required"` + NewPassword string `json:"new_password" binding:"required,min=6"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse password change request", err) + appErr := errors.NewValidationError("Invalid password change request").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Get current user + user, err := h.userService.GetCurrentUser(ctx) + if err != nil { + logger.Errorf(ctx, "Failed to get current user: %v", err) + appErr := errors.NewUnauthorizedError("Failed to get user information").WithDetails(err.Error()) + c.Error(appErr) + return + } + + // Change password + err = h.userService.ChangePassword(ctx, user.ID, req.OldPassword, req.NewPassword) + if err != nil { + logger.Errorf(ctx, "Failed to change password: %v", err) + appErr := errors.NewBadRequestError("Password change failed").WithDetails(err.Error()) + c.Error(appErr) + return + } + + logger.Infof(ctx, "Password changed successfully for user: %s", user.Email) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Password changed successfully", + }) +} + +// ValidateToken handles the HTTP request for validating access tokens +// It extracts the token from the Authorization header and validates it +// Parameters: +// - c: Gin context for the HTTP request +func (h *AuthHandler) ValidateToken(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start token validation") + + // Extract token from Authorization header + authHeader := c.GetHeader("Authorization") + if authHeader == "" { + logger.Error(ctx, "Missing Authorization header") + appErr := errors.NewValidationError("Authorization header is required") + c.Error(appErr) + return + } + + // Parse Bearer token + tokenParts := strings.Split(authHeader, " ") + if len(tokenParts) != 2 || tokenParts[0] != "Bearer" { + logger.Error(ctx, "Invalid Authorization header format") + appErr := errors.NewValidationError("Invalid Authorization header format") + c.Error(appErr) + return + } + + token := tokenParts[1] + + // Validate token + user, err := h.userService.ValidateToken(ctx, token) + if err != nil { + logger.Errorf(ctx, "Failed to validate token: %v", err) + appErr := errors.NewUnauthorizedError("Token validation failed").WithDetails(err.Error()) + c.Error(appErr) + return + } + + logger.Infof(ctx, "Token validated successfully for user: %s", user.Email) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Token is valid", + "user": user.ToUserInfo(), + }) +} diff --git a/internal/handler/chunk.go b/internal/handler/chunk.go new file mode 100644 index 0000000..40d7b5e --- /dev/null +++ b/internal/handler/chunk.go @@ -0,0 +1,241 @@ +package handler + +import ( + "net/http" + + "github.com/Tencent/WeKnora/internal/application/service" + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" + "github.com/gin-gonic/gin" +) + +// ChunkHandler defines HTTP handlers for chunk operations +type ChunkHandler struct { + service interfaces.ChunkService +} + +// NewChunkHandler creates a new chunk handler +func NewChunkHandler(service interfaces.ChunkService) *ChunkHandler { + return &ChunkHandler{service: service} +} + +// ListKnowledgeChunks lists all chunks for a given knowledge ID +func (h *ChunkHandler) ListKnowledgeChunks(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start retrieving knowledge chunks list") + + knowledgeID := c.Param("knowledge_id") + if knowledgeID == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + + // Parse pagination parameters + var pagination types.Pagination + if err := c.ShouldBindQuery(&pagination); err != nil { + logger.Error(ctx, "Failed to parse pagination parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + logger.Infof(ctx, "Retrieving knowledge chunks list, knowledge ID: %s, page: %d, page size: %d", + knowledgeID, pagination.Page, pagination.PageSize) + + // Use pagination for query + result, err := h.service.ListPagedChunksByKnowledgeID(ctx, knowledgeID, &pagination) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // 对 chunk 内容进行安全清理 + for _, chunk := range result.Data.([]*types.Chunk) { + if chunk.Content != "" { + chunk.Content = secutils.SanitizeForDisplay(chunk.Content) + } + } + + logger.Infof( + ctx, "Successfully retrieved knowledge chunks list, knowledge ID: %s, total: %d", + knowledgeID, result.Total, + ) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result.Data, + "total": result.Total, + "page": result.Page, + "page_size": result.PageSize, + }) +} + +// UpdateChunkRequest defines the request structure for updating a chunk +type UpdateChunkRequest struct { + Content string `json:"content"` + Embedding []float32 `json:"embedding"` + ChunkIndex int `json:"chunk_index"` + IsEnabled bool `json:"is_enabled"` + StartAt int `json:"start_at"` + EndAt int `json:"end_at"` + ImageInfo string `json:"image_info"` +} + +// validateAndGetChunk validates request parameters and retrieves the chunk +// Returns chunk information, knowledge ID, and error +func (h *ChunkHandler) validateAndGetChunk(c *gin.Context) (*types.Chunk, string, error) { + ctx := c.Request.Context() + + // Validate knowledge ID + knowledgeID := c.Param("knowledge_id") + if knowledgeID == "" { + logger.Error(ctx, "Knowledge ID is empty") + return nil, "", errors.NewBadRequestError("Knowledge ID cannot be empty") + } + + // Validate chunk ID + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Chunk ID is empty") + return nil, knowledgeID, errors.NewBadRequestError("Chunk ID cannot be empty") + } + + // Get tenant ID from context + tenantID, exists := c.Get(types.TenantIDContextKey.String()) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + return nil, knowledgeID, errors.NewUnauthorizedError("Unauthorized") + } + + logger.Infof(ctx, "Retrieving knowledge chunk information, knowledge ID: %s, chunk ID: %s", knowledgeID, id) + + // Get existing chunk + chunk, err := h.service.GetChunkByID(ctx, knowledgeID, id) + if err != nil { + if err == service.ErrChunkNotFound { + logger.Warnf(ctx, "Chunk not found, knowledge ID: %s, chunk ID: %s", knowledgeID, id) + return nil, knowledgeID, errors.NewNotFoundError("Chunk not found") + } + logger.ErrorWithFields(ctx, err, nil) + return nil, knowledgeID, errors.NewInternalServerError(err.Error()) + } + + // Validate tenant ID + if chunk.TenantID != tenantID.(uint) || chunk.KnowledgeID != knowledgeID { + logger.Warnf( + ctx, + "Tenant has no permission to access chunk, knowledge ID: %s, chunk ID: %s, req tenant: %d, chunk tenant: %d", + knowledgeID, id, tenantID.(uint), chunk.TenantID, + ) + return nil, knowledgeID, errors.NewForbiddenError("No permission to access this chunk") + } + + return chunk, knowledgeID, nil +} + +// UpdateChunk updates a chunk's properties +func (h *ChunkHandler) UpdateChunk(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start updating knowledge chunk") + + // Validate parameters and get chunk + chunk, knowledgeID, err := h.validateAndGetChunk(c) + if err != nil { + c.Error(err) + return + } + var req UpdateChunkRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // Update chunk properties + if req.Content != "" { + chunk.Content = req.Content + } + + chunk.IsEnabled = req.IsEnabled + + logger.Infof(ctx, "Updating knowledge chunk, knowledge ID: %s, chunk ID: %s", knowledgeID, chunk.ID) + + if err := h.service.UpdateChunk(ctx, chunk); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge chunk updated successfully, knowledge ID: %s, chunk ID: %s", knowledgeID, chunk.ID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": chunk, + }) +} + +// DeleteChunk deletes a specific chunk +func (h *ChunkHandler) DeleteChunk(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start deleting knowledge chunk") + + // Validate parameters and get chunk + chunk, knowledgeID, err := h.validateAndGetChunk(c) + if err != nil { + c.Error(err) + return + } + + logger.Infof(ctx, "Deleting knowledge chunk, knowledge ID: %s, chunk ID: %s", knowledgeID, chunk.ID) + + if err := h.service.DeleteChunk(ctx, chunk.ID); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge chunk deleted successfully, knowledge ID: %s, chunk ID: %s", knowledgeID, chunk.ID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Chunk deleted", + }) +} + +// DeleteChunksByKnowledgeID deletes all chunks for a given knowledge ID +func (h *ChunkHandler) DeleteChunksByKnowledgeID(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start deleting all chunks under knowledge") + + knowledgeID := c.Param("knowledge_id") + if knowledgeID == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + + // Get tenant ID from context + tenantID, exists := c.Get(types.TenantIDContextKey.String()) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + c.Error(errors.NewUnauthorizedError("Unauthorized")) + return + } + + logger.Infof(ctx, "Deleting all chunks under knowledge, knowledge ID: %s, tenant ID: %d", knowledgeID, tenantID.(uint)) + + // Delete all chunks under the knowledge + err := h.service.DeleteChunksByKnowledgeID(ctx, knowledgeID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "All chunks under knowledge deleted successfully, knowledge ID: %s", knowledgeID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "All chunks under knowledge deleted", + }) +} diff --git a/internal/handler/evaluation.go b/internal/handler/evaluation.go new file mode 100644 index 0000000..ac62092 --- /dev/null +++ b/internal/handler/evaluation.go @@ -0,0 +1,105 @@ +package handler + +import ( + "net/http" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// EvaluationHandler handles evaluation related HTTP requests +type EvaluationHandler struct { + evaluationService interfaces.EvaluationService // Service for evaluation operations +} + +// NewEvaluationHandler creates a new EvaluationHandler instance +func NewEvaluationHandler(evaluationService interfaces.EvaluationService) *EvaluationHandler { + return &EvaluationHandler{evaluationService: evaluationService} +} + +// EvaluationRequest contains parameters for evaluation request +type EvaluationRequest struct { + DatasetID string `json:"dataset_id"` // ID of dataset to evaluate + KnowledgeBaseID string `json:"knowledge_base_id"` // ID of knowledge base to use + ChatModelID string `json:"chat_id"` // ID of chat model to use + RerankModelID string `json:"rerank_id"` // ID of rerank model to use +} + +// Evaluation handles evaluation request +func (e *EvaluationHandler) Evaluation(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start processing evaluation request") + + var request *EvaluationRequest + if err := c.ShouldBind(&request); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error())) + return + } + + tenantID, exists := c.Get(string(types.TenantIDContextKey)) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + c.Error(errors.NewUnauthorizedError("Unauthorized")) + return + } + + logger.Infof(ctx, "Executing evaluation, tenant: %v, dataset: %s, knowledge_base: %s, chat: %s, rerank: %s", + tenantID, request.DatasetID, request.KnowledgeBaseID, request.ChatModelID, request.RerankModelID) + + task, err := e.evaluationService.Evaluation(ctx, + request.DatasetID, + request.KnowledgeBaseID, + request.ChatModelID, + request.RerankModelID, + ) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Evaluation task created successfully") + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": task, + }) +} + +// GetEvaluationRequest contains parameters for getting evaluation result +type GetEvaluationRequest struct { + TaskID string `form:"task_id" binding:"required"` // ID of evaluation task +} + +// GetEvaluationResult retrieves evaluation result by task ID +func (e *EvaluationHandler) GetEvaluationResult(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving evaluation result") + + var request *GetEvaluationRequest + if err := c.ShouldBind(&request); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error())) + return + } + + logger.Infof(ctx, "Retrieving evaluation result, task ID: %s", request.TaskID) + + result, err := e.evaluationService.EvaluationResult(ctx, request.TaskID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Retrieved evaluation result successfully, task ID: %s", request.TaskID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go new file mode 100644 index 0000000..e82a0d9 --- /dev/null +++ b/internal/handler/initialization.go @@ -0,0 +1,1775 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" + + chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline" + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/models/utils/ollama" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/services/docreader/src/client" + "github.com/Tencent/WeKnora/services/docreader/src/proto" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/ollama/ollama/api" +) + +// DownloadTask 下载任务信息 +type DownloadTask struct { + ID string `json:"id"` + ModelName string `json:"modelName"` + Status string `json:"status"` // pending, downloading, completed, failed + Progress float64 `json:"progress"` + Message string `json:"message"` + StartTime time.Time `json:"startTime"` + EndTime *time.Time `json:"endTime,omitempty"` +} + +// 全局下载任务管理器 +var ( + downloadTasks = make(map[string]*DownloadTask) + tasksMutex sync.RWMutex +) + +// InitializationHandler 初始化处理器 +type InitializationHandler struct { + config *config.Config + tenantService interfaces.TenantService + modelService interfaces.ModelService + kbService interfaces.KnowledgeBaseService + kbRepository interfaces.KnowledgeBaseRepository + knowledgeService interfaces.KnowledgeService + ollamaService *ollama.OllamaService + docReaderClient *client.Client +} + +// NewInitializationHandler 创建初始化处理器 +func NewInitializationHandler( + config *config.Config, + tenantService interfaces.TenantService, + modelService interfaces.ModelService, + kbService interfaces.KnowledgeBaseService, + kbRepository interfaces.KnowledgeBaseRepository, + knowledgeService interfaces.KnowledgeService, + ollamaService *ollama.OllamaService, + docReaderClient *client.Client, +) *InitializationHandler { + return &InitializationHandler{ + config: config, + tenantService: tenantService, + modelService: modelService, + kbService: kbService, + kbRepository: kbRepository, + knowledgeService: knowledgeService, + ollamaService: ollamaService, + docReaderClient: docReaderClient, + } +} + +// InitializationRequest 初始化请求结构 +type InitializationRequest struct { + LLM struct { + Source string `json:"source" binding:"required"` + ModelName string `json:"modelName" binding:"required"` + BaseURL string `json:"baseUrl"` + APIKey string `json:"apiKey"` + } `json:"llm" binding:"required"` + + Embedding struct { + Source string `json:"source" binding:"required"` + ModelName string `json:"modelName" binding:"required"` + BaseURL string `json:"baseUrl"` + APIKey string `json:"apiKey"` + Dimension int `json:"dimension"` // 添加embedding维度字段 + } `json:"embedding" binding:"required"` + + Rerank struct { + Enabled bool `json:"enabled"` + ModelName string `json:"modelName"` + BaseURL string `json:"baseUrl"` + APIKey string `json:"apiKey"` + } `json:"rerank"` + + Multimodal struct { + Enabled bool `json:"enabled"` + VLM *struct { + ModelName string `json:"modelName"` + BaseURL string `json:"baseUrl"` + APIKey string `json:"apiKey"` + InterfaceType string `json:"interfaceType"` // "ollama" or "openai" + } `json:"vlm,omitempty"` + StorageType string `json:"storageType"` + COS *struct { + SecretID string `json:"secretId"` + SecretKey string `json:"secretKey"` + Region string `json:"region"` + BucketName string `json:"bucketName"` + AppID string `json:"appId"` + PathPrefix string `json:"pathPrefix"` + } `json:"cos,omitempty"` + Minio *struct { + BucketName string `json:"bucketName"` + PathPrefix string `json:"pathPrefix"` + } `json:"minio,omitempty"` + } `json:"multimodal"` + + DocumentSplitting struct { + ChunkSize int `json:"chunkSize" binding:"required,min=100,max=10000"` + ChunkOverlap int `json:"chunkOverlap" binding:"min=0"` + Separators []string `json:"separators" binding:"required,min=1"` + } `json:"documentSplitting" binding:"required"` + + NodeExtract struct { + Enabled bool `json:"enabled"` + Text string `json:"text"` + Tags []string `json:"tags"` + Nodes []struct { + Name string `json:"name"` + Attributes []string `json:"attributes"` + } `json:"nodes"` + Relations []struct { + Node1 string `json:"node1"` + Node2 string `json:"node2"` + Type string `json:"type"` + } `json:"relations"` + } `json:"nodeExtract"` +} + +// InitializeByKB 根据知识库ID执行配置更新 +func (h *InitializationHandler) InitializeByKB(c *gin.Context) { + ctx := c.Request.Context() + kbIdStr := c.Param("kbId") + + logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s", kbIdStr) + + var req InitializationRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse initialization request", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // 获取指定知识库信息 + kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": kbIdStr}) + c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error())) + return + } + + if kb == nil { + logger.Error(ctx, "Knowledge base not found", "kbId", kbIdStr) + c.Error(errors.NewNotFoundError("知识库不存在")) + return + } + + // 验证多模态配置（如果启用） + if req.Multimodal.Enabled { + storageType := strings.ToLower(req.Multimodal.StorageType) + if req.Multimodal.VLM == nil { + logger.Error(ctx, "Multimodal enabled but missing VLM configuration") + c.Error(errors.NewBadRequestError("启用多模态时需要配置VLM信息")) + return + } + if req.Multimodal.VLM.InterfaceType == "ollama" { + req.Multimodal.VLM.BaseURL = os.Getenv("OLLAMA_BASE_URL") + "/v1" + } + if req.Multimodal.VLM.ModelName == "" || req.Multimodal.VLM.BaseURL == "" { + logger.Error(ctx, "VLM configuration incomplete") + c.Error(errors.NewBadRequestError("VLM配置不完整")) + return + } + switch storageType { + case "cos": + if req.Multimodal.COS == nil || req.Multimodal.COS.SecretID == "" || req.Multimodal.COS.SecretKey == "" || + req.Multimodal.COS.Region == "" || req.Multimodal.COS.BucketName == "" || + req.Multimodal.COS.AppID == "" { + logger.Error(ctx, "COS configuration incomplete") + c.Error(errors.NewBadRequestError("COS配置不完整")) + return + } + case "minio": + if req.Multimodal.Minio == nil || req.Multimodal.Minio.BucketName == "" || + os.Getenv("MINIO_ACCESS_KEY_ID") == "" || os.Getenv("MINIO_SECRET_ACCESS_KEY") == "" { + logger.Error(ctx, "MinIO configuration incomplete") + c.Error(errors.NewBadRequestError("MinIO配置不完整")) + return + } + } + } + + // 验证Rerank配置（如果启用） + if req.Rerank.Enabled { + if req.Rerank.ModelName == "" || req.Rerank.BaseURL == "" { + logger.Error(ctx, "Rerank configuration incomplete") + c.Error(errors.NewBadRequestError("Rerank配置不完整")) + return + } + } + + // 验证Node Extractor配置（如果启用） + if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" && req.NodeExtract.Enabled { + logger.Error(ctx, "Node Extractor configuration incomplete") + c.Error(errors.NewBadRequestError("请正确配置环境变量NEO4J_ENABLE")) + return + } + if req.NodeExtract.Enabled { + if req.NodeExtract.Text == "" || len(req.NodeExtract.Tags) == 0 { + logger.Error(ctx, "Node Extractor configuration incomplete") + c.Error(errors.NewBadRequestError("Node Extractor配置不完整")) + return + } + if len(req.NodeExtract.Nodes) == 0 || len(req.NodeExtract.Relations) == 0 { + logger.Error(ctx, "Node Extractor configuration incomplete") + c.Error(errors.NewBadRequestError("请先提取实体和关系")) + return + } + } + + // 处理模型创建/更新 + modelsToProcess := []struct { + modelType types.ModelType + name string + source types.ModelSource + description string + baseURL string + apiKey string + dimension int + }{ + { + modelType: types.ModelTypeKnowledgeQA, + name: req.LLM.ModelName, + source: types.ModelSource(req.LLM.Source), + description: "LLM Model for Knowledge QA", + baseURL: req.LLM.BaseURL, + apiKey: req.LLM.APIKey, + }, + { + modelType: types.ModelTypeEmbedding, + name: req.Embedding.ModelName, + source: types.ModelSource(req.Embedding.Source), + description: "Embedding Model", + baseURL: req.Embedding.BaseURL, + apiKey: req.Embedding.APIKey, + dimension: req.Embedding.Dimension, + }, + } + + // 如果启用Rerank，添加Rerank模型 + if req.Rerank.Enabled { + modelsToProcess = append(modelsToProcess, struct { + modelType types.ModelType + name string + source types.ModelSource + description string + baseURL string + apiKey string + dimension int + }{ + modelType: types.ModelTypeRerank, + name: req.Rerank.ModelName, + source: types.ModelSourceRemote, + description: "Rerank Model", + baseURL: req.Rerank.BaseURL, + apiKey: req.Rerank.APIKey, + }) + } + + // 如果启用多模态，添加VLM模型 + if req.Multimodal.Enabled && req.Multimodal.VLM != nil { + modelsToProcess = append(modelsToProcess, struct { + modelType types.ModelType + name string + source types.ModelSource + description string + baseURL string + apiKey string + dimension int + }{ + modelType: types.ModelTypeVLLM, + name: req.Multimodal.VLM.ModelName, + source: types.ModelSourceRemote, + description: "VLM Model", + baseURL: req.Multimodal.VLM.BaseURL, + apiKey: req.Multimodal.VLM.APIKey, + }) + } + + // 处理模型 + var processedModels []*types.Model + for _, modelInfo := range modelsToProcess { + model := &types.Model{ + Type: modelInfo.modelType, + Name: modelInfo.name, + Source: modelInfo.source, + Description: modelInfo.description, + Parameters: types.ModelParameters{ + BaseURL: modelInfo.baseURL, + APIKey: modelInfo.apiKey, + }, + IsDefault: false, + Status: types.ModelStatusActive, + } + + if modelInfo.modelType == types.ModelTypeEmbedding { + model.Parameters.EmbeddingParameters = types.EmbeddingParameters{ + Dimension: modelInfo.dimension, + } + } + + // 检查模型是否已存在 + existingModel, err := h.modelService.GetModelByID(ctx, model.ID) + if err == nil && existingModel != nil { + // 更新现有模型 + existingModel.Name = model.Name + existingModel.Source = model.Source + existingModel.Description = model.Description + existingModel.Parameters = model.Parameters + existingModel.UpdatedAt = time.Now() + + err = h.modelService.UpdateModel(ctx, existingModel) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "kb_id": kbIdStr, + }) + c.Error(errors.NewInternalServerError("更新模型失败: " + err.Error())) + return + } + processedModels = append(processedModels, existingModel) + } else { + // 创建新模型 + err = h.modelService.CreateModel(ctx, model) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_id": model.ID, + "kb_id": kbIdStr, + }) + c.Error(errors.NewInternalServerError("创建模型失败: " + err.Error())) + return + } + processedModels = append(processedModels, model) + } + } + + // 找到模型ID + var embeddingModelID, llmModelID, rerankModelID, vlmModelID string + for _, model := range processedModels { + if model.Type == types.ModelTypeEmbedding { + embeddingModelID = model.ID + } + if model.Type == types.ModelTypeKnowledgeQA { + llmModelID = model.ID + } + if model.Type == types.ModelTypeRerank { + rerankModelID = model.ID + } + if model.Type == types.ModelTypeVLLM { + vlmModelID = model.ID + } + } + + // 更新知识库配置 + kb.SummaryModelID = llmModelID + kb.EmbeddingModelID = embeddingModelID + if req.Rerank.Enabled { + kb.RerankModelID = rerankModelID + } else { + kb.RerankModelID = "" + } + + // 更新文档分割配置 + kb.ChunkingConfig = types.ChunkingConfig{ + ChunkSize: req.DocumentSplitting.ChunkSize, + ChunkOverlap: req.DocumentSplitting.ChunkOverlap, + Separators: req.DocumentSplitting.Separators, + } + + // 更新多模态配置 + if req.Multimodal.Enabled { + kb.ChunkingConfig.EnableMultimodal = true + kb.VLMModelID = vlmModelID + kb.VLMConfig = types.VLMConfig{ + ModelName: req.Multimodal.VLM.ModelName, + BaseURL: req.Multimodal.VLM.BaseURL, + APIKey: req.Multimodal.VLM.APIKey, + InterfaceType: req.Multimodal.VLM.InterfaceType, + } + switch req.Multimodal.StorageType { + case "cos": + if req.Multimodal.COS != nil { + kb.StorageConfig = types.StorageConfig{ + Provider: req.Multimodal.StorageType, + BucketName: req.Multimodal.COS.BucketName, + AppID: req.Multimodal.COS.AppID, + PathPrefix: req.Multimodal.COS.PathPrefix, + SecretID: req.Multimodal.COS.SecretID, + SecretKey: req.Multimodal.COS.SecretKey, + Region: req.Multimodal.COS.Region, + } + } + case "minio": + if req.Multimodal.Minio != nil { + kb.StorageConfig = types.StorageConfig{ + Provider: req.Multimodal.StorageType, + BucketName: req.Multimodal.Minio.BucketName, + PathPrefix: req.Multimodal.Minio.PathPrefix, + SecretID: os.Getenv("MINIO_ACCESS_KEY_ID"), + SecretKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"), + } + } + } + } else { + kb.VLMModelID = "" + kb.VLMConfig = types.VLMConfig{} + kb.StorageConfig = types.StorageConfig{} + } + + if req.NodeExtract.Enabled { + kb.ExtractConfig = &types.ExtractConfig{ + Text: req.NodeExtract.Text, + Tags: req.NodeExtract.Tags, + Nodes: make([]*types.GraphNode, 0), + Relations: make([]*types.GraphRelation, 0), + } + for _, rnode := range req.NodeExtract.Nodes { + node := &types.GraphNode{ + Name: rnode.Name, + Attributes: rnode.Attributes, + } + kb.ExtractConfig.Nodes = append(kb.ExtractConfig.Nodes, node) + } + for _, relation := range req.NodeExtract.Relations { + kb.ExtractConfig.Relations = append(kb.ExtractConfig.Relations, &types.GraphRelation{ + Node1: relation.Node1, + Node2: relation.Node2, + Type: relation.Type, + }) + } + } + + err = h.kbRepository.UpdateKnowledgeBase(ctx, kb) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": kbIdStr}) + c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error())) + return + } + + logger.Info(ctx, "Knowledge base configuration updated successfully", "kbId", kbIdStr) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "知识库配置更新成功", + "data": gin.H{ + "models": processedModels, + "knowledge_base": kb, + }, + }) +} + +// CheckOllamaStatus 检查Ollama服务状态 +func (h *InitializationHandler) CheckOllamaStatus(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Checking Ollama service status") + + // Determine Ollama base URL for display + baseURL := os.Getenv("OLLAMA_BASE_URL") + if baseURL == "" { + baseURL = "http://host.docker.internal:11434" + } + + // 检查Ollama服务是否可用 + err := h.ollamaService.StartService(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "available": false, + "error": err.Error(), + "baseUrl": baseURL, + }, + }) + return + } + + version, err := h.ollamaService.GetVersion(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + version = "unknown" + } + + logger.Info(ctx, "Ollama service is available") + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "available": h.ollamaService.IsAvailable(), + "version": version, + "baseUrl": baseURL, + }, + }) +} + +// CheckOllamaModels 检查Ollama模型状态 +func (h *InitializationHandler) CheckOllamaModels(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Checking Ollama models status") + + var req struct { + Models []string `json:"models" binding:"required"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse models check request", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // 检查Ollama服务是否可用 + if !h.ollamaService.IsAvailable() { + err := h.ollamaService.StartService(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Ollama服务不可用: " + err.Error())) + return + } + } + + modelStatus := make(map[string]bool) + + // 检查每个模型是否存在 + for _, modelName := range req.Models { + checkModelName := modelName + if !strings.Contains(modelName, ":") { + checkModelName = modelName + ":latest" + } + available, err := h.ollamaService.IsModelAvailable(ctx, checkModelName) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_name": modelName, + }) + modelStatus[modelName] = false + } else { + modelStatus[modelName] = available + } + + logger.Infof(ctx, "Model %s availability: %v", modelName, modelStatus[modelName]) + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "models": modelStatus, + }, + }) +} + +// DownloadOllamaModel 异步下载Ollama模型 +func (h *InitializationHandler) DownloadOllamaModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Starting async Ollama model download") + + var req struct { + ModelName string `json:"modelName" binding:"required"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse model download request", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // 检查Ollama服务是否可用 + if !h.ollamaService.IsAvailable() { + err := h.ollamaService.StartService(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Ollama服务不可用: " + err.Error())) + return + } + } + + // 检查模型是否已存在 + available, err := h.ollamaService.IsModelAvailable(ctx, req.ModelName) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_name": req.ModelName, + }) + c.Error(errors.NewInternalServerError("检查模型状态失败: " + err.Error())) + return + } + + if available { + logger.Infof(ctx, "Model %s already exists", req.ModelName) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "模型已存在", + "data": gin.H{ + "modelName": req.ModelName, + "status": "completed", + "progress": 100.0, + }, + }) + return + } + + // 检查是否已有相同模型的下载任务 + tasksMutex.RLock() + for _, task := range downloadTasks { + if task.ModelName == req.ModelName && (task.Status == "pending" || task.Status == "downloading") { + tasksMutex.RUnlock() + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "模型下载任务已存在", + "data": gin.H{ + "taskId": task.ID, + "modelName": task.ModelName, + "status": task.Status, + "progress": task.Progress, + }, + }) + return + } + } + tasksMutex.RUnlock() + + // 创建下载任务 + taskID := uuid.New().String() + task := &DownloadTask{ + ID: taskID, + ModelName: req.ModelName, + Status: "pending", + Progress: 0.0, + Message: "准备下载", + StartTime: time.Now(), + } + + tasksMutex.Lock() + downloadTasks[taskID] = task + tasksMutex.Unlock() + + // 启动异步下载 + newCtx, cancel := context.WithTimeout(context.Background(), 12*time.Hour) + go func() { + defer cancel() + h.downloadModelAsync(newCtx, taskID, req.ModelName) + }() + + logger.Infof(ctx, "Created download task for model: %s, task ID: %s", req.ModelName, taskID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "模型下载任务已创建", + "data": gin.H{ + "taskId": taskID, + "modelName": req.ModelName, + "status": "pending", + "progress": 0.0, + }, + }) +} + +// GetDownloadProgress 获取下载进度 +func (h *InitializationHandler) GetDownloadProgress(c *gin.Context) { + taskID := c.Param("taskId") + + if taskID == "" { + c.Error(errors.NewBadRequestError("任务ID不能为空")) + return + } + + tasksMutex.RLock() + task, exists := downloadTasks[taskID] + tasksMutex.RUnlock() + + if !exists { + c.Error(errors.NewNotFoundError("下载任务不存在")) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": task, + }) +} + +// ListDownloadTasks 列出所有下载任务 +func (h *InitializationHandler) ListDownloadTasks(c *gin.Context) { + tasksMutex.RLock() + tasks := make([]*DownloadTask, 0, len(downloadTasks)) + for _, task := range downloadTasks { + tasks = append(tasks, task) + } + tasksMutex.RUnlock() + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": tasks, + }) +} + +// ListOllamaModels 列出已安装的 Ollama 模型 +func (h *InitializationHandler) ListOllamaModels(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Listing installed Ollama models") + + // 确保服务可用 + if !h.ollamaService.IsAvailable() { + if err := h.ollamaService.StartService(ctx); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Ollama服务不可用: " + err.Error())) + return + } + } + + models, err := h.ollamaService.ListModels(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("获取模型列表失败: " + err.Error())) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "models": models, + }, + }) +} + +// downloadModelAsync 异步下载模型 +func (h *InitializationHandler) downloadModelAsync(ctx context.Context, + taskID, modelName string, +) { + logger.Infof(ctx, "Starting async download for model: %s, task: %s", modelName, taskID) + + // 更新任务状态为下载中 + h.updateTaskStatus(taskID, "downloading", 0.0, "开始下载模型") + + // 执行下载，带进度回调 + err := h.pullModelWithProgress(ctx, modelName, func(progress float64, message string) { + h.updateTaskStatus(taskID, "downloading", progress, message) + }) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_name": modelName, + "task_id": taskID, + }) + h.updateTaskStatus(taskID, "failed", 0.0, fmt.Sprintf("下载失败: %v", err)) + return + } + + // 下载成功 + logger.Infof(ctx, "Model %s downloaded successfully, task: %s", modelName, taskID) + h.updateTaskStatus(taskID, "completed", 100.0, "下载完成") +} + +// pullModelWithProgress 下载模型并提供进度回调 +func (h *InitializationHandler) pullModelWithProgress(ctx context.Context, + modelName string, + progressCallback func(float64, string), +) error { + // 检查服务是否可用 + if err := h.ollamaService.StartService(ctx); err != nil { + logger.ErrorWithFields(ctx, err, nil) + return err + } + + // 检查模型是否已存在 + available, err := h.ollamaService.IsModelAvailable(ctx, modelName) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "model_name": modelName, + }) + return err + } + if available { + progressCallback(100.0, "模型已存在") + return nil + } + + logger.GetLogger(ctx).Infof("Pulling model %s...", modelName) + + // 创建下载请求 + pullReq := &api.PullRequest{ + Name: modelName, + } + + // 使用Ollama客户端的Pull方法，带进度回调 + err = h.ollamaService.GetClient().Pull(ctx, pullReq, func(progress api.ProgressResponse) error { + var progressPercent float64 = 0.0 + var message string = "下载中" + + if progress.Total > 0 && progress.Completed > 0 { + progressPercent = float64(progress.Completed) / float64(progress.Total) * 100 + message = fmt.Sprintf("下载中: %.1f%% (%s)", progressPercent, progress.Status) + } else if progress.Status != "" { + message = progress.Status + } + + // 调用进度回调 + progressCallback(progressPercent, message) + + logger.Infof(ctx, + "Download progress for %s: %.2f%% - %s", + modelName, progressPercent, message, + ) + return nil + }) + if err != nil { + return fmt.Errorf("failed to pull model: %w", err) + } + + return nil +} + +// updateTaskStatus 更新任务状态 +func (h *InitializationHandler) updateTaskStatus( + taskID, status string, progress float64, message string, +) { + tasksMutex.Lock() + defer tasksMutex.Unlock() + + if task, exists := downloadTasks[taskID]; exists { + task.Status = status + task.Progress = progress + task.Message = message + + if status == "completed" || status == "failed" { + now := time.Now() + task.EndTime = &now + } + } +} + +// GetCurrentConfigByKB 根据知识库ID获取配置信息 +func (h *InitializationHandler) GetCurrentConfigByKB(c *gin.Context) { + ctx := c.Request.Context() + kbIdStr := c.Param("kbId") + + logger.Info(ctx, "Getting configuration for knowledge base", "kbId", kbIdStr) + + // 获取指定知识库信息 + kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": kbIdStr}) + c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error())) + return + } + + if kb == nil { + logger.Error(ctx, "Knowledge base not found", "kbId", kbIdStr) + c.Error(errors.NewNotFoundError("知识库不存在")) + return + } + + // 根据知识库的模型ID获取特定模型 + var models []*types.Model + modelIDs := []string{ + kb.EmbeddingModelID, + kb.SummaryModelID, + kb.RerankModelID, + kb.VLMModelID, + } + + for _, modelID := range modelIDs { + if modelID != "" { + model, err := h.modelService.GetModelByID(ctx, modelID) + if err != nil { + logger.Warn(ctx, "Failed to get model", "kbId", kbIdStr, "modelId", modelID, "error", err) + // 如果模型不存在或获取失败，继续处理其他模型 + continue + } + if model != nil { + models = append(models, model) + } + } + } + + // 检查知识库是否有文件 + knowledgeList, err := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx, + kbIdStr, &types.Pagination{ + Page: 1, + PageSize: 1, + }) + hasFiles := false + if err == nil && knowledgeList != nil && knowledgeList.Total > 0 { + hasFiles = true + } + + // 构建配置响应 + config := buildConfigResponse(models, kb, hasFiles) + + logger.Info(ctx, "Knowledge base configuration retrieved successfully", "kbId", kbIdStr) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": config, + }) +} + +// buildConfigResponse 构建配置响应数据 +func buildConfigResponse(models []*types.Model, + kb *types.KnowledgeBase, hasFiles bool, +) map[string]interface{} { + config := map[string]interface{}{ + "hasFiles": hasFiles, + } + + // 按类型分组模型 + for _, model := range models { + switch model.Type { + case types.ModelTypeKnowledgeQA: + config["llm"] = map[string]interface{}{ + "source": string(model.Source), + "modelName": model.Name, + "baseUrl": model.Parameters.BaseURL, + "apiKey": model.Parameters.APIKey, + } + case types.ModelTypeEmbedding: + config["embedding"] = map[string]interface{}{ + "source": string(model.Source), + "modelName": model.Name, + "baseUrl": model.Parameters.BaseURL, + "apiKey": model.Parameters.APIKey, + "dimension": model.Parameters.EmbeddingParameters.Dimension, + } + case types.ModelTypeRerank: + config["rerank"] = map[string]interface{}{ + "enabled": true, + "modelName": model.Name, + "baseUrl": model.Parameters.BaseURL, + "apiKey": model.Parameters.APIKey, + } + case types.ModelTypeVLLM: + if config["multimodal"] == nil { + config["multimodal"] = map[string]interface{}{ + "enabled": true, + } + } + multimodal := config["multimodal"].(map[string]interface{}) + multimodal["vlm"] = map[string]interface{}{ + "modelName": model.Name, + "baseUrl": model.Parameters.BaseURL, + "apiKey": model.Parameters.APIKey, + "interfaceType": kb.VLMConfig.InterfaceType, + } + } + } + + // 如果没有VLM模型，设置multimodal为disabled + if config["multimodal"] == nil { + config["multimodal"] = map[string]interface{}{ + "enabled": false, + } + } + + // 如果没有Rerank模型，设置rerank为disabled + if config["rerank"] == nil { + config["rerank"] = map[string]interface{}{ + "enabled": false, + "modelName": "", + "baseUrl": "", + "apiKey": "", + } + } + + // 添加知识库的文档分割配置 + if kb != nil { + config["documentSplitting"] = map[string]interface{}{ + "chunkSize": kb.ChunkingConfig.ChunkSize, + "chunkOverlap": kb.ChunkingConfig.ChunkOverlap, + "separators": kb.ChunkingConfig.Separators, + } + + // 添加多模态的COS配置信息 + if kb.StorageConfig.SecretID != "" { + if config["multimodal"] == nil { + config["multimodal"] = map[string]interface{}{ + "enabled": true, + } + } + multimodal := config["multimodal"].(map[string]interface{}) + multimodal["storageType"] = kb.StorageConfig.Provider + switch kb.StorageConfig.Provider { + case "cos": + multimodal["cos"] = map[string]interface{}{ + "secretId": kb.StorageConfig.SecretID, + "secretKey": kb.StorageConfig.SecretKey, + "region": kb.StorageConfig.Region, + "bucketName": kb.StorageConfig.BucketName, + "appId": kb.StorageConfig.AppID, + "pathPrefix": kb.StorageConfig.PathPrefix, + } + case "minio": + multimodal["minio"] = map[string]interface{}{ + "bucketName": kb.StorageConfig.BucketName, + "pathPrefix": kb.StorageConfig.PathPrefix, + } + } + } + } + + if kb.ExtractConfig != nil { + config["nodeExtract"] = map[string]interface{}{ + "enabled": true, + "text": kb.ExtractConfig.Text, + "tags": kb.ExtractConfig.Tags, + "nodes": kb.ExtractConfig.Nodes, + "relations": kb.ExtractConfig.Relations, + } + } else { + config["nodeExtract"] = map[string]interface{}{ + "enabled": false, + } + } + + return config +} + +// RemoteModelCheckRequest 远程模型检查请求结构 +type RemoteModelCheckRequest struct { + ModelName string `json:"modelName" binding:"required"` + BaseURL string `json:"baseUrl" binding:"required"` + APIKey string `json:"apiKey"` +} + +// CheckRemoteModel 检查远程API模型连接 +func (h *InitializationHandler) CheckRemoteModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Checking remote model connection") + + var req RemoteModelCheckRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse remote model check request", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // 验证请求参数 + if req.ModelName == "" || req.BaseURL == "" { + logger.Error(ctx, "Model name and base URL are required") + c.Error(errors.NewBadRequestError("模型名称和Base URL不能为空")) + return + } + + // 创建模型配置进行测试 + modelConfig := &types.Model{ + Name: req.ModelName, + Source: "remote", + Parameters: types.ModelParameters{ + BaseURL: req.BaseURL, + APIKey: req.APIKey, + }, + Type: "llm", // 默认类型，实际检查时不区分具体类型 + } + + // 检查远程模型连接 + available, message := h.checkRemoteModelConnection(ctx, modelConfig) + + logger.Info(ctx, + fmt.Sprintf( + "Remote model check completed: modelName=%s, baseUrl=%s, available=%v, message=%s", + req.ModelName, req.BaseURL, available, message, + ), + ) + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "available": available, + "message": message, + }, + }) +} + +// TestEmbeddingModel 测试 Embedding 接口（本地或远程）是否可用 +func (h *InitializationHandler) TestEmbeddingModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Testing embedding model connectivity and functionality") + + var req struct { + Source string `json:"source" binding:"required"` + ModelName string `json:"modelName" binding:"required"` + BaseURL string `json:"baseUrl"` + APIKey string `json:"apiKey"` + Dimension int `json:"dimension"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse embedding test request", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // 构造 embedder 配置 + cfg := embedding.Config{ + Source: types.ModelSource(strings.ToLower(req.Source)), + BaseURL: req.BaseURL, + ModelName: req.ModelName, + APIKey: req.APIKey, + TruncatePromptTokens: 256, + Dimensions: req.Dimension, + ModelID: "", + } + + emb, err := embedding.NewEmbedder(cfg) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": req.ModelName}) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{`available`: false, `message`: fmt.Sprintf("创建Embedder失败: %v", err), `dimension`: 0}, + }) + return + } + + // 执行一次最小化 embedding 调用 + sample := "hello" + vec, err := emb.Embed(ctx, sample) + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": req.ModelName}) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{`available`: false, `message`: fmt.Sprintf("调用Embedding失败: %v", err), `dimension`: 0}, + }) + return + } + + logger.Infof(ctx, "Embedding test succeeded, dim=%d", len(vec)) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{`available`: true, `message`: fmt.Sprintf("测试成功，向量维度=%d", len(vec)), `dimension`: len(vec)}, + }) +} + +// checkRemoteModelConnection 检查远程模型连接的内部方法 +func (h *InitializationHandler) checkRemoteModelConnection(ctx context.Context, + model *types.Model, +) (bool, string) { + // 使用 models/chat 进行连接检查 + // 创建聊天配置 + chatConfig := &chat.ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + APIKey: model.Parameters.APIKey, + ModelID: model.Name, + } + + // 创建聊天实例 + chatInstance, err := chat.NewChat(chatConfig) + if err != nil { + return false, fmt.Sprintf("创建聊天实例失败: %v", err) + } + + // 构造测试消息 + testMessages := []chat.Message{ + { + Role: "user", + Content: "test", + }, + } + + // 构造测试选项 + testOptions := &chat.ChatOptions{ + MaxTokens: 1, + Thinking: &[]bool{false}[0], // for dashscope.aliyuncs qwen3-32b + } + + // 使用聊天实例进行测试 + _, err = chatInstance.Chat(ctx, testMessages, testOptions) + if err != nil { + // 根据错误类型返回不同的错误信息 + if strings.Contains(err.Error(), "401") || strings.Contains(err.Error(), "unauthorized") { + return false, "认证失败，请检查API Key" + } else if strings.Contains(err.Error(), "403") || strings.Contains(err.Error(), "forbidden") { + return false, "权限不足，请检查API Key权限" + } else if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "not found") { + return false, "API端点不存在，请检查Base URL" + } else if strings.Contains(err.Error(), "timeout") { + return false, "连接超时，请检查网络连接" + } else { + return false, fmt.Sprintf("连接失败: %v", err) + } + } + + // 连接成功，模型可用 + return true, "连接正常，模型可用" +} + +// checkRerankModelConnection 检查Rerank模型连接和功能的内部方法 +func (h *InitializationHandler) checkRerankModelConnection(ctx context.Context, + modelName, baseURL, apiKey string, +) (bool, string) { + // 创建Reranker配置 + config := &rerank.RerankerConfig{ + APIKey: apiKey, + BaseURL: baseURL, + ModelName: modelName, + Source: types.ModelSourceRemote, // 默认值，实际会根据URL判断 + } + + // 创建Reranker实例 + reranker, err := rerank.NewReranker(config) + if err != nil { + return false, fmt.Sprintf("创建Reranker失败: %v", err) + } + + // 简化的测试数据 + testQuery := "ping" + testDocuments := []string{ + "pong", + } + + // 使用Reranker进行测试 + results, err := reranker.Rerank(ctx, testQuery, testDocuments) + if err != nil { + return false, fmt.Sprintf("重排测试失败: %v", err) + } + + // 检查结果 + if len(results) > 0 { + return true, fmt.Sprintf("重排功能正常，返回%d个结果", len(results)) + } else { + return false, "重排接口连接成功，但未返回重排结果" + } +} + +// CheckRerankModel 检查Rerank模型连接和功能 +func (h *InitializationHandler) CheckRerankModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Checking rerank model connection and functionality") + + var req struct { + ModelName string `json:"modelName" binding:"required"` + BaseURL string `json:"baseUrl" binding:"required"` + APIKey string `json:"apiKey"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse rerank model check request", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // 验证请求参数 + if req.ModelName == "" || req.BaseURL == "" { + logger.Error(ctx, "Model name and base URL are required") + c.Error(errors.NewBadRequestError("模型名称和Base URL不能为空")) + return + } + + // 检查Rerank模型连接和功能 + available, message := h.checkRerankModelConnection( + ctx, req.ModelName, req.BaseURL, req.APIKey, + ) + + logger.Info(ctx, + fmt.Sprintf("Rerank model check completed: modelName=%s, baseUrl=%s, available=%v, message=%s", + req.ModelName, req.BaseURL, available, message, + ), + ) + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "available": available, + "message": message, + }, + }) +} + +// 使用结构体解析表单数据 +type testMultimodalForm struct { + VLMModel string `form:"vlm_model"` + VLMBaseURL string `form:"vlm_base_url"` + VLMAPIKey string `form:"vlm_api_key"` + VLMInterfaceType string `form:"vlm_interface_type"` + + StorageType string `form:"storage_type"` + + // COS 配置 + COSSecretID string `form:"cos_secret_id"` + COSSecretKey string `form:"cos_secret_key"` + COSRegion string `form:"cos_region"` + COSBucketName string `form:"cos_bucket_name"` + COSAppID string `form:"cos_app_id"` + COSPathPrefix string `form:"cos_path_prefix"` + + // MinIO 配置（当存储为 minio 时） + MinioBucketName string `form:"minio_bucket_name"` + MinioPathPrefix string `form:"minio_path_prefix"` + + // 文档切分配置（字符串后续自行解析，以避免类型绑定失败） + ChunkSize string `form:"chunk_size"` + ChunkOverlap string `form:"chunk_overlap"` + SeparatorsRaw string `form:"separators"` +} + +// TestMultimodalFunction 测试多模态功能 +func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Testing multimodal functionality") + + var req testMultimodalForm + if err := c.ShouldBind(&req); err != nil { + logger.Error(ctx, "Failed to parse form data", err) + c.Error(errors.NewBadRequestError("表单参数解析失败")) + return + } + // ollama 场景自动拼接 base url + if req.VLMInterfaceType == "ollama" { + req.VLMBaseURL = os.Getenv("OLLAMA_BASE_URL") + "/v1" + } + + req.StorageType = strings.ToLower(req.StorageType) + + if req.VLMModel == "" || req.VLMBaseURL == "" { + logger.Error(ctx, "VLM model name and base URL are required") + c.Error(errors.NewBadRequestError("VLM模型名称和Base URL不能为空")) + return + } + switch req.StorageType { + case "cos": + logger.Infof(ctx, "COS config: Region=%s, Bucket=%s, App=%s, Prefix=%s", + req.COSRegion, req.COSBucketName, req.COSAppID, req.COSPathPrefix) + // 必填：SecretID/SecretKey/Region/BucketName/AppID；PathPrefix 可选 + if req.COSSecretID == "" || req.COSSecretKey == "" || + req.COSRegion == "" || req.COSBucketName == "" || + req.COSAppID == "" { + logger.Error(ctx, "COS configuration is required") + c.Error(errors.NewBadRequestError("COS配置信息不能为空")) + return + } + case "minio": + logger.Infof(ctx, "MinIO config: Bucket=%s, PathPrefix=%s", req.MinioBucketName, req.MinioPathPrefix) + if req.MinioBucketName == "" { + logger.Error(ctx, "MinIO configuration is required") + c.Error(errors.NewBadRequestError("MinIO配置信息不能为空")) + return + } + default: + logger.Error(ctx, "Invalid storage type") + c.Error(errors.NewBadRequestError("无效的存储类型")) + return + } + + logger.Infof(ctx, "VLM config: Model=%s, URL=%s, HasKey=%v, Type=%s", + req.VLMModel, req.VLMBaseURL, req.VLMAPIKey != "", req.VLMInterfaceType) + + // 获取上传的图片文件 + file, header, err := c.Request.FormFile("image") + if err != nil { + logger.Error(ctx, "Failed to get uploaded image", err) + c.Error(errors.NewBadRequestError("获取上传图片失败")) + return + } + defer file.Close() + + // 验证文件类型 + if !strings.HasPrefix(header.Header.Get("Content-Type"), "image/") { + logger.Error(ctx, "Invalid file type, only images are allowed") + c.Error(errors.NewBadRequestError("只允许上传图片文件")) + return + } + + // 验证文件大小 (10MB) + if header.Size > 10*1024*1024 { + logger.Error(ctx, "File size too large") + c.Error(errors.NewBadRequestError("图片文件大小不能超过10MB")) + return + } + logger.Infof(ctx, "Processing image: %s, size: %d bytes", header.Filename, header.Size) + + // 解析文档分割配置 + chunkSize, err := strconv.Atoi(req.ChunkSize) + if err != nil || chunkSize < 100 || chunkSize > 10000 { + chunkSize = 1000 + } + + chunkOverlap, err := strconv.Atoi(req.ChunkOverlap) + if err != nil || chunkOverlap < 0 || chunkOverlap >= chunkSize { + chunkOverlap = 200 + } + + var separators []string + if req.SeparatorsRaw != "" { + if err := json.Unmarshal([]byte(req.SeparatorsRaw), &separators); err != nil { + separators = []string{"\n\n", "\n", "。", "！", "？", ";", "；"} + } + } else { + separators = []string{"\n\n", "\n", "。", "！", "？", ";", "；"} + } + + // 读取图片文件内容 + imageContent, err := io.ReadAll(file) + if err != nil { + logger.Error(ctx, "Failed to read image file", err) + c.Error(errors.NewBadRequestError("读取图片文件失败")) + return + } + + // 调用多模态测试 + startTime := time.Now() + result, err := h.testMultimodalWithDocReader( + ctx, + imageContent, header.Filename, + chunkSize, chunkOverlap, separators, &req, + ) + processingTime := time.Since(startTime).Milliseconds() + + if err != nil { + logger.ErrorWithFields(ctx, err, map[string]interface{}{ + "vlm_model": req.VLMModel, + "vlm_base_url": req.VLMBaseURL, + "filename": header.Filename, + }) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "success": false, + "message": err.Error(), + "processing_time": processingTime, + }, + }) + return + } + + logger.Info(ctx, fmt.Sprintf("Multimodal test completed successfully in %dms", processingTime)) + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "success": true, + "caption": result["caption"], + "ocr": result["ocr"], + "processing_time": processingTime, + }, + }) +} + +// testMultimodalWithDocReader 调用docreader服务进行多模态处理 +func (h *InitializationHandler) testMultimodalWithDocReader( + ctx context.Context, + imageContent []byte, filename string, + chunkSize, chunkOverlap int, separators []string, + req *testMultimodalForm, +) (map[string]string, error) { + // 获取文件扩展名 + fileExt := "" + if idx := strings.LastIndex(filename, "."); idx != -1 { + fileExt = strings.ToLower(filename[idx+1:]) + } + + // 检查docreader服务配置 + if h.docReaderClient == nil { + return nil, fmt.Errorf("DocReader service not configured") + } + + // 构造请求 + request := &proto.ReadFromFileRequest{ + FileContent: imageContent, + FileName: filename, + FileType: fileExt, + ReadConfig: &proto.ReadConfig{ + ChunkSize: int32(chunkSize), + ChunkOverlap: int32(chunkOverlap), + Separators: separators, + EnableMultimodal: true, // 启用多模态处理 + VlmConfig: &proto.VLMConfig{ + ModelName: req.VLMModel, + BaseUrl: req.VLMBaseURL, + ApiKey: req.VLMAPIKey, + InterfaceType: req.VLMInterfaceType, + }, + }, + RequestId: ctx.Value(types.RequestIDContextKey).(string), + } + + // 设置对象存储配置（通用） + switch strings.ToLower(req.StorageType) { + case "cos": + request.ReadConfig.StorageConfig = &proto.StorageConfig{ + Provider: proto.StorageProvider_COS, + Region: req.COSRegion, + BucketName: req.COSBucketName, + AccessKeyId: req.COSSecretID, + SecretAccessKey: req.COSSecretKey, + AppId: req.COSAppID, + PathPrefix: req.COSPathPrefix, + } + case "minio": + request.ReadConfig.StorageConfig = &proto.StorageConfig{ + Provider: proto.StorageProvider_MINIO, + BucketName: req.MinioBucketName, + PathPrefix: req.MinioPathPrefix, + AccessKeyId: os.Getenv("MINIO_ACCESS_KEY_ID"), + SecretAccessKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"), + } + } + + // 调用docreader服务 + response, err := h.docReaderClient.ReadFromFile(ctx, request) + if err != nil { + return nil, fmt.Errorf("调用DocReader服务失败: %v", err) + } + + if response.Error != "" { + return nil, fmt.Errorf("DocReader服务返回错误: %s", response.Error) + } + + // 处理响应，提取Caption和OCR信息 + result := make(map[string]string) + var allCaptions, allOCRTexts []string + + for _, chunk := range response.Chunks { + if len(chunk.Images) > 0 { + for _, image := range chunk.Images { + if image.Caption != "" { + allCaptions = append(allCaptions, image.Caption) + } + if image.OcrText != "" { + allOCRTexts = append(allOCRTexts, image.OcrText) + } + } + } + } + + // 合并所有Caption和OCR结果 + result["caption"] = strings.Join(allCaptions, "; ") + result["ocr"] = strings.Join(allOCRTexts, "; ") + + return result, nil +} + +// TextRelationExtractionRequest 文本关系提取请求结构 +type TextRelationExtractionRequest struct { + Text string `json:"text" binding:"required"` + Tags []string `json:"tags" binding:"required"` + LLMConfig LLMConfig `json:"llmConfig"` +} + +type LLMConfig struct { + Source string `json:"source"` + ModelName string `json:"modelName"` + BaseUrl string `json:"baseUrl"` + ApiKey string `json:"apiKey"` +} + +// TextRelationExtractionResponse 文本关系提取响应结构 +type TextRelationExtractionResponse struct { + Nodes []*types.GraphNode `json:"nodes"` + Relations []*types.GraphRelation `json:"relations"` +} + +func (h *InitializationHandler) ExtractTextRelations(c *gin.Context) { + ctx := c.Request.Context() + + var req TextRelationExtractionRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "文本关系提取请求参数错误") + c.Error(errors.NewBadRequestError("文本关系提取请求参数错误")) + return + } + + // 验证文本内容 + if len(req.Text) == 0 { + c.Error(errors.NewBadRequestError("文本内容不能为空")) + return + } + + if len(req.Text) > 5000 { + c.Error(errors.NewBadRequestError("文本内容长度不能超过5000字符")) + return + } + + // 验证标签 + if len(req.Tags) == 0 { + c.Error(errors.NewBadRequestError("至少需要选择一个关系标签")) + return + } + + // 调用模型服务进行文本关系提取 + result, err := h.extractRelationsFromText(ctx, req.Text, req.Tags, req.LLMConfig) + if err != nil { + logger.Error(ctx, "文本关系提取失败", err) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "success": false, + "message": err.Error(), + }, + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result, + }) +} + +// extractRelationsFromText 从文本中提取关系 +func (h *InitializationHandler) extractRelationsFromText(ctx context.Context, text string, tags []string, llm LLMConfig) (*TextRelationExtractionResponse, error) { + chatModel, err := chat.NewChat(&chat.ChatConfig{ + ModelID: "initialization", + APIKey: llm.ApiKey, + BaseURL: llm.BaseUrl, + ModelName: llm.ModelName, + Source: types.ModelSource(llm.Source), + }) + if err != nil { + logger.Error(ctx, "初始化模型服务失败", err) + return nil, err + } + + template := &types.PromptTemplateStructured{ + Description: h.config.ExtractManager.ExtractGraph.Description, + Tags: tags, + Examples: h.config.ExtractManager.ExtractGraph.Examples, + } + + extractor := chatpipline.NewExtractor(chatModel, template) + graph, err := extractor.Extract(ctx, text) + if err != nil { + logger.Error(ctx, "文本关系提取失败", err) + return nil, err + } + extractor.RemoveUnknownRelation(ctx, graph) + + result := &TextRelationExtractionResponse{ + Nodes: graph.Node, + Relations: graph.Relation, + } + + return result, nil +} + +type FabriTextRequest struct { + Tags []string `json:"tags"` + LLMConfig LLMConfig `json:"llmConfig"` +} + +type FabriTextResponse struct { + Text string `json:"text"` +} + +func (h *InitializationHandler) FabriText(c *gin.Context) { + ctx := c.Request.Context() + + var req FabriTextRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "生成示例文本请求参数错误") + c.Error(errors.NewBadRequestError("生成示例文本请求参数错误")) + return + } + + result, err := h.fabriText(ctx, req.Tags, req.LLMConfig) + if err != nil { + logger.Error(ctx, "生成示例文本失败", err) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "success": false, + "message": err.Error(), + }, + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": FabriTextResponse{Text: result}, + }) +} + +func (h *InitializationHandler) fabriText(ctx context.Context, tags []string, llm LLMConfig) (string, error) { + chatModel, err := chat.NewChat(&chat.ChatConfig{ + ModelID: "initialization", + APIKey: llm.ApiKey, + BaseURL: llm.BaseUrl, + ModelName: llm.ModelName, + Source: types.ModelSource(llm.Source), + }) + if err != nil { + logger.Error(ctx, "初始化模型服务失败", err) + return "", err + } + + content := h.config.ExtractManager.FabriText.WithNoTag + if len(tags) > 0 { + tagStr, _ := json.Marshal(tags) + content = fmt.Sprintf(h.config.ExtractManager.FabriText.WithTag, string(tagStr)) + } + + think := false + result, err := chatModel.Chat(ctx, []chat.Message{ + {Role: "user", Content: content}, + }, &chat.ChatOptions{ + Temperature: 0.3, + MaxTokens: 4096, + Thinking: &think, + }) + if err != nil { + logger.Error(ctx, "生成示例文本失败", err) + return "", err + } + return result.Content, nil +} + +type FabriTagRequest struct { + LLMConfig LLMConfig `json:"llmConfig"` +} + +type FabriTagResponse struct { + Tags []string `json:"tags"` +} + +var tagOptions = []string{ + "内容", "文化", "人物", "事件", "时间", "地点", "作品", "作者", "关系", "属性", +} + +func (h *InitializationHandler) FabriTag(c *gin.Context) { + tagRandom := RandomSelect(tagOptions, rand.Intn(len(tagOptions)-1)+1) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": FabriTagResponse{Tags: tagRandom}, + }) +} + +func RandomSelect(strs []string, n int) []string { + if n <= 0 { + return []string{} + } + result := make([]string, len(strs)) + copy(result, strs) + rand.Shuffle(len(result), func(i, j int) { + result[i], result[j] = result[j], result[i] + }) + + if n > len(strs) { + n = len(strs) + } + return result[:n] +} diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go new file mode 100644 index 0000000..dac6ffe --- /dev/null +++ b/internal/handler/knowledge.go @@ -0,0 +1,476 @@ +package handler + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// KnowledgeHandler processes HTTP requests related to knowledge resources +type KnowledgeHandler struct { + kgService interfaces.KnowledgeService + kbService interfaces.KnowledgeBaseService +} + +// NewKnowledgeHandler creates a new knowledge handler instance +func NewKnowledgeHandler( + kgService interfaces.KnowledgeService, + kbService interfaces.KnowledgeBaseService, +) *KnowledgeHandler { + return &KnowledgeHandler{kgService: kgService, kbService: kbService} +} + +// validateKnowledgeBaseAccess validates access permissions to a knowledge base +// Returns the knowledge base, the knowledge base ID, and any errors encountered +func (h *KnowledgeHandler) validateKnowledgeBaseAccess(c *gin.Context) (*types.KnowledgeBase, string, error) { + ctx := c.Request.Context() + + // Get knowledge base ID from URL path parameter + kbID := c.Param("id") + if kbID == "" { + logger.Error(ctx, "Knowledge base ID is empty") + return nil, "", errors.NewBadRequestError("Knowledge base ID cannot be empty") + } + + // Get knowledge base details + kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + return nil, kbID, errors.NewInternalServerError(err.Error()) + } + + // Verify tenant permissions + if kb.TenantID != c.GetUint(types.TenantIDContextKey.String()) { + logger.Warnf( + ctx, + "Permission denied to access this knowledge base, tenant ID mismatch, "+ + "requested tenant ID: %d, knowledge base tenant ID: %d", + c.GetUint(types.TenantIDContextKey.String()), + kb.TenantID, + ) + return nil, kbID, errors.NewForbiddenError("Permission denied to access this knowledge base") + } + + return kb, kbID, nil +} + +// handleDuplicateKnowledgeError handles cases where duplicate knowledge is detected +// Returns true if the error was a duplicate error and was handled, false otherwise +func (h *KnowledgeHandler) handleDuplicateKnowledgeError(c *gin.Context, + err error, knowledge *types.Knowledge, duplicateType string, +) bool { + if dupErr, ok := err.(*types.DuplicateKnowledgeError); ok { + ctx := c.Request.Context() + logger.Warnf(ctx, "Detected duplicate %s: %s", duplicateType, dupErr.Error()) + c.JSON(http.StatusConflict, gin.H{ + "success": false, + "message": dupErr.Error(), + "data": knowledge, // knowledge contains the existing document + "code": fmt.Sprintf("duplicate_%s", duplicateType), + }) + return true + } + return false +} + +// CreateKnowledgeFromFile handles requests to create knowledge from an uploaded file +func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start creating knowledge from file") + + // Validate access to the knowledge base + _, kbID, err := h.validateKnowledgeBaseAccess(c) + if err != nil { + c.Error(err) + return + } + + // Get the uploaded file + file, err := c.FormFile("file") + if err != nil { + logger.Error(ctx, "File upload failed", err) + c.Error(errors.NewBadRequestError("File upload failed").WithDetails(err.Error())) + return + } + + logger.Infof(ctx, "File upload successful, filename: %s, size: %.2f KB", file.Filename, float64(file.Size)/1024) + logger.Infof(ctx, "Creating knowledge, knowledge base ID: %s, filename: %s", kbID, file.Filename) + + // Parse metadata if provided + var metadata map[string]string + metadataStr := c.PostForm("metadata") + if metadataStr != "" { + if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil { + logger.Error(ctx, "Failed to parse metadata", err) + c.Error(errors.NewBadRequestError("Invalid metadata format").WithDetails(err.Error())) + return + } + logger.Infof(ctx, "Received file metadata: %v", metadata) + } + + enableMultimodelForm := c.PostForm("enable_multimodel") + var enableMultimodel *bool + if enableMultimodelForm != "" { + parseBool, err := strconv.ParseBool(enableMultimodelForm) + if err != nil { + logger.Error(ctx, "Failed to parse enable_multimodel", err) + c.Error(errors.NewBadRequestError("Invalid enable_multimodel format").WithDetails(err.Error())) + return + } + enableMultimodel = &parseBool + } + + // Create knowledge entry from the file + knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel) + // Check for duplicate knowledge error + if err != nil { + if h.handleDuplicateKnowledgeError(c, err, knowledge, "file") { + return + } + if appErr, ok := errors.IsAppError(err); ok { + c.Error(appErr) + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge created successfully, ID: %s, title: %s", knowledge.ID, knowledge.Title) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": knowledge, + }) +} + +// CreateKnowledgeFromURL handles requests to create knowledge from a URL +func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start creating knowledge from URL") + + // Validate access to the knowledge base + _, kbID, err := h.validateKnowledgeBaseAccess(c) + if err != nil { + c.Error(err) + return + } + + // Parse URL from request body + var req struct { + URL string `json:"url" binding:"required"` + EnableMultimodel *bool `json:"enable_multimodel"` + } + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse URL request", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + logger.Infof(ctx, "Received URL request: %s", req.URL) + logger.Infof(ctx, "Creating knowledge from URL, knowledge base ID: %s, URL: %s", kbID, req.URL) + + // Create knowledge entry from the URL + knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel) + // Check for duplicate knowledge error + if err != nil { + if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") { + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge created successfully from URL, ID: %s, title: %s", knowledge.ID, knowledge.Title) + c.JSON(http.StatusCreated, gin.H{ + "success": true, + "data": knowledge, + }) +} + +// GetKnowledge retrieves a knowledge entry by its ID +func (h *KnowledgeHandler) GetKnowledge(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving knowledge") + + // Get knowledge ID from URL path parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + + logger.Infof(ctx, "Retrieving knowledge, ID: %s", id) + knowledge, err := h.kgService.GetKnowledgeByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge retrieved successfully, ID: %s, title: %s", knowledge.ID, knowledge.Title) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": knowledge, + }) +} + +// ListKnowledge retrieves a paginated list of knowledge entries from a knowledge base +func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving knowledge list") + + // Get knowledge base ID from URL path parameter + kbID := c.Param("id") + if kbID == "" { + logger.Error(ctx, "Knowledge base ID is empty") + c.Error(errors.NewBadRequestError("Knowledge base ID cannot be empty")) + return + } + + // Parse pagination parameters from query string + var pagination types.Pagination + if err := c.ShouldBindQuery(&pagination); err != nil { + logger.Error(ctx, "Failed to parse pagination parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + logger.Infof(ctx, "Retrieving knowledge list under knowledge base, knowledge base ID: %s, page: %d, page size: %d", + kbID, pagination.Page, pagination.PageSize) + + // Retrieve paginated knowledge entries + result, err := h.kgService.ListPagedKnowledgeByKnowledgeBaseID(ctx, kbID, &pagination) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge list retrieved successfully, knowledge base ID: %s, total: %d", kbID, result.Total) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result.Data, + "total": result.Total, + "page": result.Page, + "page_size": result.PageSize, + }) +} + +// DeleteKnowledge handles requests to delete a knowledge entry by its ID +func (h *KnowledgeHandler) DeleteKnowledge(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start deleting knowledge") + + // Get knowledge ID from URL path parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + + logger.Infof(ctx, "Deleting knowledge, ID: %s", id) + err := h.kgService.DeleteKnowledge(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge deleted successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Deleted successfully", + }) +} + +// DownloadKnowledgeFile handles requests to download a file associated with a knowledge entry +func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start downloading knowledge file") + + // Get knowledge ID from URL path parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + + logger.Infof(ctx, "Retrieving knowledge file, ID: %s", id) + + // Get file content and filename + file, filename, err := h.kgService.GetKnowledgeFile(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error())) + return + } + defer file.Close() + + logger.Infof(ctx, "Knowledge file retrieved successfully, ID: %s, filename: %s", id, filename) + + // Set response headers for file download + c.Header("Content-Description", "File Transfer") + c.Header("Content-Transfer-Encoding", "binary") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filename)) + c.Header("Content-Type", "application/octet-stream") + c.Header("Expires", "0") + c.Header("Cache-Control", "must-revalidate") + c.Header("Pragma", "public") + + // Stream file content to response + c.Stream(func(w io.Writer) bool { + if _, err := io.Copy(w, file); err != nil { + logger.Errorf(ctx, "Failed to send file: %v", err) + return false + } + logger.Debug(ctx, "File sending completed") + return false + }) +} + +// GetKnowledgeBatchRequest defines parameters for batch knowledge retrieval +type GetKnowledgeBatchRequest struct { + IDs []string `form:"ids" binding:"required"` // List of knowledge IDs +} + +// GetKnowledgeBatch handles requests to retrieve multiple knowledge entries in a batch +func (h *KnowledgeHandler) GetKnowledgeBatch(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start batch retrieving knowledge") + + // Get tenant ID from context + tenantID, ok := c.Get(types.TenantIDContextKey.String()) + if !ok { + logger.Error(ctx, "Failed to get tenant ID") + c.Error(errors.NewUnauthorizedError("Unauthorized")) + return + } + + // Parse request parameters from query string + var req GetKnowledgeBatchRequest + if err := c.ShouldBindQuery(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error())) + return + } + + logger.Infof( + ctx, + "Batch retrieving knowledge, tenant ID: %d, number of knowledge IDs: %d", + tenantID.(uint), len(req.IDs), + ) + + // Retrieve knowledge entries in batch + knowledges, err := h.kgService.GetKnowledgeBatch(ctx, tenantID.(uint), req.IDs) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to retrieve knowledge list").WithDetails(err.Error())) + return + } + + logger.Infof( + ctx, + "Batch knowledge retrieval successful, requested count: %d, returned count: %d", + len(req.IDs), len(knowledges), + ) + + // Return results + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": knowledges, + }) +} + +func (h *KnowledgeHandler) UpdateKnowledge(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start Update knowledge") + + // Get knowledge ID from URL path parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + + var knowledge types.Knowledge + if err := c.ShouldBindJSON(&knowledge); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + if err := h.kgService.UpdateKnowledge(ctx, &knowledge); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge updated successfully, knowledge ID: %s", knowledge.ID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Knowledge chunk updated successfully", + }) +} + +// UpdateImageInfo updates a chunk's properties +func (h *KnowledgeHandler) UpdateImageInfo(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start updating image info") + + // Get knowledge ID from URL path parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Knowledge ID is empty") + c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty")) + return + } + chunkID := c.Param("chunk_id") + if id == "" { + logger.Error(ctx, "Chunk ID is empty") + c.Error(errors.NewBadRequestError("Chunk ID cannot be empty")) + return + } + + var request struct { + ImageInfo string `json:"image_info"` + } + + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // Update chunk properties + logger.Infof(ctx, "Updating knowledge chunk, knowledge ID: %s, chunk ID: %s", id, chunkID) + err := h.kgService.UpdateImageInfo(ctx, id, chunkID, request.ImageInfo) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge chunk updated successfully, knowledge ID: %s, chunk ID: %s", id, chunkID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Knowledge chunk image updated successfully", + }) +} diff --git a/internal/handler/knowledgebase.go b/internal/handler/knowledgebase.go new file mode 100644 index 0000000..29593b1 --- /dev/null +++ b/internal/handler/knowledgebase.go @@ -0,0 +1,297 @@ +package handler + +import ( + "context" + "net/http" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// KnowledgeBaseHandler defines the HTTP handler for knowledge base operations +type KnowledgeBaseHandler struct { + service interfaces.KnowledgeBaseService + knowledgeService interfaces.KnowledgeService +} + +// NewKnowledgeBaseHandler creates a new knowledge base handler instance +func NewKnowledgeBaseHandler( + service interfaces.KnowledgeBaseService, + knowledgeService interfaces.KnowledgeService, +) *KnowledgeBaseHandler { + return &KnowledgeBaseHandler{service: service, knowledgeService: knowledgeService} +} + +// HybridSearch handles requests to perform hybrid vector and keyword search on a knowledge base +func (h *KnowledgeBaseHandler) HybridSearch(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start hybrid search") + + // Validate knowledge base ID + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Knowledge base ID is empty") + c.Error(errors.NewBadRequestError("Knowledge base ID cannot be empty")) + return + } + + // Parse request body + var req types.SearchParams + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error())) + return + } + + logger.Infof(ctx, "Executing hybrid search, knowledge base ID: %s, query: %s", id, req.QueryText) + + // Execute hybrid search with default search parameters + results, err := h.service.HybridSearch(ctx, id, req) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Hybrid search completed, knowledge base ID: %s, result count: %d", id, len(results)) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": results, + }) +} + +// CreateKnowledgeBase handles requests to create a new knowledge base +func (h *KnowledgeBaseHandler) CreateKnowledgeBase(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start creating knowledge base") + + // Parse request body + var req types.KnowledgeBase + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error())) + return + } + + logger.Infof(ctx, "Creating knowledge base, name: %s", req.Name) + // Create knowledge base using the service + kb, err := h.service.CreateKnowledgeBase(ctx, &req) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge base created successfully, ID: %s, name: %s", kb.ID, kb.Name) + c.JSON(http.StatusCreated, gin.H{ + "success": true, + "data": kb, + }) +} + +// validateAndGetKnowledgeBase validates request parameters and retrieves the knowledge base +// Returns the knowledge base, knowledge base ID, and any errors encountered +func (h *KnowledgeBaseHandler) validateAndGetKnowledgeBase(c *gin.Context) (*types.KnowledgeBase, string, error) { + ctx := c.Request.Context() + + // Get tenant ID from context + tenantID, exists := c.Get(types.TenantIDContextKey.String()) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + return nil, "", errors.NewUnauthorizedError("Unauthorized") + } + + // Get knowledge base ID from URL parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Knowledge base ID is empty") + return nil, "", errors.NewBadRequestError("Knowledge base ID cannot be empty") + } + + logger.Infof(ctx, "Retrieving knowledge base, ID: %s", id) + + // Verify tenant has permission to access this knowledge base + kb, err := h.service.GetKnowledgeBaseByID(ctx, id) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + return nil, id, errors.NewInternalServerError(err.Error()) + } + + // Verify tenant ownership + if kb.TenantID != tenantID.(uint) { + logger.Warnf( + ctx, + "Tenant has no permission to access this knowledge base, knowledge base ID: %s, "+ + "request tenant ID: %d, knowledge base tenant ID: %d", + id, tenantID.(uint), kb.TenantID, + ) + return nil, id, errors.NewForbiddenError("No permission to operate") + } + + return kb, id, nil +} + +// GetKnowledgeBase handles requests to retrieve a knowledge base by ID +func (h *KnowledgeBaseHandler) GetKnowledgeBase(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start retrieving knowledge base") + + // Validate and get the knowledge base + kb, id, err := h.validateAndGetKnowledgeBase(c) + if err != nil { + c.Error(err) + return + } + + logger.Infof(ctx, "Retrieved knowledge base successfully, ID: %s, name: %s", id, kb.Name) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": kb, + }) +} + +// ListKnowledgeBases handles requests to list all knowledge bases for a tenant +func (h *KnowledgeBaseHandler) ListKnowledgeBases(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving knowledge base list") + + // Get tenant ID from context + tenantID, exists := c.Get(types.TenantIDContextKey.String()) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + c.Error(errors.NewUnauthorizedError("Unauthorized")) + return + } + + logger.Infof(ctx, "Retrieving knowledge base list for tenant, tenant ID: %d", tenantID.(uint)) + + // Get all knowledge bases for this tenant + kbs, err := h.service.ListKnowledgeBases(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof( + ctx, + "Retrieved knowledge base list successfully, tenant ID: %d, total: %d knowledge bases", + tenantID.(uint), len(kbs), + ) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": kbs, + }) +} + +// UpdateKnowledgeBaseRequest defines the request body structure for updating a knowledge base +type UpdateKnowledgeBaseRequest struct { + Name string `json:"name" binding:"required"` + Description string `json:"description"` + Config *types.KnowledgeBaseConfig `json:"config" binding:"required"` +} + +// UpdateKnowledgeBase handles requests to update an existing knowledge base +func (h *KnowledgeBaseHandler) UpdateKnowledgeBase(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start updating knowledge base") + + // Validate and get the knowledge base + _, id, err := h.validateAndGetKnowledgeBase(c) + if err != nil { + c.Error(err) + return + } + + // Parse request body + var req UpdateKnowledgeBaseRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error())) + return + } + + logger.Infof(ctx, "Updating knowledge base, ID: %s, name: %s", id, req.Name) + + // Update the knowledge base + kb, err := h.service.UpdateKnowledgeBase(ctx, id, req.Name, req.Description, req.Config) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge base updated successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": kb, + }) +} + +// DeleteKnowledgeBase handles requests to delete a knowledge base +func (h *KnowledgeBaseHandler) DeleteKnowledgeBase(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start deleting knowledge base") + + // Validate and get the knowledge base + kb, id, err := h.validateAndGetKnowledgeBase(c) + if err != nil { + c.Error(err) + return + } + + logger.Infof(ctx, "Deleting knowledge base, ID: %s, name: %s", id, kb.Name) + + // Delete the knowledge base + if err := h.service.DeleteKnowledgeBase(ctx, id); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge base deleted successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Knowledge base deleted successfully", + }) +} + +type CopyKnowledgeBaseRequest struct { + SourceID string `json:"source_id" binding:"required"` + TargetID string `json:"target_id"` +} + +func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Start copy knowledge base") + + var req CopyKnowledgeBaseRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error())) + return + } + + logger.Infof(ctx, "Copy knowledge base, ID: %s to ID: %s", req.SourceID, req.TargetID) + + go func(ctx context.Context) { + err := h.knowledgeService.CloneKnowledgeBase(ctx, req.SourceID, req.TargetID) + if err != nil { + logger.Errorf(ctx, "Failed to copy knowledge base, ID: %s to ID: %s", req.SourceID, req.TargetID) + return + } + logger.Infof(ctx, "Knowledge base copy from ID: %s to ID: %s successfully", req.SourceID, req.TargetID) + }(logger.CloneContext(ctx)) + + logger.Infof(ctx, "Knowledge base start copy from ID: %s to ID: %s", req.SourceID, req.TargetID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Knowledge base copy successfully", + }) +} diff --git a/internal/handler/message.go b/internal/handler/message.go new file mode 100644 index 0000000..f84a18c --- /dev/null +++ b/internal/handler/message.go @@ -0,0 +1,135 @@ +package handler + +import ( + "net/http" + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// MessageHandler handles HTTP requests related to messages within chat sessions +// It provides endpoints for loading and managing message history +type MessageHandler struct { + MessageService interfaces.MessageService // Service that implements message business logic +} + +// NewMessageHandler creates a new message handler instance with the required service +// Parameters: +// - messageService: Service that implements message business logic +// +// Returns a pointer to a new MessageHandler +func NewMessageHandler(messageService interfaces.MessageService) *MessageHandler { + return &MessageHandler{ + MessageService: messageService, + } +} + +// LoadMessages handles requests to load message history +// It supports both loading recent messages and loading messages before a specific timestamp +// This endpoint is used for scrolling through conversation history +func (h *MessageHandler) LoadMessages(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start loading messages") + + // Get path parameters and query parameters + sessionID := c.Param("session_id") + limit := c.DefaultQuery("limit", "20") + beforeTimeStr := c.DefaultQuery("before_time", "") + + logger.Infof(ctx, "Loading messages params, session ID: %s, limit: %s, before time: %s", + sessionID, limit, beforeTimeStr) + + // Parse limit parameter with fallback to default + limitInt, err := strconv.Atoi(limit) + if err != nil { + logger.Warnf(ctx, "Invalid limit value, using default value 20, input: %s", limit) + limitInt = 20 + } + + // If no beforeTime is provided, retrieve the most recent messages + if beforeTimeStr == "" { + logger.Infof(ctx, "Getting recent messages for session, session ID: %s, limit: %d", sessionID, limitInt) + messages, err := h.MessageService.GetRecentMessagesBySession(ctx, sessionID, limitInt) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof( + ctx, + "Successfully retrieved recent messages, session ID: %s, message count: %d", + sessionID, len(messages), + ) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": messages, + }) + return + } + + // If beforeTime is provided, parse the timestamp + beforeTime, err := time.Parse(time.RFC3339Nano, beforeTimeStr) + if err != nil { + logger.Errorf( + ctx, + "Invalid time format, please use RFC3339Nano format, err: %v, beforeTimeStr: %s", + err, beforeTimeStr, + ) + c.Error(errors.NewBadRequestError("Invalid time format, please use RFC3339Nano format")) + return + } + + // Retrieve messages before the specified timestamp + logger.Infof(ctx, "Getting messages before specific time, session ID: %s, before time: %s, limit: %d", + sessionID, beforeTime.Format(time.RFC3339Nano), limitInt) + messages, err := h.MessageService.GetMessagesBySessionBeforeTime(ctx, sessionID, beforeTime, limitInt) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof( + ctx, + "Successfully retrieved messages before time, session ID: %s, message count: %d", + sessionID, len(messages), + ) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": messages, + }) +} + +// DeleteMessage handles requests to delete a message from a session +// It requires both session ID and message ID to identify the specific message to delete +func (h *MessageHandler) DeleteMessage(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start deleting message") + + // Get path parameters for session and message identification + sessionID := c.Param("session_id") + messageID := c.Param("id") + + logger.Infof(ctx, "Deleting message, session ID: %s, message ID: %s", sessionID, messageID) + + // Delete the message using the message service + if err := h.MessageService.DeleteMessage(ctx, sessionID, messageID); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Message deleted successfully, session ID: %s, message ID: %s", sessionID, messageID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Message deleted successfully", + }) +} diff --git a/internal/handler/model.go b/internal/handler/model.go new file mode 100644 index 0000000..7be800a --- /dev/null +++ b/internal/handler/model.go @@ -0,0 +1,265 @@ +package handler + +import ( + "net/http" + + "github.com/Tencent/WeKnora/internal/application/service" + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// ModelHandler handles HTTP requests for model-related operations +// It implements the necessary methods to create, retrieve, update, and delete models +type ModelHandler struct { + service interfaces.ModelService +} + +// NewModelHandler creates a new instance of ModelHandler +// It requires a model service implementation that handles business logic +// Parameters: +// - service: An implementation of the ModelService interface +// +// Returns a pointer to the newly created ModelHandler +func NewModelHandler(service interfaces.ModelService) *ModelHandler { + return &ModelHandler{service: service} +} + +// CreateModelRequest defines the structure for model creation requests +// Contains all fields required to create a new model in the system +type CreateModelRequest struct { + Name string `json:"name" binding:"required"` + Type types.ModelType `json:"type" binding:"required"` + Source types.ModelSource `json:"source" binding:"required"` + Description string `json:"description"` + Parameters types.ModelParameters `json:"parameters" binding:"required"` + IsDefault bool `json:"is_default"` +} + +// CreateModel handles the HTTP request to create a new model +// It validates the request, processes it using the model service, +// and returns the created model to the client +// Parameters: +// - c: Gin context for the HTTP request +func (h *ModelHandler) CreateModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start creating model") + + var req CreateModelRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + tenantID := c.GetUint(types.TenantIDContextKey.String()) + if tenantID == 0 { + logger.Error(ctx, "Tenant ID is empty") + c.Error(errors.NewBadRequestError("Tenant ID cannot be empty")) + return + } + + logger.Infof(ctx, "Creating model, Tenant ID: %d, Model name: %s, Model type: %s", + tenantID, req.Name, req.Type) + + model := &types.Model{ + TenantID: tenantID, + Name: req.Name, + Type: req.Type, + Source: req.Source, + Description: req.Description, + Parameters: req.Parameters, + IsDefault: req.IsDefault, + } + + if err := h.service.CreateModel(ctx, model); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Model created successfully, ID: %s, Name: %s", model.ID, model.Name) + c.JSON(http.StatusCreated, gin.H{ + "success": true, + "data": model, + }) +} + +// GetModel handles the HTTP request to retrieve a model by its ID +// It fetches the model from the service and returns it to the client, +// or returns appropriate error messages if the model cannot be found +// Parameters: +// - c: Gin context for the HTTP request +func (h *ModelHandler) GetModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving model") + + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Model ID is empty") + c.Error(errors.NewBadRequestError("Model ID cannot be empty")) + return + } + + logger.Infof(ctx, "Retrieving model, ID: %s", id) + model, err := h.service.GetModelByID(ctx, id) + if err != nil { + if err == service.ErrModelNotFound { + logger.Warnf(ctx, "Model not found, ID: %s", id) + c.Error(errors.NewNotFoundError("Model not found")) + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Retrieved model successfully, ID: %s, Name: %s", model.ID, model.Name) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": model, + }) +} + +// ListModels handles the HTTP request to retrieve all models for a tenant +// It validates the tenant ID, fetches models from the service, and returns them to the client +// Parameters: +// - c: Gin context for the HTTP request +func (h *ModelHandler) ListModels(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving model list") + + tenantID := c.GetUint(types.TenantIDContextKey.String()) + if tenantID == 0 { + logger.Error(ctx, "Tenant ID is empty") + c.Error(errors.NewBadRequestError("Tenant ID cannot be empty")) + return + } + + logger.Infof(ctx, "Retrieving model list, Tenant ID: %d", tenantID) + models, err := h.service.ListModels(ctx) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Retrieved model list successfully, Tenant ID: %d, Total: %d models", tenantID, len(models)) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": models, + }) +} + +// UpdateModelRequest defines the structure for model update requests +// Contains fields that can be updated for an existing model +type UpdateModelRequest struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters types.ModelParameters `json:"parameters"` + IsDefault bool `json:"is_default"` +} + +// UpdateModel handles the HTTP request to update an existing model +// It validates the request, retrieves the current model, applies changes, +// and updates the model in the service +// Parameters: +// - c: Gin context for the HTTP request +func (h *ModelHandler) UpdateModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start updating model") + + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Model ID is empty") + c.Error(errors.NewBadRequestError("Model ID cannot be empty")) + return + } + + var req UpdateModelRequest + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + logger.Infof(ctx, "Retrieving model information, ID: %s", id) + model, err := h.service.GetModelByID(ctx, id) + if err != nil { + if err == service.ErrModelNotFound { + logger.Warnf(ctx, "Model not found, ID: %s", id) + c.Error(errors.NewNotFoundError("Model not found")) + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Update model fields if they are provided in the request + if req.Name != "" { + model.Name = req.Name + } + if req.Description != "" { + model.Description = req.Description + } + if req.Parameters != (types.ModelParameters{}) { + model.Parameters = req.Parameters + } + model.IsDefault = req.IsDefault + + logger.Infof(ctx, "Updating model, ID: %s, Name: %s", id, model.Name) + if err := h.service.UpdateModel(ctx, model); err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Model updated successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": model, + }) +} + +// DeleteModel handles the HTTP request to delete a model by its ID +// It validates the model ID, attempts to delete the model through the service, +// and returns appropriate status and messages +// Parameters: +// - c: Gin context for the HTTP request +func (h *ModelHandler) DeleteModel(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start deleting model") + + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Model ID is empty") + c.Error(errors.NewBadRequestError("Model ID cannot be empty")) + return + } + + logger.Infof(ctx, "Deleting model, ID: %s", id) + if err := h.service.DeleteModel(ctx, id); err != nil { + if err == service.ErrModelNotFound { + logger.Warnf(ctx, "Model not found, ID: %s", id) + c.Error(errors.NewNotFoundError("Model not found")) + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Model deleted successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Model deleted", + }) +} diff --git a/internal/handler/session.go b/internal/handler/session.go new file mode 100644 index 0000000..374a614 --- /dev/null +++ b/internal/handler/session.go @@ -0,0 +1,805 @@ +package handler + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// SessionHandler handles all HTTP requests related to conversation sessions +type SessionHandler struct { + messageService interfaces.MessageService // Service for managing messages + sessionService interfaces.SessionService // Service for managing sessions + streamManager interfaces.StreamManager // Manager for handling streaming responses + config *config.Config // Application configuration + knowledgebaseService interfaces.KnowledgeBaseService +} + +// NewSessionHandler creates a new instance of SessionHandler with all necessary dependencies +func NewSessionHandler( + sessionService interfaces.SessionService, + messageService interfaces.MessageService, + streamManager interfaces.StreamManager, + config *config.Config, + knowledgebaseService interfaces.KnowledgeBaseService, +) *SessionHandler { + return &SessionHandler{ + sessionService: sessionService, + messageService: messageService, + streamManager: streamManager, + config: config, + knowledgebaseService: knowledgebaseService, + } +} + +// SessionStrategy defines the configuration for a conversation session strategy +type SessionStrategy struct { + // Maximum number of conversation rounds to maintain + MaxRounds int `json:"max_rounds"` + // Whether to enable query rewrite for multi-round conversations + EnableRewrite bool `json:"enable_rewrite"` + // Strategy to use when no relevant knowledge is found + FallbackStrategy types.FallbackStrategy `json:"fallback_strategy"` + // Fixed response content for fallback + FallbackResponse string `json:"fallback_response"` + // Number of top results to retrieve from vector search + EmbeddingTopK int `json:"embedding_top_k"` + // Threshold for keyword-based retrieval + KeywordThreshold float64 `json:"keyword_threshold"` + // Threshold for vector-based retrieval + VectorThreshold float64 `json:"vector_threshold"` + // ID of the model used for reranking results + RerankModelID string `json:"rerank_model_id"` + // Number of top results after reranking + RerankTopK int `json:"rerank_top_k"` + // Threshold for reranking results + RerankThreshold float64 `json:"rerank_threshold"` + // ID of the model used for summarization + SummaryModelID string `json:"summary_model_id"` + // Parameters for the summary model + SummaryParameters *types.SummaryConfig `json:"summary_parameters" gorm:"type:json"` + // Prefix for responses when no match is found + NoMatchPrefix string `json:"no_match_prefix"` +} + +// CreateSessionRequest represents a request to create a new session +type CreateSessionRequest struct { + // ID of the associated knowledge base + KnowledgeBaseID string `json:"knowledge_base_id" binding:"required"` + // Session strategy configuration + SessionStrategy *SessionStrategy `json:"session_strategy"` +} + +// CreateSession handles the creation of a new conversation session +func (h *SessionHandler) CreateSession(c *gin.Context) { + ctx := c.Request.Context() + + // logger.Infof(ctx, "Start creating session, config: %+v", h.config.Conversation) + + // Parse and validate the request body + var request CreateSessionRequest + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "Failed to validate session creation parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // Get tenant ID from context + tenantID, exists := c.Get(types.TenantIDContextKey.String()) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + c.Error(errors.NewUnauthorizedError("Unauthorized")) + return + } + + // Validate knowledge base ID + if request.KnowledgeBaseID == "" { + logger.Error(ctx, "Knowledge base ID is empty") + c.Error(errors.NewBadRequestError("Knowledge base cannot be empty")) + return + } + + logger.Infof( + ctx, + "Processing session creation request, tenant ID: %d, knowledge base ID: %s", + tenantID.(uint), + request.KnowledgeBaseID, + ) + + // Create session object with base properties + createdSession := &types.Session{ + TenantID: tenantID.(uint), + KnowledgeBaseID: request.KnowledgeBaseID, + } + + // If summary model parameters are empty, set defaults + if request.SessionStrategy != nil { + createdSession.RerankModelID = request.SessionStrategy.RerankModelID + createdSession.SummaryModelID = request.SessionStrategy.SummaryModelID + createdSession.MaxRounds = request.SessionStrategy.MaxRounds + createdSession.EnableRewrite = request.SessionStrategy.EnableRewrite + createdSession.FallbackStrategy = request.SessionStrategy.FallbackStrategy + createdSession.FallbackResponse = request.SessionStrategy.FallbackResponse + createdSession.EmbeddingTopK = request.SessionStrategy.EmbeddingTopK + createdSession.KeywordThreshold = request.SessionStrategy.KeywordThreshold + createdSession.VectorThreshold = request.SessionStrategy.VectorThreshold + createdSession.RerankTopK = request.SessionStrategy.RerankTopK + createdSession.RerankThreshold = request.SessionStrategy.RerankThreshold + if request.SessionStrategy.SummaryParameters != nil { + createdSession.SummaryParameters = request.SessionStrategy.SummaryParameters + } else { + createdSession.SummaryParameters = &types.SummaryConfig{ + MaxTokens: h.config.Conversation.Summary.MaxTokens, + TopP: h.config.Conversation.Summary.TopP, + TopK: h.config.Conversation.Summary.TopK, + FrequencyPenalty: h.config.Conversation.Summary.FrequencyPenalty, + PresencePenalty: h.config.Conversation.Summary.PresencePenalty, + RepeatPenalty: h.config.Conversation.Summary.RepeatPenalty, + NoMatchPrefix: h.config.Conversation.Summary.NoMatchPrefix, + Temperature: h.config.Conversation.Summary.Temperature, + Seed: h.config.Conversation.Summary.Seed, + MaxCompletionTokens: h.config.Conversation.Summary.MaxCompletionTokens, + } + } + if createdSession.SummaryParameters.Prompt == "" { + createdSession.SummaryParameters.Prompt = h.config.Conversation.Summary.Prompt + } + if createdSession.SummaryParameters.ContextTemplate == "" { + createdSession.SummaryParameters.ContextTemplate = h.config.Conversation.Summary.ContextTemplate + } + if createdSession.SummaryParameters.NoMatchPrefix == "" { + createdSession.SummaryParameters.NoMatchPrefix = h.config.Conversation.Summary.NoMatchPrefix + } + + logger.Debug(ctx, "Custom session strategy set") + } else { + // Use default configuration from global config + createdSession.MaxRounds = h.config.Conversation.MaxRounds + createdSession.EnableRewrite = h.config.Conversation.EnableRewrite + createdSession.FallbackStrategy = types.FallbackStrategy(h.config.Conversation.FallbackStrategy) + createdSession.FallbackResponse = h.config.Conversation.FallbackResponse + createdSession.EmbeddingTopK = h.config.Conversation.EmbeddingTopK + createdSession.KeywordThreshold = h.config.Conversation.KeywordThreshold + createdSession.VectorThreshold = h.config.Conversation.VectorThreshold + createdSession.RerankThreshold = h.config.Conversation.RerankThreshold + createdSession.RerankTopK = h.config.Conversation.RerankTopK + createdSession.SummaryParameters = &types.SummaryConfig{ + MaxTokens: h.config.Conversation.Summary.MaxTokens, + TopP: h.config.Conversation.Summary.TopP, + TopK: h.config.Conversation.Summary.TopK, + FrequencyPenalty: h.config.Conversation.Summary.FrequencyPenalty, + PresencePenalty: h.config.Conversation.Summary.PresencePenalty, + RepeatPenalty: h.config.Conversation.Summary.RepeatPenalty, + Prompt: h.config.Conversation.Summary.Prompt, + ContextTemplate: h.config.Conversation.Summary.ContextTemplate, + NoMatchPrefix: h.config.Conversation.Summary.NoMatchPrefix, + Temperature: h.config.Conversation.Summary.Temperature, + Seed: h.config.Conversation.Summary.Seed, + MaxCompletionTokens: h.config.Conversation.Summary.MaxCompletionTokens, + } + + logger.Debug(ctx, "Using default session strategy") + } + + kb, err := h.knowledgebaseService.GetKnowledgeBaseByID(ctx, request.KnowledgeBaseID) + if err != nil { + logger.Error(ctx, "Failed to get knowledge base", err) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Get model IDs from knowledge base if not provided + if createdSession.SummaryModelID == "" { + createdSession.SummaryModelID = kb.SummaryModelID + } + if createdSession.RerankModelID == "" { + createdSession.RerankModelID = kb.RerankModelID + } + + // Call service to create session + logger.Infof(ctx, "Calling session service to create session") + createdSession, err = h.sessionService.CreateSession(ctx, createdSession) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Return created session + logger.Infof(ctx, "Session created successfully, ID: %s", createdSession.ID) + c.JSON(http.StatusCreated, gin.H{ + "success": true, + "data": createdSession, + }) +} + +// GetSession retrieves a session by its ID +func (h *SessionHandler) GetSession(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving session") + + // Get session ID from URL parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Session ID is empty") + c.Error(errors.NewBadRequestError(errors.ErrInvalidSessionID.Error())) + return + } + + // Call service to get session details + logger.Infof(ctx, "Retrieving session, ID: %s", id) + session, err := h.sessionService.GetSession(ctx, id) + if err != nil { + if err == errors.ErrSessionNotFound { + logger.Warnf(ctx, "Session not found, ID: %s", id) + c.Error(errors.NewNotFoundError(err.Error())) + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Return session data + logger.Infof(ctx, "Session retrieved successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": session, + }) +} + +// GetSessionsByTenant retrieves all sessions for the current tenant with pagination +func (h *SessionHandler) GetSessionsByTenant(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving all sessions for tenant") + + // Parse pagination parameters from query + var pagination types.Pagination + if err := c.ShouldBindQuery(&pagination); err != nil { + logger.Error(ctx, "Failed to parse pagination parameters", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + logger.Debugf(ctx, "Using pagination parameters: page=%d, page_size=%d", pagination.Page, pagination.PageSize) + + // Use paginated query to get sessions + result, err := h.sessionService.GetPagedSessionsByTenant(ctx, &pagination) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Return sessions with pagination data + logger.Infof(ctx, "Successfully retrieved tenant sessions, total: %d", result.Total) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": result.Data, + "total": result.Total, + "page": result.Page, + "page_size": result.PageSize, + }) +} + +// UpdateSession updates an existing session's properties +func (h *SessionHandler) UpdateSession(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start updating session") + + // Get session ID from URL parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Session ID is empty") + c.Error(errors.NewBadRequestError(errors.ErrInvalidSessionID.Error())) + return + } + + // Verify tenant ID from context for authorization + tenantID, exists := c.Get(types.TenantIDContextKey.String()) + if !exists { + logger.Error(ctx, "Failed to get tenant ID") + c.Error(errors.NewUnauthorizedError("Unauthorized")) + return + } + + // Parse request body to session object + var session types.Session + if err := c.ShouldBindJSON(&session); err != nil { + logger.Error(ctx, "Failed to parse session data", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // Set session ID and tenant ID + logger.Infof(ctx, "Updating session, ID: %s, tenant ID: %d", id, tenantID.(uint)) + session.ID = id + session.TenantID = tenantID.(uint) + + // Call service to update session + if err := h.sessionService.UpdateSession(ctx, &session); err != nil { + if err == errors.ErrSessionNotFound { + logger.Warnf(ctx, "Session not found, ID: %s", id) + c.Error(errors.NewNotFoundError(err.Error())) + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Return updated session + logger.Infof(ctx, "Session updated successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": session, + }) +} + +// DeleteSession deletes a session by its ID +func (h *SessionHandler) DeleteSession(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start deleting session") + + // Get session ID from URL parameter + id := c.Param("id") + if id == "" { + logger.Error(ctx, "Session ID is empty") + c.Error(errors.NewBadRequestError(errors.ErrInvalidSessionID.Error())) + return + } + + // Call service to delete session + logger.Infof(ctx, "Deleting session, ID: %s", id) + if err := h.sessionService.DeleteSession(ctx, id); err != nil { + if err == errors.ErrSessionNotFound { + logger.Warnf(ctx, "Session not found, ID: %s", id) + c.Error(errors.NewNotFoundError(err.Error())) + return + } + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Return success message + logger.Infof(ctx, "Session deleted successfully, ID: %s", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Session deleted successfully", + }) +} + +// GenerateTitleRequest defines the request structure for generating a session title +type GenerateTitleRequest struct { + Messages []types.Message `json:"messages" binding:"required"` // Messages to use as context for title generation +} + +// GenerateTitle generates a title for a session based on message content +func (h *SessionHandler) GenerateTitle(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start generating session title") + + // Get session ID from URL parameter + sessionID := c.Param("session_id") + if sessionID == "" { + logger.Error(ctx, "Session ID is empty") + c.Error(errors.NewBadRequestError(errors.ErrInvalidSessionID.Error())) + return + } + + // Parse request body + var request GenerateTitleRequest + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "Failed to parse request data", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // Call service to generate title + logger.Infof(ctx, "Generating session title, session ID: %s, message count: %d", sessionID, len(request.Messages)) + title, err := h.sessionService.GenerateTitle(ctx, sessionID, request.Messages) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Return generated title + logger.Infof(ctx, "Session title generated successfully, session ID: %s, title: %s", sessionID, title) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": title, + }) +} + +// CreateKnowledgeQARequest defines the request structure for knowledge QA +type CreateKnowledgeQARequest struct { + Query string `json:"query" binding:"required"` // Query text for knowledge base search +} + +// SearchKnowledgeRequest defines the request structure for searching knowledge without LLM summarization +type SearchKnowledgeRequest struct { + Query string `json:"query" binding:"required"` // Query text to search for + KnowledgeBaseID string `json:"knowledge_base_id" binding:"required"` // ID of the knowledge base to search +} + +// SearchKnowledge performs knowledge base search without LLM summarization +func (h *SessionHandler) SearchKnowledge(c *gin.Context) { + ctx := logger.CloneContext(c.Request.Context()) + + logger.Info(ctx, "Start processing knowledge search request") + + // Parse request body + var request SearchKnowledgeRequest + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "Failed to parse request data", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // Validate request parameters + if request.Query == "" { + logger.Error(ctx, "Query content is empty") + c.Error(errors.NewBadRequestError("Query content cannot be empty")) + return + } + + if request.KnowledgeBaseID == "" { + logger.Error(ctx, "Knowledge base ID is empty") + c.Error(errors.NewBadRequestError("Knowledge base ID cannot be empty")) + return + } + + logger.Infof( + ctx, + "Knowledge search request, knowledge base ID: %s, query: %s", + request.KnowledgeBaseID, + request.Query, + ) + + // Directly call knowledge retrieval service without LLM summarization + searchResults, err := h.sessionService.SearchKnowledge(ctx, request.KnowledgeBaseID, request.Query) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + logger.Infof(ctx, "Knowledge search completed, found %d results", len(searchResults)) + + // Return search results + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": searchResults, + }) +} + +// ContinueStream handles continued streaming of an active response stream +func (h *SessionHandler) ContinueStream(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start continuing stream response processing") + + // Get session ID from URL parameter + sessionID := c.Param("session_id") + if sessionID == "" { + logger.Error(ctx, "Session ID is empty") + c.Error(errors.NewBadRequestError(errors.ErrInvalidSessionID.Error())) + return + } + + // Get message ID from query parameter + messageID := c.Query("message_id") + if messageID == "" { + logger.Error(ctx, "Message ID is empty") + c.Error(errors.NewBadRequestError("Missing message ID")) + return + } + + logger.Infof(ctx, "Continuing stream, session ID: %s, message ID: %s", sessionID, messageID) + + // Verify that the session exists and belongs to this tenant + _, err := h.sessionService.GetSession(ctx, sessionID) + if err != nil { + if err == errors.ErrSessionNotFound { + logger.Warnf(ctx, "Session not found, ID: %s", sessionID) + c.Error(errors.NewNotFoundError(err.Error())) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + } + return + } + + // Get the incomplete message + message, err := h.messageService.GetMessage(ctx, sessionID, messageID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + if message == nil { + logger.Warnf(ctx, "Incomplete message not found, session ID: %s, message ID: %s", sessionID, messageID) + c.JSON(http.StatusNotFound, gin.H{ + "success": false, + "error": "Incomplete message not found", + }) + return + } + + // Get stream information + streamInfo, err := h.streamManager.GetStream(ctx, sessionID, messageID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(fmt.Sprintf("Failed to get stream data: %s", err.Error()))) + return + } + + if streamInfo == nil { + logger.Warnf(ctx, "Active stream not found, session ID: %s, message ID: %s", sessionID, messageID) + c.JSON(http.StatusNotFound, gin.H{ + "success": false, + "error": "Active stream not found", + }) + return + } + + // If stream is already completed, return the full message + if streamInfo.IsCompleted { + logger.Infof( + ctx, "Stream already completed, returning directly, session ID: %s, message ID: %s", sessionID, messageID, + ) + c.JSON(http.StatusOK, gin.H{ + "id": message.ID, + "role": message.Role, + "content": message.Content, + "created_at": message.CreatedAt, + "done": true, + }) + return + } + + logger.Infof( + ctx, "Preparing to set SSE headers and send stream data, session ID: %s, message ID: %s", sessionID, messageID, + ) + + // Send knowledge references first if available + if len(streamInfo.KnowledgeReferences) > 0 { + logger.Debug(ctx, "Sending knowledge references") + c.SSEvent("message", &types.StreamResponse{ + ID: message.RequestID, + ResponseType: types.ResponseTypeReferences, + Done: false, + KnowledgeReferences: streamInfo.KnowledgeReferences, + }) + } + + // Send existing content + if streamInfo.Content != "" { + logger.Debug(ctx, "Sending current existing content") + c.SSEvent("message", &types.StreamResponse{ + ID: message.RequestID, + ResponseType: types.ResponseTypeAnswer, + Content: streamInfo.Content, + Done: streamInfo.IsCompleted, + }) + } + + // Create channels to monitor content updates + contentCh := make(chan string, 10) + doneCh := make(chan bool, 1) + + logger.Debug(ctx, "Starting content update monitoring") + + // Start a goroutine to monitor for content updates + go func() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + currentContent := streamInfo.Content + + for { + select { + case <-ticker.C: + latestStreamInfo, err := h.streamManager.GetStream(ctx, sessionID, messageID) + if err != nil { + logger.Errorf(ctx, "Failed to get stream data: %v", err) + doneCh <- true + return + } + + if latestStreamInfo == nil { + logger.Debug(ctx, "Stream no longer exists") + doneCh <- true + return + } + + if latestStreamInfo.IsCompleted { + logger.Debug(ctx, "Stream completed") + doneCh <- true + return + } + + // Calculate new content delta + if len(latestStreamInfo.Content) > len(currentContent) { + newContent := latestStreamInfo.Content[len(currentContent):] + contentCh <- newContent + currentContent = latestStreamInfo.Content + logger.Debugf(ctx, "Sending new content: %d bytes", len(newContent)) + } + + case <-c.Request.Context().Done(): + logger.Debug(ctx, "Client connection closed") + return + } + } + }() + + logger.Info(ctx, "Starting stream response") + + // Stream updated content to client + c.Stream(func(w io.Writer) bool { + select { + case <-c.Request.Context().Done(): + logger.Debug(ctx, "Client connection closed") + return false + + case <-doneCh: + logger.Debug(ctx, "Stream completed, sending completion notification") + c.SSEvent("message", &types.StreamResponse{ + ID: message.RequestID, + ResponseType: types.ResponseTypeAnswer, + Content: "", + Done: true, + }) + return false + + case content := <-contentCh: + logger.Debugf(ctx, "Sending content fragment: %d bytes", len(content)) + c.SSEvent("message", &types.StreamResponse{ + ID: message.RequestID, + ResponseType: types.ResponseTypeAnswer, + Content: content, + Done: false, + }) + return true + } + }) +} + +// KnowledgeQA handles knowledge base question answering requests with LLM summarization +func (h *SessionHandler) KnowledgeQA(c *gin.Context) { + ctx := logger.CloneContext(c.Request.Context()) + + logger.Info(ctx, "Start processing knowledge QA request") + + // Get session ID from URL parameter + sessionID := c.Param("session_id") + if sessionID == "" { + logger.Error(ctx, "Session ID is empty") + c.Error(errors.NewBadRequestError(errors.ErrInvalidSessionID.Error())) + return + } + + // Parse request body + var request CreateKnowledgeQARequest + if err := c.ShouldBindJSON(&request); err != nil { + logger.Error(ctx, "Failed to parse request data", err) + c.Error(errors.NewBadRequestError(err.Error())) + return + } + + // Create assistant message + assistantMessage := &types.Message{ + SessionID: sessionID, + Role: "assistant", + RequestID: c.GetString(types.RequestIDContextKey.String()), + IsCompleted: false, + } + defer h.completeAssistantMessage(ctx, assistantMessage) + + // Validate query content + if request.Query == "" { + logger.Error(ctx, "Query content is empty") + c.Error(errors.NewBadRequestError("Query content cannot be empty")) + return + } + + logger.Infof(ctx, "Knowledge QA request, session ID: %s, query: %s", sessionID, request.Query) + + // Create user message + if _, err := h.messageService.CreateMessage(ctx, &types.Message{ + SessionID: sessionID, + Role: "user", + Content: request.Query, + RequestID: c.GetString(types.RequestIDContextKey.String()), + CreatedAt: time.Now(), + IsCompleted: true, + }); err != nil { + c.Error(errors.NewInternalServerError(err.Error())) + return + } + + // Create assistant message (response) + assistantMessage.CreatedAt = time.Now() + if _, err := h.messageService.CreateMessage(ctx, assistantMessage); err != nil { + c.Error(errors.NewInternalServerError(err.Error())) + return + } + logger.Infof(ctx, "Calling knowledge QA service, session ID: %s", sessionID) + + // Call service to perform knowledge QA + searchResults, respCh, err := h.sessionService.KnowledgeQA(ctx, sessionID, request.Query) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError(err.Error())) + return + } + assistantMessage.KnowledgeReferences = searchResults + + // Register new stream with stream manager + requestID := c.GetString(types.RequestIDContextKey.String()) + if err := h.streamManager.RegisterStream(ctx, sessionID, assistantMessage.ID, request.Query); err != nil { + logger.GetLogger(ctx).Error("Register stream failed", "error", err) + } + + // Send knowledge references if available + if len(searchResults) > 0 { + logger.Debugf(ctx, "Sending reference content, total %d", len(searchResults)) + c.SSEvent("message", &types.StreamResponse{ + ID: requestID, + ResponseType: types.ResponseTypeReferences, + KnowledgeReferences: searchResults, + }) + c.Writer.Flush() + } else { + logger.Debug(ctx, "No reference content to send") + } + + // Process streamed response + func() { + defer func() { + // Mark stream as completed when done + if err := h.streamManager.CompleteStream(ctx, sessionID, assistantMessage.ID); err != nil { + logger.GetLogger(ctx).Error("Complete stream failed", "error", err) + } + }() + for response := range respCh { + response.ID = requestID + c.SSEvent("message", response) + c.Writer.Flush() + if response.ResponseType == types.ResponseTypeAnswer { + assistantMessage.Content += response.Content + // Update stream manager with new content + if err := h.streamManager.UpdateStream( + ctx, sessionID, assistantMessage.ID, response.Content, searchResults, + ); err != nil { + logger.GetLogger(ctx).Error("Update stream content failed", "error", err) + } + } + } + }() +} + +// completeAssistantMessage marks an assistant message as complete and updates it +func (h *SessionHandler) completeAssistantMessage(ctx context.Context, assistantMessage *types.Message) { + assistantMessage.UpdatedAt = time.Now() + assistantMessage.IsCompleted = true + _ = h.messageService.UpdateMessage(ctx, assistantMessage) +} diff --git a/internal/handler/system.go b/internal/handler/system.go new file mode 100644 index 0000000..43329d9 --- /dev/null +++ b/internal/handler/system.go @@ -0,0 +1,49 @@ +package handler + +import ( + "github.com/Tencent/WeKnora/internal/logger" + "github.com/gin-gonic/gin" +) + +// SystemHandler handles system-related requests +type SystemHandler struct{} + +// NewSystemHandler creates a new system handler +func NewSystemHandler() *SystemHandler { + return &SystemHandler{} +} + +// GetSystemInfoResponse defines the response structure for system info +type GetSystemInfoResponse struct { + Version string `json:"version"` + CommitID string `json:"commit_id,omitempty"` + BuildTime string `json:"build_time,omitempty"` + GoVersion string `json:"go_version,omitempty"` +} + +// 编译时注入的版本信息 +var ( + Version = "unknown" + CommitID = "unknown" + BuildTime = "unknown" + GoVersion = "unknown" +) + +// GetSystemInfo gets system information including version and commit ID +func (h *SystemHandler) GetSystemInfo(c *gin.Context) { + ctx := logger.CloneContext(c.Request.Context()) + + response := GetSystemInfoResponse{ + Version: Version, + CommitID: CommitID, + BuildTime: BuildTime, + GoVersion: GoVersion, + } + + logger.Info(ctx, "System info retrieved successfully") + c.JSON(200, gin.H{ + "code": 0, + "msg": "success", + "data": response, + }) +} diff --git a/internal/handler/tenant.go b/internal/handler/tenant.go new file mode 100644 index 0000000..c862894 --- /dev/null +++ b/internal/handler/tenant.go @@ -0,0 +1,226 @@ +package handler + +import ( + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// TenantHandler implements HTTP request handlers for tenant management +// Provides functionality for creating, retrieving, updating, and deleting tenants +// through the REST API endpoints +type TenantHandler struct { + service interfaces.TenantService +} + +// NewTenantHandler creates a new tenant handler instance with the provided service +// Parameters: +// - service: An implementation of the TenantService interface for business logic +// +// Returns a pointer to the newly created TenantHandler +func NewTenantHandler(service interfaces.TenantService) *TenantHandler { + return &TenantHandler{ + service: service, + } +} + +// CreateTenant handles the HTTP request for creating a new tenant +// It deserializes the request body into a tenant object, validates it, +// calls the service to create the tenant, and returns the result +// Parameters: +// - c: Gin context for the HTTP request +func (h *TenantHandler) CreateTenant(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start creating tenant") + + var tenantData types.Tenant + if err := c.ShouldBindJSON(&tenantData); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + appErr := errors.NewValidationError("Invalid request parameters").WithDetails(err.Error()) + c.Error(appErr) + return + } + + logger.Infof(ctx, "Creating tenant, name: %s", tenantData.Name) + + createdTenant, err := h.service.CreateTenant(ctx, &tenantData) + if err != nil { + // Check if this is an application-specific error + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to create tenant: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to create tenant").WithDetails(err.Error())) + } + return + } + + logger.Infof(ctx, "Tenant created successfully, ID: %d, name: %s", createdTenant.ID, createdTenant.Name) + c.JSON(http.StatusCreated, gin.H{ + "success": true, + "data": createdTenant, + }) +} + +// GetTenant handles the HTTP request for retrieving a tenant by ID +// It extracts and validates the tenant ID from the URL parameter, +// retrieves the tenant from the service, and returns it in the response +// Parameters: +// - c: Gin context for the HTTP request +func (h *TenantHandler) GetTenant(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving tenant") + + id, err := strconv.ParseUint(c.Param("id"), 10, 64) + if err != nil { + logger.Errorf(ctx, "Invalid tenant ID: %s", c.Param("id")) + c.Error(errors.NewBadRequestError("Invalid tenant ID")) + return + } + + logger.Infof(ctx, "Retrieving tenant, ID: %d", id) + + tenant, err := h.service.GetTenantByID(ctx, uint(id)) + if err != nil { + // Check if this is an application-specific error + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to retrieve tenant: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to retrieve tenant").WithDetails(err.Error())) + } + return + } + + logger.Infof(ctx, "Retrieved tenant successfully, ID: %d, Name: %s", tenant.ID, tenant.Name) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": tenant, + }) +} + +// UpdateTenant handles the HTTP request for updating an existing tenant +// It extracts the tenant ID from the URL parameter, deserializes the request body, +// validates the data, updates the tenant through the service, and returns the result +// Parameters: +// - c: Gin context for the HTTP request +func (h *TenantHandler) UpdateTenant(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start updating tenant") + + id, err := strconv.ParseUint(c.Param("id"), 10, 64) + if err != nil { + logger.Errorf(ctx, "Invalid tenant ID: %s", c.Param("id")) + c.Error(errors.NewBadRequestError("Invalid tenant ID")) + return + } + + var tenantData types.Tenant + if err := c.ShouldBindJSON(&tenantData); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewValidationError("Invalid request data").WithDetails(err.Error())) + return + } + + logger.Infof(ctx, "Updating tenant, ID: %d, Name: %s", id, tenantData.Name) + + tenantData.ID = uint(id) + updatedTenant, err := h.service.UpdateTenant(ctx, &tenantData) + if err != nil { + // Check if this is an application-specific error + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to update tenant: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to update tenant").WithDetails(err.Error())) + } + return + } + + logger.Infof(ctx, "Tenant updated successfully, ID: %d, Name: %s", updatedTenant.ID, updatedTenant.Name) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": updatedTenant, + }) +} + +// DeleteTenant handles the HTTP request for deleting a tenant +// It extracts and validates the tenant ID from the URL parameter, +// calls the service to delete the tenant, and returns the result +// Parameters: +// - c: Gin context for the HTTP request +func (h *TenantHandler) DeleteTenant(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start deleting tenant") + + id, err := strconv.ParseUint(c.Param("id"), 10, 64) + if err != nil { + logger.Errorf(ctx, "Invalid tenant ID: %s", c.Param("id")) + c.Error(errors.NewBadRequestError("Invalid tenant ID")) + return + } + + logger.Infof(ctx, "Deleting tenant, ID: %d", id) + + if err := h.service.DeleteTenant(ctx, uint(id)); err != nil { + // Check if this is an application-specific error + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to delete tenant: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to delete tenant").WithDetails(err.Error())) + } + return + } + + logger.Infof(ctx, "Tenant deleted successfully, ID: %d", id) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "message": "Tenant deleted successfully", + }) +} + +// ListTenants handles the HTTP request for retrieving a list of all tenants +// It calls the service to fetch the tenant list and returns it in the response +// Parameters: +// - c: Gin context for the HTTP request +func (h *TenantHandler) ListTenants(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start retrieving tenant list") + + tenants, err := h.service.ListTenants(ctx) + if err != nil { + // Check if this is an application-specific error + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to retrieve tenant list: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to retrieve tenant list").WithDetails(err.Error())) + } + return + } + + logger.Infof(ctx, "Retrieved tenant list successfully, Total: %d tenants", len(tenants)) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "items": tenants, + }, + }) +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..fd9ef9d --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,252 @@ +package logger + +import ( + "context" + "fmt" + "path" + "runtime" + "sort" + "strings" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/sirupsen/logrus" +) + +// LogLevel 日志级别类型 +type LogLevel string + +// 日志级别常量 +const ( + LevelDebug LogLevel = "debug" + LevelInfo LogLevel = "info" + LevelWarn LogLevel = "warn" + LevelError LogLevel = "error" + LevelFatal LogLevel = "fatal" +) + +// ANSI颜色代码 +const ( + colorRed = "\033[31m" + colorGreen = "\033[32m" + colorYellow = "\033[33m" + colorBlue = "\033[34m" + colorPurple = "\033[35m" + colorCyan = "\033[36m" + colorReset = "\033[0m" +) + +type CustomFormatter struct { + ForceColor bool // 是否强制使用颜色，即使在非终端环境下 +} + +func (f *CustomFormatter) Format(entry *logrus.Entry) ([]byte, error) { + timestamp := entry.Time.Format("2006-01-02 15:04:05.000") + level := strings.ToUpper(entry.Level.String()) + + // 根据日志级别设置颜色 + var levelColor, resetColor string + if f.ForceColor { + switch entry.Level { + case logrus.DebugLevel: + levelColor = colorCyan + case logrus.InfoLevel: + levelColor = colorGreen + case logrus.WarnLevel: + levelColor = colorYellow + case logrus.ErrorLevel: + levelColor = colorRed + case logrus.FatalLevel: + levelColor = colorPurple + default: + levelColor = colorReset + } + resetColor = colorReset + } + + // 取出 caller 字段 + caller := "" + if val, ok := entry.Data["caller"]; ok { + caller = fmt.Sprintf("%v", val) + } + + // 拼接字段部分：request_id 优先，其他排序后输出 + fields := "" + + // request_id 优先输出 + if v, ok := entry.Data["request_id"]; ok { + fields += fmt.Sprintf("request_id=%v ", v) + } + + // 其余字段排序后输出 + keys := make([]string, 0, len(entry.Data)) + for k := range entry.Data { + if k != "caller" && k != "request_id" { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, k := range keys { + fields += fmt.Sprintf("%s=%v ", k, entry.Data[k]) + } + + fields = strings.TrimSpace(fields) + + // 拼接最终输出内容，添加颜色 + return []byte(fmt.Sprintf("%s%-5s%s[%s] [%s] %-20s | %s\n", + levelColor, level, resetColor, timestamp, fields, caller, entry.Message)), nil +} + +// 初始化全局日志设置 +func init() { + // 设置日志格式而不修改全局时区 + logrus.SetFormatter(&CustomFormatter{ForceColor: true}) + logrus.SetReportCaller(false) +} + +// GetLogger 获取日志实例 +func GetLogger(c context.Context) *logrus.Entry { + if logger := c.Value(types.LoggerContextKey); logger != nil { + return logger.(*logrus.Entry) + } + newLogger := logrus.New() + newLogger.SetFormatter(&CustomFormatter{ForceColor: true}) + // 设置默认日志级别 + newLogger.SetLevel(logrus.DebugLevel) + // 启用调用者信息 + return logrus.NewEntry(newLogger) +} + +// SetLogLevel 设置日志级别 +func SetLogLevel(level LogLevel) { + var logLevel logrus.Level + + switch level { + case LevelDebug: + logLevel = logrus.DebugLevel + case LevelInfo: + logLevel = logrus.InfoLevel + case LevelWarn: + logLevel = logrus.WarnLevel + case LevelError: + logLevel = logrus.ErrorLevel + case LevelFatal: + logLevel = logrus.FatalLevel + default: + logLevel = logrus.InfoLevel + } + + logrus.SetLevel(logLevel) +} + +// 添加调用者字段 +func addCaller(entry *logrus.Entry, skip int) *logrus.Entry { + pc, file, line, ok := runtime.Caller(skip) + if !ok { + return entry + } + shortFile := path.Base(file) + funcName := "unknown" + if fn := runtime.FuncForPC(pc); fn != nil { + // 只保留函数名，不带包路径（如 doSomething） + fullName := path.Base(fn.Name()) + parts := strings.Split(fullName, ".") + funcName = parts[len(parts)-1] + } + return entry.WithField("caller", fmt.Sprintf("%s:%d[%s]", shortFile, line, funcName)) +} + +// WithRequestID 在日志中添加请求ID +func WithRequestID(c context.Context, requestID string) context.Context { + return WithField(c, "request_id", requestID) +} + +// WithField 向日志中添加一个字段 +func WithField(c context.Context, key string, value interface{}) context.Context { + logger := GetLogger(c).WithField(key, value) + return context.WithValue(c, types.LoggerContextKey, logger) +} + +// WithFields 向日志中添加多个字段 +func WithFields(c context.Context, fields logrus.Fields) context.Context { + logger := GetLogger(c).WithFields(fields) + return context.WithValue(c, types.LoggerContextKey, logger) +} + +// Debug 输出调试级别的日志 +func Debug(c context.Context, args ...interface{}) { + addCaller(GetLogger(c), 2).Debug(args...) +} + +// Debugf 使用格式化字符串输出调试级别的日志 +func Debugf(c context.Context, format string, args ...interface{}) { + addCaller(GetLogger(c), 2).Debugf(format, args...) +} + +// Info 输出信息级别的日志 +func Info(c context.Context, args ...interface{}) { + addCaller(GetLogger(c), 2).Info(args...) +} + +// Infof 使用格式化字符串输出信息级别的日志 +func Infof(c context.Context, format string, args ...interface{}) { + addCaller(GetLogger(c), 2).Infof(format, args...) +} + +// Warn 输出警告级别的日志 +func Warn(c context.Context, args ...interface{}) { + addCaller(GetLogger(c), 2).Warn(args...) +} + +// Warnf 使用格式化字符串输出警告级别的日志 +func Warnf(c context.Context, format string, args ...interface{}) { + addCaller(GetLogger(c), 2).Warnf(format, args...) +} + +// Error 输出错误级别的日志 +func Error(c context.Context, args ...interface{}) { + addCaller(GetLogger(c), 2).Error(args...) +} + +// Errorf 使用格式化字符串输出错误级别的日志 +func Errorf(c context.Context, format string, args ...interface{}) { + addCaller(GetLogger(c), 2).Errorf(format, args...) +} + +// ErrorWithFields 输出带有额外字段的错误级别日志 +func ErrorWithFields(c context.Context, err error, fields logrus.Fields) { + if fields == nil { + fields = logrus.Fields{} + } + if err != nil { + fields["error"] = err.Error() + } + addCaller(GetLogger(c), 2).WithFields(fields).Error("发生错误") +} + +// Fatal 输出致命级别的日志并退出程序 +func Fatal(c context.Context, args ...interface{}) { + addCaller(GetLogger(c), 2).Fatal(args...) +} + +// Fatalf 使用格式化字符串输出致命级别的日志并退出程序 +func Fatalf(c context.Context, format string, args ...interface{}) { + addCaller(GetLogger(c), 2).Fatalf(format, args...) +} + +// CloneContext 复制上下文中的关键信息到新上下文 +func CloneContext(ctx context.Context) context.Context { + newCtx := context.Background() + + for _, k := range []types.ContextKey{ + types.LoggerContextKey, + types.TenantIDContextKey, + types.RequestIDContextKey, + types.TenantInfoContextKey, + } { + if v := ctx.Value(k); v != nil { + newCtx = context.WithValue(newCtx, k, v) + } + } + + return newCtx +} diff --git a/internal/middleware/auth.go b/internal/middleware/auth.go new file mode 100644 index 0000000..334162e --- /dev/null +++ b/internal/middleware/auth.go @@ -0,0 +1,149 @@ +package middleware + +import ( + "context" + "errors" + "log" + "net/http" + "slices" + "strings" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// 无需认证的API列表 +var noAuthAPI = map[string][]string{ + "/health": {"GET"}, + "/api/v1/auth/register": {"POST"}, + "/api/v1/auth/login": {"POST"}, + "/api/v1/auth/refresh": {"POST"}, +} + +// 检查请求是否在无需认证的API列表中 +func isNoAuthAPI(path string, method string) bool { + for api, methods := range noAuthAPI { + // 如果以*结尾，按照前缀匹配，否则按照全路径匹配 + if strings.HasSuffix(api, "*") { + if strings.HasPrefix(path, strings.TrimSuffix(api, "*")) && slices.Contains(methods, method) { + return true + } + } else if path == api && slices.Contains(methods, method) { + return true + } + } + return false +} + +// Auth 认证中间件 +func Auth(tenantService interfaces.TenantService, userService interfaces.UserService, cfg *config.Config) gin.HandlerFunc { + return func(c *gin.Context) { + // ignore OPTIONS request + if c.Request.Method == "OPTIONS" { + c.Next() + return + } + + // 检查请求是否在无需认证的API列表中 + if isNoAuthAPI(c.Request.URL.Path, c.Request.Method) { + c.Next() + return + } + + // 尝试JWT Token认证 + authHeader := c.GetHeader("Authorization") + if authHeader != "" && strings.HasPrefix(authHeader, "Bearer ") { + token := strings.TrimPrefix(authHeader, "Bearer ") + user, err := userService.ValidateToken(c.Request.Context(), token) + if err == nil && user != nil { + // JWT Token认证成功 + // 获取租户信息 + tenant, err := tenantService.GetTenantByID(c.Request.Context(), user.TenantID) + if err != nil { + log.Printf("Error getting tenant by ID: %v, tenantID: %d, userID: %s", err, user.TenantID, user.ID) + c.JSON(http.StatusUnauthorized, gin.H{ + "error": "Unauthorized: invalid tenant", + }) + c.Abort() + return + } + + // 存储用户和租户信息到上下文 + c.Set(types.TenantIDContextKey.String(), user.TenantID) + c.Set(types.TenantInfoContextKey.String(), tenant) + c.Set("user", user) + c.Request = c.Request.WithContext( + context.WithValue( + context.WithValue( + context.WithValue(c.Request.Context(), types.TenantIDContextKey, user.TenantID), + types.TenantInfoContextKey, tenant, + ), + "user", user, + ), + ) + c.Next() + return + } + } + + // 尝试X-API-Key认证（兼容模式） + apiKey := c.GetHeader("X-API-Key") + if apiKey != "" { + // Get tenant information + tenantID, err := tenantService.ExtractTenantIDFromAPIKey(apiKey) + if err != nil { + c.JSON(http.StatusUnauthorized, gin.H{ + "error": "Unauthorized: invalid API key format", + }) + c.Abort() + return + } + + // Verify API key validity (matches the one in database) + t, err := tenantService.GetTenantByID(c.Request.Context(), tenantID) + if err != nil { + log.Printf("Error getting tenant by ID: %v, tenantID: %d, apiKey: %s", err, tenantID, apiKey) + c.JSON(http.StatusUnauthorized, gin.H{ + "error": "Unauthorized: invalid API key", + }) + c.Abort() + return + } + + if t == nil || t.APIKey != apiKey { + c.JSON(http.StatusUnauthorized, gin.H{ + "error": "Unauthorized: invalid API key", + }) + c.Abort() + return + } + + // Store tenant ID in context + c.Set(types.TenantIDContextKey.String(), tenantID) + c.Set(types.TenantInfoContextKey.String(), t) + c.Request = c.Request.WithContext( + context.WithValue( + context.WithValue(c.Request.Context(), types.TenantIDContextKey, tenantID), + types.TenantInfoContextKey, t, + ), + ) + c.Next() + return + } + + // 没有提供任何认证信息 + c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized: missing authentication"}) + c.Abort() + } +} + +// GetTenantIDFromContext helper function to get tenant ID from context +func GetTenantIDFromContext(ctx context.Context) (uint, error) { + tenantID, ok := ctx.Value("tenantID").(uint) + if !ok { + return 0, errors.New("tenant ID not found in context") + } + return tenantID, nil +} diff --git a/internal/middleware/error_handler.go b/internal/middleware/error_handler.go new file mode 100644 index 0000000..2379e6a --- /dev/null +++ b/internal/middleware/error_handler.go @@ -0,0 +1,46 @@ +package middleware + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/Tencent/WeKnora/internal/errors" +) + +// ErrorHandler 是一个处理应用错误的中间件 +func ErrorHandler() gin.HandlerFunc { + return func(c *gin.Context) { + // 处理请求 + c.Next() + + // 检查是否有错误 + if len(c.Errors) > 0 { + // 获取最后一个错误 + err := c.Errors.Last().Err + + // 检查是否为应用错误 + if appErr, ok := errors.IsAppError(err); ok { + // 返回应用错误 + c.JSON(appErr.HTTPCode, gin.H{ + "success": false, + "error": gin.H{ + "code": appErr.Code, + "message": appErr.Message, + "details": appErr.Details, + }, + }) + return + } + + // 处理其他类型的错误 + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "error": gin.H{ + "code": errors.ErrInternalServer, + "message": "Internal server error", + }, + }) + } + } +} diff --git a/internal/middleware/logger.go b/internal/middleware/logger.go new file mode 100644 index 0000000..346b91e --- /dev/null +++ b/internal/middleware/logger.go @@ -0,0 +1,83 @@ +package middleware + +import ( + "context" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +// RequestID middleware adds a unique request ID to the context +func RequestID() gin.HandlerFunc { + return func(c *gin.Context) { + // Get request ID from header or generate a new one + requestID := c.GetHeader("X-Request-ID") + if requestID == "" { + requestID = uuid.New().String() + } + // Set request ID in header + c.Header("X-Request-ID", requestID) + + // Set request ID in context + c.Set(types.RequestIDContextKey.String(), requestID) + + // Set logger in context + requestLogger := logger.GetLogger(c) + requestLogger = requestLogger.WithField("request_id", requestID) + c.Set(types.LoggerContextKey.String(), requestLogger) + + // Set request ID in the global context for logging + c.Request = c.Request.WithContext( + context.WithValue( + context.WithValue(c.Request.Context(), types.RequestIDContextKey, requestID), + types.LoggerContextKey, requestLogger, + ), + ) + + c.Next() + } +} + +// Logger middleware logs request details with request ID +func Logger() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + path := c.Request.URL.Path + raw := c.Request.URL.RawQuery + + // Process request + c.Next() + + // Get request ID from context + requestID, exists := c.Get(types.RequestIDContextKey.String()) + if !exists { + requestID = "unknown" + } + + // Calculate latency + latency := time.Since(start) + + // Get client IP and status code + clientIP := c.ClientIP() + statusCode := c.Writer.Status() + method := c.Request.Method + + if raw != "" { + path = path + "?" + raw + } + + // Log with request ID + logger.GetLogger(c).Infof("[%s] %d | %3d | %13v | %15s | %s %s", + requestID, + statusCode, + c.Writer.Size(), + latency, + clientIP, + method, + path, + ) + } +} diff --git a/internal/middleware/recovery.go b/internal/middleware/recovery.go new file mode 100644 index 0000000..9b60f13 --- /dev/null +++ b/internal/middleware/recovery.go @@ -0,0 +1,34 @@ +package middleware + +import ( + "fmt" + "log" + "runtime/debug" + + "github.com/gin-gonic/gin" +) + +// Recovery is a middleware that recovers from panics +func Recovery() gin.HandlerFunc { + return func(c *gin.Context) { + defer func() { + if err := recover(); err != nil { + // Get request ID + requestID, _ := c.Get("RequestID") + + // Print stacktrace + stacktrace := debug.Stack() + // Log error + log.Printf("[PANIC] %s | %v | %s", requestID, err, stacktrace) + + // 返回500错误 + c.AbortWithStatusJSON(500, gin.H{ + "error": "Internal Server Error", + "message": fmt.Sprintf("%v", err), + }) + } + }() + + c.Next() + } +} diff --git a/internal/middleware/trace.go b/internal/middleware/trace.go new file mode 100644 index 0000000..e7febfc --- /dev/null +++ b/internal/middleware/trace.go @@ -0,0 +1,124 @@ +package middleware + +import ( + "bytes" + "fmt" + "io" + "strings" + + "github.com/gin-gonic/gin" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + + "github.com/Tencent/WeKnora/internal/tracing" + "github.com/Tencent/WeKnora/internal/types" +) + +// Custom ResponseWriter to capture response content +type responseBodyWriter struct { + gin.ResponseWriter + body *bytes.Buffer +} + +// Override Write method to write response content to buffer and original writer +func (r responseBodyWriter) Write(b []byte) (int, error) { + r.body.Write(b) + return r.ResponseWriter.Write(b) +} + +// TracingMiddleware provides a Gin middleware that creates a trace span for each request +func TracingMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + // Extract trace context from request headers + propagator := tracing.GetTracer() + if propagator == nil { + c.Next() + return + } + + // Get request ID as Span ID + requestID := c.GetString(string(types.RequestIDContextKey)) + if requestID == "" { + requestID = c.GetHeader("X-Request-ID") + } + + // Create new span + spanName := fmt.Sprintf("%s %s", c.Request.Method, c.FullPath()) + ctx, span := tracing.ContextWithSpan(c.Request.Context(), spanName) + defer span.End() + + // Set basic span attributes + span.SetAttributes( + attribute.String("http.method", c.Request.Method), + attribute.String("http.url", c.Request.URL.String()), + attribute.String("http.path", c.FullPath()), + ) + + // Record request headers (optional, or selectively record important headers) + for key, values := range c.Request.Header { + // Skip sensitive or unnecessary headers + if strings.ToLower(key) == "authorization" || strings.ToLower(key) == "cookie" { + continue + } + span.SetAttributes(attribute.String("http.request.header."+key, strings.Join(values, ";"))) + } + + // Record request body (for POST/PUT/PATCH requests) + if c.Request.Method == "POST" || c.Request.Method == "PUT" || c.Request.Method == "PATCH" { + if c.Request.Body != nil { + bodyBytes, _ := io.ReadAll(c.Request.Body) + span.SetAttributes(attribute.String("http.request.body", string(bodyBytes))) + // Reset request body because ReadAll consumes the Reader content + c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) + } + } + + // Record query parameters + if len(c.Request.URL.RawQuery) > 0 { + span.SetAttributes(attribute.String("http.request.query", c.Request.URL.RawQuery)) + } + + // Set request context with span context + c.Request = c.Request.WithContext(ctx) + + // Store tracing context in Gin context + c.Set("trace.span", span) + c.Set("trace.ctx", ctx) + + // Create response body capturer + responseBody := &bytes.Buffer{} + responseWriter := &responseBodyWriter{ + ResponseWriter: c.Writer, + body: responseBody, + } + c.Writer = responseWriter + + // Process request + c.Next() + + // Set response status code + statusCode := c.Writer.Status() + span.SetAttributes(attribute.Int("http.status_code", statusCode)) + + // Record response body + responseContent := responseBody.String() + if len(responseContent) > 0 { + span.SetAttributes(attribute.String("http.response.body", responseContent)) + } + + // Record response headers (optional, or selectively record important headers) + for key, values := range c.Writer.Header() { + span.SetAttributes(attribute.String("http.response.header."+key, strings.Join(values, ";"))) + } + + // Mark as error if status code >= 400 + if statusCode >= 400 { + span.SetStatus(codes.Error, fmt.Sprintf("HTTP %d", statusCode)) + if err := c.Errors.Last(); err != nil { + span.RecordError(err.Err) + } + } else { + span.SetStatus(codes.Ok, "") + } + } +} diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go new file mode 100644 index 0000000..04d300d --- /dev/null +++ b/internal/models/chat/chat.go @@ -0,0 +1,72 @@ +package chat + +import ( + "context" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/models/utils/ollama" + "github.com/Tencent/WeKnora/internal/runtime" + "github.com/Tencent/WeKnora/internal/types" +) + +// ChatOptions 聊天选项 +type ChatOptions struct { + Temperature float64 `json:"temperature"` // 温度参数 + TopP float64 `json:"top_p"` // Top P 参数 + Seed int `json:"seed"` // 随机种子 + MaxTokens int `json:"max_tokens"` // 最大 token 数 + MaxCompletionTokens int `json:"max_completion_tokens"` // 最大完成 token 数 + FrequencyPenalty float64 `json:"frequency_penalty"` // 频率惩罚 + PresencePenalty float64 `json:"presence_penalty"` // 存在惩罚 + Thinking *bool `json:"thinking"` // 是否启用思考 +} + +// Message 表示聊天消息 +type Message struct { + Role string `json:"role"` // 角色：system, user, assistant + Content string `json:"content"` // 消息内容 +} + +// Chat 定义了聊天接口 +type Chat interface { + // Chat 进行非流式聊天 + Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) + + // ChatStream 进行流式聊天 + ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) + + // GetModelName 获取模型名称 + GetModelName() string + + // GetModelID 获取模型ID + GetModelID() string +} + +type ChatConfig struct { + Source types.ModelSource + BaseURL string + ModelName string + APIKey string + ModelID string +} + +// NewChat 创建聊天实例 +func NewChat(config *ChatConfig) (Chat, error) { + var chat Chat + var err error + switch strings.ToLower(string(config.Source)) { + case string(types.ModelSourceLocal): + runtime.GetContainer().Invoke(func(ollamaService *ollama.OllamaService) { + chat, err = NewOllamaChat(config, ollamaService) + }) + if err != nil { + return nil, err + } + return chat, nil + case string(types.ModelSourceRemote): + return NewRemoteAPIChat(config) + default: + return nil, fmt.Errorf("unsupported chat model source: %s", config.Source) + } +} diff --git a/internal/models/chat/ollama.go b/internal/models/chat/ollama.go new file mode 100644 index 0000000..4371f29 --- /dev/null +++ b/internal/models/chat/ollama.go @@ -0,0 +1,191 @@ +package chat + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/utils/ollama" + "github.com/Tencent/WeKnora/internal/types" + ollamaapi "github.com/ollama/ollama/api" +) + +// OllamaChat 实现了基于 Ollama 的聊天 +type OllamaChat struct { + modelName string + modelID string + ollamaService *ollama.OllamaService +} + +// NewOllamaChat 创建 Ollama 聊天实例 +func NewOllamaChat(config *ChatConfig, ollamaService *ollama.OllamaService) (*OllamaChat, error) { + return &OllamaChat{ + modelName: config.ModelName, + modelID: config.ModelID, + ollamaService: ollamaService, + }, nil +} + +// convertMessages 转换消息格式为Ollama API格式 +func (c *OllamaChat) convertMessages(messages []Message) []ollamaapi.Message { + ollamaMessages := make([]ollamaapi.Message, len(messages)) + for i, msg := range messages { + ollamaMessages[i] = ollamaapi.Message{ + Role: msg.Role, + Content: msg.Content, + } + } + return ollamaMessages +} + +// buildChatRequest 构建聊天请求参数 +func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isStream bool) *ollamaapi.ChatRequest { + // 设置流式标志 + streamFlag := isStream + + // 构建请求参数 + chatReq := &ollamaapi.ChatRequest{ + Model: c.modelName, + Messages: c.convertMessages(messages), + Stream: &streamFlag, + Options: make(map[string]interface{}), + } + + // 添加可选参数 + if opts != nil { + if opts.Temperature > 0 { + chatReq.Options["temperature"] = opts.Temperature + } + if opts.TopP > 0 { + chatReq.Options["top_p"] = opts.TopP + } + if opts.MaxTokens > 0 { + chatReq.Options["num_predict"] = opts.MaxTokens + } + if opts.Thinking != nil { + chatReq.Think = &ollamaapi.ThinkValue{ + Value: *opts.Thinking, + } + } + } + + return chatReq +} + +// Chat 进行非流式聊天 +func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) { + // 确保模型可用 + if err := c.ensureModelAvailable(ctx); err != nil { + return nil, err + } + + // 构建请求参数 + chatReq := c.buildChatRequest(messages, opts, false) + + // 记录请求日志 + logger.GetLogger(ctx).Infof("发送聊天请求到模型 %s", c.modelName) + + var responseContent string + var promptTokens, completionTokens int + + // 使用 Ollama 客户端发送请求 + err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error { + responseContent = resp.Message.Content + + // 获取token计数 + if resp.EvalCount > 0 { + promptTokens = resp.PromptEvalCount + completionTokens = resp.EvalCount - promptTokens + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("聊天请求失败: %w", err) + } + + // 构建响应 + return &types.ChatResponse{ + Content: responseContent, + Usage: struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + }{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + }, + }, nil +} + +// ChatStream 进行流式聊天 +func (c *OllamaChat) ChatStream( + ctx context.Context, + messages []Message, + opts *ChatOptions, +) (<-chan types.StreamResponse, error) { + // 确保模型可用 + if err := c.ensureModelAvailable(ctx); err != nil { + return nil, err + } + + // 构建请求参数 + chatReq := c.buildChatRequest(messages, opts, true) + + // 记录请求日志 + logger.GetLogger(ctx).Infof("发送流式聊天请求到模型 %s", c.modelName) + + // 创建流式响应通道 + streamChan := make(chan types.StreamResponse) + + // 启动goroutine处理流式响应 + go func() { + defer close(streamChan) + + err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error { + if resp.Message.Content != "" { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: resp.Message.Content, + Done: false, + } + } + + if resp.Done { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Done: true, + } + } + + return nil + }) + if err != nil { + logger.GetLogger(ctx).Errorf("流式聊天请求失败: %v", err) + // 发送错误响应 + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Done: true, + } + } + }() + + return streamChan, nil +} + +// 确保模型可用 +func (c *OllamaChat) ensureModelAvailable(ctx context.Context) error { + logger.GetLogger(ctx).Infof("确保模型 %s 可用", c.modelName) + return c.ollamaService.EnsureModelAvailable(ctx, c.modelName) +} + +// GetModelName 获取模型名称 +func (c *OllamaChat) GetModelName() string { + return c.modelName +} + +// GetModelID 获取模型ID +func (c *OllamaChat) GetModelID() string { + return c.modelID +} diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go new file mode 100644 index 0000000..283dc14 --- /dev/null +++ b/internal/models/chat/remote_api.go @@ -0,0 +1,271 @@ +package chat + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/sashabaranov/go-openai" +) + +// RemoteAPIChat 实现了基于的聊天 +type RemoteAPIChat struct { + modelName string + client *openai.Client + modelID string + baseURL string + apiKey string +} + +// QwenChatCompletionRequest 用于 qwen 模型的自定义请求结构体 +type QwenChatCompletionRequest struct { + openai.ChatCompletionRequest + EnableThinking *bool `json:"enable_thinking,omitempty"` // qwen 模型专用字段 +} + +// NewRemoteAPIChat 调用远程API 聊天实例 +func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { + apiKey := chatConfig.APIKey + config := openai.DefaultConfig(apiKey) + if baseURL := chatConfig.BaseURL; baseURL != "" { + config.BaseURL = baseURL + } + return &RemoteAPIChat{ + modelName: chatConfig.ModelName, + client: openai.NewClientWithConfig(config), + modelID: chatConfig.ModelID, + baseURL: chatConfig.BaseURL, + apiKey: apiKey, + }, nil +} + +// convertMessages 转换消息格式为OpenAI格式 +func (c *RemoteAPIChat) convertMessages(messages []Message) []openai.ChatCompletionMessage { + openaiMessages := make([]openai.ChatCompletionMessage, len(messages)) + for i, msg := range messages { + openaiMessages[i] = openai.ChatCompletionMessage{ + Role: msg.Role, + Content: msg.Content, + } + } + return openaiMessages +} + +// isQwenModel 检查是否为 qwen 模型 +func (c *RemoteAPIChat) isAliyunQwen3Model() bool { + return strings.HasPrefix(c.modelName, "qwen3-") && c.baseURL == "https://dashscope.aliyuncs.com/compatible-mode/v1" +} + +// buildQwenChatCompletionRequest 构建 qwen 模型的聊天请求参数 +func (c *RemoteAPIChat) buildQwenChatCompletionRequest(messages []Message, + opts *ChatOptions, isStream bool, +) QwenChatCompletionRequest { + req := QwenChatCompletionRequest{ + ChatCompletionRequest: c.buildChatCompletionRequest(messages, opts, isStream), + } + + // 对于 qwen 模型，在非流式调用中强制设置 enable_thinking: false + if !isStream { + enableThinking := false + req.EnableThinking = &enableThinking + } + return req +} + +// buildChatCompletionRequest 构建聊天请求参数 +func (c *RemoteAPIChat) buildChatCompletionRequest(messages []Message, + opts *ChatOptions, isStream bool, +) openai.ChatCompletionRequest { + req := openai.ChatCompletionRequest{ + Model: c.modelName, + Messages: c.convertMessages(messages), + Stream: isStream, + } + thinking := false + + // 添加可选参数 + if opts != nil { + if opts.Temperature > 0 { + req.Temperature = float32(opts.Temperature) + } + if opts.TopP > 0 { + req.TopP = float32(opts.TopP) + } + if opts.MaxTokens > 0 { + req.MaxTokens = opts.MaxTokens + } + if opts.MaxCompletionTokens > 0 { + req.MaxCompletionTokens = opts.MaxCompletionTokens + } + if opts.FrequencyPenalty > 0 { + req.FrequencyPenalty = float32(opts.FrequencyPenalty) + } + if opts.PresencePenalty > 0 { + req.PresencePenalty = float32(opts.PresencePenalty) + } + if opts.Thinking != nil { + thinking = *opts.Thinking + } + } + + req.ChatTemplateKwargs = map[string]interface{}{ + "enable_thinking": thinking, + } + + return req +} + +// Chat 进行非流式聊天 +func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) { + // 如果是 qwen 模型，使用自定义请求 + if c.isAliyunQwen3Model() { + return c.chatWithQwen(ctx, messages, opts) + } + + // 构建请求参数 + req := c.buildChatCompletionRequest(messages, opts, false) + + // 发送请求 + resp, err := c.client.CreateChatCompletion(ctx, req) + if err != nil { + return nil, fmt.Errorf("create chat completion: %w", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("no response from OpenAI") + } + + // 转换响应格式 + return &types.ChatResponse{ + Content: resp.Choices[0].Message.Content, + Usage: struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + }{ + PromptTokens: resp.Usage.PromptTokens, + CompletionTokens: resp.Usage.CompletionTokens, + TotalTokens: resp.Usage.TotalTokens, + }, + }, nil +} + +// chatWithQwen 使用自定义请求处理 qwen 模型 +func (c *RemoteAPIChat) chatWithQwen(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) { + // 构建 qwen 请求参数 + req := c.buildQwenChatCompletionRequest(messages, opts, false) + + // 序列化请求 + jsonData, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + // 构建 URL + endpoint := c.baseURL + "/chat/completions" + + // 创建 HTTP 请求 + httpReq, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + // 设置请求头 + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) + + // 发送请求 + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("send request: %w", err) + } + defer resp.Body.Close() + + // 检查响应状态 + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("API request failed with status: %d", resp.StatusCode) + } + + // 解析响应 + var chatResp openai.ChatCompletionResponse + if err := json.NewDecoder(resp.Body).Decode(&chatResp); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + + if len(chatResp.Choices) == 0 { + return nil, fmt.Errorf("no response from API") + } + + // 转换响应格式 + return &types.ChatResponse{ + Content: chatResp.Choices[0].Message.Content, + Usage: struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + }{ + PromptTokens: chatResp.Usage.PromptTokens, + CompletionTokens: chatResp.Usage.CompletionTokens, + TotalTokens: chatResp.Usage.TotalTokens, + }, + }, nil +} + +// ChatStream 进行流式聊天 +func (c *RemoteAPIChat) ChatStream(ctx context.Context, + messages []Message, opts *ChatOptions, +) (<-chan types.StreamResponse, error) { + // 构建请求参数 + req := c.buildChatCompletionRequest(messages, opts, true) + + // 创建流式响应通道 + streamChan := make(chan types.StreamResponse) + + // 启动流式请求 + stream, err := c.client.CreateChatCompletionStream(ctx, req) + if err != nil { + close(streamChan) + return nil, fmt.Errorf("create chat completion stream: %w", err) + } + + // 在后台处理流式响应 + go func() { + defer close(streamChan) + defer stream.Close() + + for { + response, err := stream.Recv() + if err != nil { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Done: true, + } + return + } + if len(response.Choices) > 0 { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Content: response.Choices[0].Delta.Content, + Done: false, + } + } + } + }() + + return streamChan, nil +} + +// GetModelName 获取模型名称 +func (c *RemoteAPIChat) GetModelName() string { + return c.modelName +} + +// GetModelID 获取模型ID +func (c *RemoteAPIChat) GetModelID() string { + return c.modelID +} diff --git a/internal/models/chat/remote_api_test.go b/internal/models/chat/remote_api_test.go new file mode 100644 index 0000000..115ad23 --- /dev/null +++ b/internal/models/chat/remote_api_test.go @@ -0,0 +1,127 @@ +package chat + +import ( + "context" + "os" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRemoteAPIChat 综合测试 Remote API Chat 的所有功能 +func TestRemoteAPIChat(t *testing.T) { + // 获取环境变量 + deepseekAPIKey := os.Getenv("DEEPSEEK_API_KEY") + aliyunAPIKey := os.Getenv("ALIYUN_API_KEY") + + // 定义测试配置 + testConfigs := []struct { + name string + apiKey string + config *ChatConfig + skipMsg string + }{ + { + name: "DeepSeek API", + apiKey: deepseekAPIKey, + config: &ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: "https://api.deepseek.com/v1", + ModelName: "deepseek-chat", + APIKey: deepseekAPIKey, + ModelID: "deepseek-chat", + }, + skipMsg: "DEEPSEEK_API_KEY environment variable not set", + }, + { + name: "Aliyun DeepSeek", + apiKey: aliyunAPIKey, + config: &ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", + ModelName: "deepseek-v3.1", + APIKey: aliyunAPIKey, + ModelID: "deepseek-v3.1", + }, + skipMsg: "ALIYUN_API_KEY environment variable not set", + }, + { + name: "Aliyun Qwen3-32b", + apiKey: aliyunAPIKey, + config: &ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", + ModelName: "qwen3-32b", + APIKey: aliyunAPIKey, + ModelID: "qwen3-32b", + }, + skipMsg: "ALIYUN_API_KEY environment variable not set", + }, + { + name: "Aliyun Qwen-max", + apiKey: aliyunAPIKey, + config: &ChatConfig{ + Source: types.ModelSourceRemote, + BaseURL: "https://dashscope.aliyuncs.com/compatible-mode/v1", + ModelName: "qwen-max", + APIKey: aliyunAPIKey, + ModelID: "qwen-max", + }, + skipMsg: "ALIYUN_API_KEY environment variable not set", + }, + } + + // 测试消息 + testMessages := []Message{ + { + Role: "user", + Content: "test", + }, + } + + // 测试选项 + testOptions := &ChatOptions{ + Temperature: 0.7, + MaxTokens: 100, + } + + // 创建上下文 + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // 遍历所有配置进行测试 + for _, tc := range testConfigs { + t.Run(tc.name, func(t *testing.T) { + // 检查 API Key + if tc.apiKey == "" { + t.Skip(tc.skipMsg) + } + + // 创建聊天实例 + chat, err := NewRemoteAPIChat(tc.config) + require.NoError(t, err) + assert.Equal(t, tc.config.ModelName, chat.GetModelName()) + assert.Equal(t, tc.config.ModelID, chat.GetModelID()) + + // 测试基本聊天功能 + t.Run("Basic Chat", func(t *testing.T) { + response, err := chat.Chat(ctx, testMessages, testOptions) + require.NoError(t, err) + assert.NotEmpty(t, response.Content) + assert.Greater(t, response.Usage.TotalTokens, 0) + assert.Greater(t, response.Usage.PromptTokens, 0) + assert.Greater(t, response.Usage.CompletionTokens, 0) + + t.Logf("%s Response: %s", tc.name, response.Content) + t.Logf("Usage: Prompt=%d, Completion=%d, Total=%d", + response.Usage.PromptTokens, + response.Usage.CompletionTokens, + response.Usage.TotalTokens) + }) + + }) + } +} diff --git a/internal/models/embedding/batch.go b/internal/models/embedding/batch.go new file mode 100644 index 0000000..2849173 --- /dev/null +++ b/internal/models/embedding/batch.go @@ -0,0 +1,83 @@ +package embedding + +import ( + "context" + "sync" + + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/panjf2000/ants/v2" +) + +type batchEmbedder struct { + pool *ants.Pool +} + +func NewBatchEmbedder(pool *ants.Pool) EmbedderPooler { + return &batchEmbedder{pool: pool} +} + +type textEmbedding struct { + text string + results []float32 +} + +func (e *batchEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error) { + // Create goroutine pool for concurrent processing of document chunks + var wg sync.WaitGroup + var mu sync.Mutex // For synchronizing access to error + var firstErr error // Record the first error that occurs + batchSize := 5 + textEmbeddings := utils.MapSlice(texts, func(text string) *textEmbedding { + return &textEmbedding{text: text} + }) + + // Function to process each document chunk + processChunk := func(texts []*textEmbedding) func() { + return func() { + defer wg.Done() + // If an error has already occurred, don't continue processing + if firstErr != nil { + return + } + // Embed text + embedding, err := model.BatchEmbed(ctx, utils.MapSlice(texts, func(text *textEmbedding) string { + return text.text + })) + if err != nil { + mu.Lock() + if firstErr == nil { + firstErr = err + } + mu.Unlock() + return + } + mu.Lock() + for i, text := range texts { + text.results = embedding[i] + } + mu.Unlock() + } + } + + // Submit all tasks to the goroutine pool + for _, texts := range utils.ChunkSlice(textEmbeddings, batchSize) { + wg.Add(1) + err := e.pool.Submit(processChunk(texts)) + if err != nil { + return nil, err + } + } + + // Wait for all tasks to complete + wg.Wait() + + // Check if any errors occurred + if firstErr != nil { + return nil, firstErr + } + + results := utils.MapSlice(textEmbeddings, func(text *textEmbedding) []float32 { + return text.results + }) + return results, nil +} diff --git a/internal/models/embedding/embedder.go b/internal/models/embedding/embedder.go new file mode 100644 index 0000000..ece81c0 --- /dev/null +++ b/internal/models/embedding/embedder.go @@ -0,0 +1,76 @@ +package embedding + +import ( + "context" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/models/utils/ollama" + "github.com/Tencent/WeKnora/internal/runtime" + "github.com/Tencent/WeKnora/internal/types" +) + +// Embedder defines the interface for text vectorization +type Embedder interface { + // Embed converts text to vector + Embed(ctx context.Context, text string) ([]float32, error) + + // BatchEmbed converts multiple texts to vectors in batch + BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) + + // GetModelName returns the model name + GetModelName() string + + // GetDimensions returns the vector dimensions + GetDimensions() int + + // GetModelID returns the model ID + GetModelID() string + + EmbedderPooler +} + +type EmbedderPooler interface { + BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error) +} + +// EmbedderType represents the embedder type +type EmbedderType string + +// Config represents the embedder configuration +type Config struct { + Source types.ModelSource `json:"source"` + BaseURL string `json:"base_url"` + ModelName string `json:"model_name"` + APIKey string `json:"api_key"` + TruncatePromptTokens int `json:"truncate_prompt_tokens"` + Dimensions int `json:"dimensions"` + ModelID string `json:"model_id"` +} + +// NewEmbedder creates an embedder based on the configuration +func NewEmbedder(config Config) (Embedder, error) { + var embedder Embedder + var err error + switch strings.ToLower(string(config.Source)) { + case string(types.ModelSourceLocal): + runtime.GetContainer().Invoke(func(pooler EmbedderPooler, ollamaService *ollama.OllamaService) { + embedder, err = NewOllamaEmbedder(config.BaseURL, + config.ModelName, config.TruncatePromptTokens, config.Dimensions, config.ModelID, pooler, ollamaService) + }) + return embedder, err + case string(types.ModelSourceRemote): + runtime.GetContainer().Invoke(func(pooler EmbedderPooler) { + embedder, err = NewOpenAIEmbedder(config.APIKey, + config.BaseURL, + config.ModelName, + config.TruncatePromptTokens, + config.Dimensions, + config.ModelID, + pooler) + }) + return embedder, err + default: + return nil, fmt.Errorf("unsupported embedder source: %s", config.Source) + } +} diff --git a/internal/models/embedding/ollama.go b/internal/models/embedding/ollama.go new file mode 100644 index 0000000..12e5196 --- /dev/null +++ b/internal/models/embedding/ollama.go @@ -0,0 +1,125 @@ +package embedding + +import ( + "context" + "fmt" + "time" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/utils/ollama" + ollamaapi "github.com/ollama/ollama/api" +) + +// OllamaEmbedder implements text vectorization functionality using Ollama +type OllamaEmbedder struct { + modelName string + truncatePromptTokens int + ollamaService *ollama.OllamaService + dimensions int + modelID string + EmbedderPooler +} + +// OllamaEmbedRequest represents an Ollama embedding request +type OllamaEmbedRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + TruncatePromptTokens int `json:"truncate_prompt_tokens"` +} + +// OllamaEmbedResponse represents an Ollama embedding response +type OllamaEmbedResponse struct { + Embedding []float32 `json:"embedding"` +} + +// NewOllamaEmbedder creates a new Ollama embedder +func NewOllamaEmbedder(baseURL, + modelName string, + truncatePromptTokens int, + dimensions int, + modelID string, + pooler EmbedderPooler, + ollamaService *ollama.OllamaService, +) (*OllamaEmbedder, error) { + if modelName == "" { + modelName = "nomic-embed-text" + } + + if truncatePromptTokens == 0 { + truncatePromptTokens = 511 + } + + return &OllamaEmbedder{ + modelName: modelName, + truncatePromptTokens: truncatePromptTokens, + ollamaService: ollamaService, + EmbedderPooler: pooler, + dimensions: dimensions, + modelID: modelID, + }, nil +} + +// ensureModelAvailable ensures that the model is available +func (e *OllamaEmbedder) ensureModelAvailable(ctx context.Context) error { + logger.GetLogger(ctx).Infof("Ensuring model %s is available", e.modelName) + return e.ollamaService.EnsureModelAvailable(ctx, e.modelName) +} + +// Embed converts text to vector +func (e *OllamaEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + embedding, err := e.BatchEmbed(ctx, []string{text}) + if err != nil { + return nil, fmt.Errorf("failed to embed text: %w", err) + } + + if len(embedding) == 0 { + return nil, fmt.Errorf("failed to embed text: %w", err) + } + + return embedding[0], nil +} + +// BatchEmbed converts multiple texts to vectors in batch +func (e *OllamaEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) { + // Ensure model is available + if err := e.ensureModelAvailable(ctx); err != nil { + return nil, err + } + + // Create request + req := &ollamaapi.EmbedRequest{ + Model: e.modelName, + Input: texts, + Options: make(map[string]interface{}), + } + + // Set truncation parameters + if e.truncatePromptTokens > 0 { + req.Options["truncate"] = e.truncatePromptTokens + } + + // Send request + startTime := time.Now() + resp, err := e.ollamaService.Embeddings(ctx, req) + if err != nil { + return nil, fmt.Errorf("failed to get embedding vectors: %w", err) + } + + logger.GetLogger(ctx).Debugf("Embedding vector retrieval took: %v", time.Since(startTime)) + return resp.Embeddings, nil +} + +// GetModelName returns the model name +func (e *OllamaEmbedder) GetModelName() string { + return e.modelName +} + +// GetDimensions returns the vector dimensions +func (e *OllamaEmbedder) GetDimensions() int { + return e.dimensions +} + +// GetModelID returns the model ID +func (e *OllamaEmbedder) GetModelID() string { + return e.modelID +} diff --git a/internal/models/embedding/openai.go b/internal/models/embedding/openai.go new file mode 100644 index 0000000..ef987d4 --- /dev/null +++ b/internal/models/embedding/openai.go @@ -0,0 +1,198 @@ +package embedding + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/Tencent/WeKnora/internal/logger" +) + +// OpenAIEmbedder implements text vectorization functionality using OpenAI API +type OpenAIEmbedder struct { + apiKey string + baseURL string + modelName string + truncatePromptTokens int + dimensions int + modelID string + httpClient *http.Client + timeout time.Duration + maxRetries int + EmbedderPooler +} + +// OpenAIEmbedRequest represents an OpenAI embedding request +type OpenAIEmbedRequest struct { + Model string `json:"model"` + Input []string `json:"input"` + TruncatePromptTokens int `json:"truncate_prompt_tokens"` +} + +// OpenAIEmbedResponse represents an OpenAI embedding response +type OpenAIEmbedResponse struct { + Data []struct { + Embedding []float32 `json:"embedding"` + Index int `json:"index"` + } `json:"data"` +} + +// NewOpenAIEmbedder creates a new OpenAI embedder +func NewOpenAIEmbedder(apiKey, baseURL, modelName string, + truncatePromptTokens int, dimensions int, modelID string, pooler EmbedderPooler, +) (*OpenAIEmbedder, error) { + if baseURL == "" { + baseURL = "https://api.openai.com/v1" + } + + if modelName == "" { + return nil, fmt.Errorf("model name is required") + } + + if truncatePromptTokens == 0 { + truncatePromptTokens = 511 + } + + timeout := 60 * time.Second + + // Create HTTP client + client := &http.Client{ + Timeout: timeout, + } + + return &OpenAIEmbedder{ + apiKey: apiKey, + baseURL: baseURL, + modelName: modelName, + httpClient: client, + truncatePromptTokens: truncatePromptTokens, + EmbedderPooler: pooler, + dimensions: dimensions, + modelID: modelID, + timeout: timeout, + maxRetries: 3, // Maximum retry count + }, nil +} + +// Embed converts text to vector +func (e *OpenAIEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + for range 3 { + embeddings, err := e.BatchEmbed(ctx, []string{text}) + if err != nil { + return nil, err + } + if len(embeddings) > 0 { + return embeddings[0], nil + } + } + return nil, fmt.Errorf("no embedding returned") +} + +func (e *OpenAIEmbedder) doRequestWithRetry(ctx context.Context, jsonData []byte) (*http.Response, error) { + var resp *http.Response + var err error + url := e.baseURL + "/embeddings" + + for i := 0; i <= e.maxRetries; i++ { + if i > 0 { + backoffTime := time.Duration(1< 10*time.Second { + backoffTime = 10 * time.Second + } + logger.GetLogger(ctx).Infof("OpenAIEmbedder retrying request (%d/%d), waiting %v", i, e.maxRetries, backoffTime) + + select { + case <-time.After(backoffTime): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + // Rebuild request each time to ensure Body is valid + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonData)) + if err != nil { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder failed to create request: %v", err) + continue + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+e.apiKey) + + resp, err = e.httpClient.Do(req) + if err == nil { + return resp, nil + } + + logger.GetLogger(ctx).Errorf("OpenAIEmbedder request failed (attempt %d/%d): %v", i+1, e.maxRetries+1, err) + } + + return nil, err +} + +func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) { + // Create request body + reqBody := OpenAIEmbedRequest{ + Model: e.modelName, + Input: texts, + TruncatePromptTokens: e.truncatePromptTokens, + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch marshal request error: %v", err) + return nil, fmt.Errorf("marshal request: %w", err) + } + + // Send request (passing jsonData instead of constructing http.Request) + resp, err := e.doRequestWithRetry(ctx, jsonData) + if err != nil { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch send request error: %v", err) + return nil, fmt.Errorf("send request: %w", err) + } + defer resp.Body.Close() + + // Read response + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch read response error: %v", err) + return nil, fmt.Errorf("read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch API error: Http Status %s", resp.Status) + return nil, fmt.Errorf("EmbedBatch API error: Http Status %s", resp.Status) + } + + // Parse response + var response OpenAIEmbedResponse + if err := json.Unmarshal(body, &response); err != nil { + logger.GetLogger(ctx).Errorf("OpenAIEmbedder EmbedBatch unmarshal response error: %v", err) + return nil, fmt.Errorf("unmarshal response: %w", err) + } + + // Extract embedding vectors + embeddings := make([][]float32, 0, len(response.Data)) + for _, data := range response.Data { + embeddings = append(embeddings, data.Embedding) + } + + return embeddings, nil +} + +// GetModelName returns the model name +func (e *OpenAIEmbedder) GetModelName() string { + return e.modelName +} + +// GetDimensions returns the vector dimensions +func (e *OpenAIEmbedder) GetDimensions() int { + return e.dimensions +} + +// GetModelID returns the model ID +func (e *OpenAIEmbedder) GetModelID() string { + return e.modelID +} diff --git a/internal/models/rerank/aliyun_reranker.go b/internal/models/rerank/aliyun_reranker.go new file mode 100644 index 0000000..3643b7f --- /dev/null +++ b/internal/models/rerank/aliyun_reranker.go @@ -0,0 +1,164 @@ +package rerank + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/Tencent/WeKnora/internal/logger" +) + +// AliyunReranker implements a reranking system based on Aliyun DashScope models +type AliyunReranker struct { + modelName string // Name of the model used for reranking + modelID string // Unique identifier of the model + apiKey string // API key for authentication + baseURL string // Base URL for API requests + client *http.Client // HTTP client for making API requests +} + +// AliyunRerankRequest represents a request to rerank documents using Aliyun DashScope API +type AliyunRerankRequest struct { + Model string `json:"model"` // Model to use for reranking + Input AliyunRerankInput `json:"input"` // Input containing query and documents + Parameters AliyunRerankParameters `json:"parameters"` // Parameters for the reranking +} + +// AliyunRerankInput contains the query and documents for reranking +type AliyunRerankInput struct { + Query string `json:"query"` // Query text to compare documents against + Documents []string `json:"documents"` // List of document texts to rerank +} + +// AliyunRerankParameters contains parameters for the reranking request +type AliyunRerankParameters struct { + ReturnDocuments bool `json:"return_documents"` // Whether to return documents in response + TopN int `json:"top_n"` // Number of top results to return +} + +// AliyunRerankResponse represents the response from Aliyun DashScope reranking request +type AliyunRerankResponse struct { + Output AliyunOutput `json:"output"` // Output containing results + Usage AliyunUsage `json:"usage"` // Token usage information +} + +// AliyunOutput contains the reranking results +type AliyunOutput struct { + Results []AliyunRankResult `json:"results"` // Ranked results with relevance scores +} + +// AliyunRankResult represents a single reranking result from Aliyun +type AliyunRankResult struct { + Document AliyunDocument `json:"document"` // Document information + Index int `json:"index"` // Original index of the document + RelevanceScore float64 `json:"relevance_score"` // Relevance score +} + +// AliyunDocument represents document information in Aliyun response +type AliyunDocument struct { + Text string `json:"text"` // Document text +} + +// AliyunUsage contains information about token usage in the Aliyun API request +type AliyunUsage struct { + TotalTokens int `json:"total_tokens"` // Total tokens consumed +} + +// NewAliyunReranker creates a new instance of Aliyun reranker with the provided configuration +func NewAliyunReranker(config *RerankerConfig) (*AliyunReranker, error) { + apiKey := config.APIKey + baseURL := "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank" + if url := config.BaseURL; url != "" { + baseURL = url + } + + return &AliyunReranker{ + modelName: config.ModelName, + modelID: config.ModelID, + apiKey: apiKey, + baseURL: baseURL, + client: &http.Client{}, + }, nil +} + +// Rerank performs document reranking based on relevance to the query using Aliyun DashScope API +func (r *AliyunReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { + // Build the request body + requestBody := &AliyunRerankRequest{ + Model: r.modelName, + Input: AliyunRerankInput{ + Query: query, + Documents: documents, + }, + Parameters: AliyunRerankParameters{ + ReturnDocuments: true, + TopN: len(documents), // Return all documents + }, + } + + jsonData, err := json.Marshal(requestBody) + if err != nil { + return nil, fmt.Errorf("marshal request body: %w", err) + } + + // Send the request + req, err := http.NewRequestWithContext(ctx, "POST", r.baseURL, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) + + // Log the curl equivalent for debugging + logger.Debugf(ctx, "curl -X POST %s -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'", + r.baseURL, r.apiKey, string(jsonData), + ) + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + // Read the response + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("aliyun rerank API error: Http Status: %s, Body: %s", resp.Status, string(body)) + } + + var response AliyunRerankResponse + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("unmarshal response: %w", err) + } + + // Convert Aliyun results to standard RankResult format + results := make([]RankResult, len(response.Output.Results)) + for i, aliyunResult := range response.Output.Results { + results[i] = RankResult{ + Index: aliyunResult.Index, + Document: DocumentInfo{ + Text: aliyunResult.Document.Text, + }, + RelevanceScore: aliyunResult.RelevanceScore, + } + } + + return results, nil +} + +// GetModelName returns the name of the reranking model +func (r *AliyunReranker) GetModelName() string { + return r.modelName +} + +// GetModelID returns the unique identifier of the reranking model +func (r *AliyunReranker) GetModelID() string { + return r.modelID +} diff --git a/internal/models/rerank/remote_api.go b/internal/models/rerank/remote_api.go new file mode 100644 index 0000000..f0ad8ea --- /dev/null +++ b/internal/models/rerank/remote_api.go @@ -0,0 +1,122 @@ +package rerank + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/Tencent/WeKnora/internal/logger" +) + +// OpenAIReranker implements a reranking system based on OpenAI models +type OpenAIReranker struct { + modelName string // Name of the model used for reranking + modelID string // Unique identifier of the model + apiKey string // API key for authentication + baseURL string // Base URL for API requests + client *http.Client // HTTP client for making API requests +} + +// RerankRequest represents a request to rerank documents based on relevance to a query +type RerankRequest struct { + Model string `json:"model"` // Model to use for reranking + Query string `json:"query"` // Query text to compare documents against + Documents []string `json:"documents"` // List of document texts to rerank + AdditionalData map[string]interface{} `json:"additional_data"` // Optional additional data for the model + TruncatePromptTokens int `json:"truncate_prompt_tokens"` // Maximum prompt tokens to use +} + +// RerankResponse represents the response from a reranking request +type RerankResponse struct { + ID string `json:"id"` // Request ID + Model string `json:"model"` // Model used for reranking + Usage UsageInfo `json:"usage"` // Token usage information + Results []RankResult `json:"results"` // Ranked results with relevance scores +} + +// UsageInfo contains information about token usage in the API request +type UsageInfo struct { + TotalTokens int `json:"total_tokens"` // Total tokens consumed +} + +// NewOpenAIReranker creates a new instance of OpenAI reranker with the provided configuration +func NewOpenAIReranker(config *RerankerConfig) (*OpenAIReranker, error) { + apiKey := config.APIKey + baseURL := "https://api.openai.com/v1" + if url := config.BaseURL; url != "" { + baseURL = url + } + + return &OpenAIReranker{ + modelName: config.ModelName, + modelID: config.ModelID, + apiKey: apiKey, + baseURL: baseURL, + client: &http.Client{}, + }, nil +} + +// Rerank performs document reranking based on relevance to the query +func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { + // Build the request body + requestBody := &RerankRequest{ + Model: r.modelName, + Query: query, + Documents: documents, + TruncatePromptTokens: 511, + } + + jsonData, err := json.Marshal(requestBody) + if err != nil { + return nil, fmt.Errorf("marshal request body: %w", err) + } + + // Send the request + req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/rerank", r.baseURL), bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey)) + + // Log the curl equivalent for debugging + logger.GetLogger(ctx).Infof( + "curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'", + r.baseURL, r.apiKey, string(jsonData), + ) + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + // Read the response + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response body: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Rerank API error: Http Status: %s", resp.Status) + } + + var response RerankResponse + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("unmarshal response: %w", err) + } + return response.Results, nil +} + +// GetModelName returns the name of the reranking model +func (r *OpenAIReranker) GetModelName() string { + return r.modelName +} + +// GetModelID returns the unique identifier of the reranking model +func (r *OpenAIReranker) GetModelID() string { + return r.modelID +} diff --git a/internal/models/rerank/reranker.go b/internal/models/rerank/reranker.go new file mode 100644 index 0000000..20a61d4 --- /dev/null +++ b/internal/models/rerank/reranker.go @@ -0,0 +1,97 @@ +package rerank + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/types" +) + +// Reranker defines the interface for document reranking +type Reranker interface { + // Rerank reranks documents based on relevance to the query + Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) + + // GetModelName returns the model name + GetModelName() string + + // GetModelID returns the model ID + GetModelID() string +} + +type RankResult struct { + Index int `json:"index"` + Document DocumentInfo `json:"document"` + RelevanceScore float64 `json:"relevance_score"` +} + +// Handles the RelevanceScore field by checking if RelevanceScore exists first, otherwise falls back to Score field +func (r *RankResult) UnmarshalJSON(data []byte) error { + + var temp struct { + Index int `json:"index"` + Document DocumentInfo `json:"document"` + RelevanceScore *float64 `json:"relevance_score"` + Score *float64 `json:"score"` + } + + if err := json.Unmarshal(data, &temp); err != nil { + return fmt.Errorf("failed to unmarshal rank result: %w", err) + } + + r.Index = temp.Index + r.Document = temp.Document + + if temp.RelevanceScore != nil { + r.RelevanceScore = *temp.RelevanceScore + } else if temp.Score != nil { + r.RelevanceScore = *temp.Score + } + + return nil +} + +type DocumentInfo struct { + Text string `json:"text"` +} + +// UnmarshalJSON handles both string and object formats for DocumentInfo +func (d *DocumentInfo) UnmarshalJSON(data []byte) error { + // First try to unmarshal as a string + var text string + if err := json.Unmarshal(data, &text); err == nil { + d.Text = text + return nil + } + + // If that fails, try to unmarshal as an object with text field + var temp struct { + Text string `json:"text"` + } + if err := json.Unmarshal(data, &temp); err != nil { + return fmt.Errorf("failed to unmarshal DocumentInfo: %w", err) + } + + d.Text = temp.Text + return nil +} + +type RerankerConfig struct { + APIKey string + BaseURL string + ModelName string + Source types.ModelSource + ModelID string +} + +// NewReranker creates a reranker +func NewReranker(config *RerankerConfig) (Reranker, error) { + // 根据URL判断模型来源，而不是依赖Source字段 + if strings.Contains(config.BaseURL, "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank") { + return NewAliyunReranker(config) + } else { + return NewOpenAIReranker(config) + } +} diff --git a/internal/models/rerank/reranker_test.go b/internal/models/rerank/reranker_test.go new file mode 100644 index 0000000..821430e --- /dev/null +++ b/internal/models/rerank/reranker_test.go @@ -0,0 +1,156 @@ +package rerank + +import ( + "encoding/json" + "testing" +) + +func TestRankResultUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + expectedText string + expectedIndex int + expectedScore float64 + expectError bool + }{ + { + name: "document as string with relevance_score", + input: `{"index": 0, "document": "This is a document", "relevance_score": 0.95}`, + expectedText: "This is a document", + expectedIndex: 0, + expectedScore: 0.95, + expectError: false, + }, + { + name: "document as object with relevance_score", + input: `{"index": 1, "document": {"text": "This is a document"}, "relevance_score": 0.87}`, + expectedText: "This is a document", + expectedIndex: 1, + expectedScore: 0.87, + expectError: false, + }, + { + name: "document as string with score field", + input: `{"index": 2, "document": "This is a document", "score": 0.92}`, + expectedText: "This is a document", + expectedIndex: 2, + expectedScore: 0.92, + expectError: false, + }, + { + name: "document as object with score field", + input: `{"index": 3, "document": {"text": "This is a document"}, "score": 0.78}`, + expectedText: "This is a document", + expectedIndex: 3, + expectedScore: 0.78, + expectError: false, + }, + { + name: "document as string with both score fields - relevance_score takes priority", + input: `{"index": 4, "document": "This is a document", "relevance_score": 0.95, "score": 0.80}`, + expectedText: "This is a document", + expectedIndex: 4, + expectedScore: 0.95, + expectError: false, + }, + { + name: "document as object with both score fields - relevance_score takes priority", + input: `{"index": 5, "document": {"text": "This is a document"}, "relevance_score": 0.88, "score": 0.75}`, + expectedText: "This is a document", + expectedIndex: 5, + expectedScore: 0.88, + expectError: false, + }, + { + name: "document as string with no score fields", + input: `{"index": 6, "document": "This is a document"}`, + expectedText: "This is a document", + expectedIndex: 6, + expectedScore: 0.0, + expectError: false, + }, + { + name: "document as object with no score fields", + input: `{"index": 7, "document": {"text": "This is a document"}}`, + expectedText: "This is a document", + expectedIndex: 7, + expectedScore: 0.0, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var result RankResult + err := json.Unmarshal([]byte(tt.input), &result) + + if tt.expectError { + if err == nil { + t.Errorf("Expected error but got none") + } + return + } + + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + if result.Document.Text != tt.expectedText { + t.Errorf("Expected document text %q, got %q", tt.expectedText, result.Document.Text) + } + if result.Index != tt.expectedIndex { + t.Errorf("Expected index %d, got %d", tt.expectedIndex, result.Index) + } + if result.RelevanceScore != tt.expectedScore { + t.Errorf("Expected score %f, got %f", tt.expectedScore, result.RelevanceScore) + } + }) + } +} + +// TestDocumentInfoMarshalJSON tests that DocumentInfo can be marshaled back to JSON +func TestDocumentInfoMarshalJSON(t *testing.T) { + doc := DocumentInfo{Text: "Test document content"} + + data, err := json.Marshal(doc) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + expected := `{"text":"Test document content"}` + if string(data) != expected { + t.Errorf("Expected %s, got %s", expected, string(data)) + } +} + +// TestRankResultMarshalJSON tests that RankResult can be marshaled back to JSON +func TestRankResultMarshalJSON(t *testing.T) { + result := RankResult{ + Index: 1, + Document: DocumentInfo{Text: "Test document"}, + RelevanceScore: 0.95, + } + + data, err := json.Marshal(result) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + // Parse back to verify structure + var parsed RankResult + err = json.Unmarshal(data, &parsed) + if err != nil { + t.Fatalf("Round-trip unmarshal failed: %v", err) + } + + if parsed.Index != result.Index { + t.Errorf("Index mismatch: expected %d, got %d", result.Index, parsed.Index) + } + if parsed.Document.Text != result.Document.Text { + t.Errorf("Document text mismatch: expected %q, got %q", result.Document.Text, parsed.Document.Text) + } + if parsed.RelevanceScore != result.RelevanceScore { + t.Errorf("Score mismatch: expected %f, got %f", result.RelevanceScore, parsed.RelevanceScore) + } +} diff --git a/internal/models/utils/ollama/ollama.go b/internal/models/utils/ollama/ollama.go new file mode 100644 index 0000000..76b88ce --- /dev/null +++ b/internal/models/utils/ollama/ollama.go @@ -0,0 +1,316 @@ +package ollama + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "sync" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/ollama/ollama/api" +) + +// OllamaService manages Ollama service +type OllamaService struct { + client *api.Client + baseURL string + mu sync.Mutex + isAvailable bool + isOptional bool // Added: marks if Ollama service is optional +} + +// GetOllamaService gets Ollama service instance (singleton pattern) +func GetOllamaService() (*OllamaService, error) { + // Get Ollama base URL from environment variable, if not set use provided baseURL or default value + logger.GetLogger(context.Background()).Infof("Ollama base URL: %s", os.Getenv("OLLAMA_BASE_URL")) + baseURL := "http://localhost:11434" + envURL := os.Getenv("OLLAMA_BASE_URL") + if envURL != "" { + baseURL = envURL + } + + // Create URL object + parsedURL, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("invalid Ollama service URL: %w", err) + } + + // Create official client + client := api.NewClient(parsedURL, http.DefaultClient) + + // Check if Ollama is set as optional + isOptional := false + if os.Getenv("OLLAMA_OPTIONAL") == "true" { + isOptional = true + logger.GetLogger(context.Background()).Info("Ollama service set to optional mode") + } + + service := &OllamaService{ + client: client, + baseURL: baseURL, + isOptional: isOptional, + } + + return service, nil +} + +// StartService checks if Ollama service is available +func (s *OllamaService) StartService(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + + // Check if service is available + err := s.client.Heartbeat(ctx) + if err != nil { + logger.GetLogger(ctx).Warnf("Ollama service unavailable: %v", err) + s.isAvailable = false + + // If configured as optional, don't return an error + if s.isOptional { + logger.GetLogger(ctx).Info("Ollama service set as optional, will continue running the application") + return nil + } + + return fmt.Errorf("Ollama service unavailable: %w", err) + } + + s.isAvailable = true + logger.GetLogger(ctx).Info("Ollama service ready") + return nil +} + +// IsAvailable returns whether the service is available +func (s *OllamaService) IsAvailable() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.isAvailable +} + +// IsModelAvailable checks if a model is available +func (s *OllamaService) IsModelAvailable(ctx context.Context, modelName string) (bool, error) { + // First check if the service is available + if err := s.StartService(ctx); err != nil { + return false, err + } + + // If service is not available but set as optional, return false but no error + if !s.isAvailable && s.isOptional { + return false, nil + } + + // Get model list + listResp, err := s.client.List(ctx) + if err != nil { + return false, fmt.Errorf("failed to get model list: %w", err) + } + + // Check if model is in the list + for _, model := range listResp.Models { + if model.Name == modelName { + return true, nil + } + } + + return false, nil +} + +// PullModel pulls a model +func (s *OllamaService) PullModel(ctx context.Context, modelName string) error { + // First check if the service is available + if err := s.StartService(ctx); err != nil { + return err + } + + // If service is not available but set as optional, return nil without further operations + if !s.isAvailable && s.isOptional { + logger.GetLogger(ctx).Warnf("Ollama service unavailable, unable to pull model %s", modelName) + return nil + } + + // Check if model already exists + available, err := s.IsModelAvailable(ctx, modelName) + if err != nil { + return err + } + if available { + logger.GetLogger(ctx).Infof("Model %s already exists", modelName) + return nil + } + + logger.GetLogger(ctx).Infof("Pulling model %s...", modelName) + + // Use official client to pull model + pullReq := &api.PullRequest{ + Name: modelName, + } + + err = s.client.Pull(ctx, pullReq, func(progress api.ProgressResponse) error { + if progress.Status != "" { + if progress.Total > 0 && progress.Completed > 0 { + percentage := float64(progress.Completed) / float64(progress.Total) * 100 + logger.GetLogger(ctx).Infof("Pull progress: %s (%.2f%%)", + progress.Status, percentage) + } else { + logger.GetLogger(ctx).Infof("Pull status: %s", progress.Status) + } + } + + if progress.Total > 0 && progress.Completed == progress.Total { + logger.GetLogger(ctx).Infof("Model %s pull completed", modelName) + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to pull model: %w", err) + } + + return nil +} + +// EnsureModelAvailable ensures the model is available, pulls it if not available +func (s *OllamaService) EnsureModelAvailable(ctx context.Context, modelName string) error { + // If service is not available but set as optional, return nil directly + if !s.IsAvailable() && s.isOptional { + logger.GetLogger(ctx).Warnf("Ollama service unavailable, skipping ensuring model %s availability", modelName) + return nil + } + + available, err := s.IsModelAvailable(ctx, modelName) + if err != nil { + if s.isOptional { + logger.GetLogger(ctx).Warnf("Failed to check model %s availability, but Ollama is set as optional", modelName) + return nil + } + return err + } + + if !available { + return s.PullModel(ctx, modelName) + } + + return nil +} + +// GetVersion gets Ollama version +func (s *OllamaService) GetVersion(ctx context.Context) (string, error) { + // If service is not available but set as optional, return empty version info + if !s.IsAvailable() && s.isOptional { + return "unavailable", nil + } + + version, err := s.client.Version(ctx) + if err != nil { + return "", fmt.Errorf("failed to get Ollama version: %w", err) + } + return version, nil +} + +// CreateModel creates a custom model +func (s *OllamaService) CreateModel(ctx context.Context, name, modelfile string) error { + req := &api.CreateRequest{ + Model: name, + Template: modelfile, // Use Template field instead of Modelfile + } + + err := s.client.Create(ctx, req, func(progress api.ProgressResponse) error { + if progress.Status != "" { + logger.GetLogger(ctx).Infof("Model creation status: %s", progress.Status) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to create model: %w", err) + } + + return nil +} + +// GetModelInfo gets model information +func (s *OllamaService) GetModelInfo(ctx context.Context, modelName string) (*api.ShowResponse, error) { + req := &api.ShowRequest{ + Name: modelName, + } + + resp, err := s.client.Show(ctx, req) + if err != nil { + return nil, fmt.Errorf("failed to get model information: %w", err) + } + + return resp, nil +} + +// ListModels lists all available models +func (s *OllamaService) ListModels(ctx context.Context) ([]string, error) { + listResp, err := s.client.List(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get model list: %w", err) + } + + modelNames := make([]string, len(listResp.Models)) + for i, model := range listResp.Models { + modelNames[i] = model.Name + } + + return modelNames, nil +} + +// DeleteModel deletes a model +func (s *OllamaService) DeleteModel(ctx context.Context, modelName string) error { + req := &api.DeleteRequest{ + Name: modelName, + } + + err := s.client.Delete(ctx, req) + if err != nil { + return fmt.Errorf("failed to delete model: %w", err) + } + + return nil +} + +// IsValidModelName checks if model name is valid +func IsValidModelName(name string) bool { + // Simple check for model name format + return name != "" && !strings.Contains(name, " ") +} + +// Chat uses Ollama chat +func (s *OllamaService) Chat(ctx context.Context, req *api.ChatRequest, fn api.ChatResponseFunc) error { + // First check if service is available + if err := s.StartService(ctx); err != nil { + return err + } + + // Use official client Chat method + return s.client.Chat(ctx, req, fn) +} + +// Embeddings gets text embedding vectors +func (s *OllamaService) Embeddings(ctx context.Context, req *api.EmbedRequest) (*api.EmbedResponse, error) { + // First check if service is available + if err := s.StartService(ctx); err != nil { + return nil, err + } + // Use official client Embed method + return s.client.Embed(ctx, req) +} + +// Generate generates text (used for Rerank) +func (s *OllamaService) Generate(ctx context.Context, req *api.GenerateRequest, fn api.GenerateResponseFunc) error { + // First check if service is available + if err := s.StartService(ctx); err != nil { + return err + } + + // Use official client Generate method + return s.client.Generate(ctx, req, fn) +} + +// GetClient returns the underlying ollama client for advanced operations +func (s *OllamaService) GetClient() *api.Client { + return s.client +} diff --git a/internal/models/utils/slices.go b/internal/models/utils/slices.go new file mode 100644 index 0000000..aac713b --- /dev/null +++ b/internal/models/utils/slices.go @@ -0,0 +1,36 @@ +package utils + +// ChunkSlice splits a slice into multiple sub-slices of the specified size +func ChunkSlice[T any](slice []T, chunkSize int) [][]T { + // Handle edge cases + if len(slice) == 0 { + return [][]T{} + } + + if chunkSize <= 0 { + panic("chunkSize must be greater than 0") + } + + // Calculate how many sub-slices are needed + chunks := make([][]T, 0, (len(slice)+chunkSize-1)/chunkSize) + + // Split the slice + for i := 0; i < len(slice); i += chunkSize { + end := i + chunkSize + if end > len(slice) { + end = len(slice) + } + chunks = append(chunks, slice[i:end]) + } + + return chunks +} + +// MapSlice applies a function to each element of a slice and returns a new slice with the results +func MapSlice[A any, B any](in []A, f func(A) B) []B { + out := make([]B, 0, len(in)) + for _, item := range in { + out = append(out, f(item)) + } + return out +} diff --git a/internal/router/router.go b/internal/router/router.go new file mode 100644 index 0000000..38785c8 --- /dev/null +++ b/internal/router/router.go @@ -0,0 +1,282 @@ +package router + +import ( + "time" + + "github.com/gin-contrib/cors" + "github.com/gin-gonic/gin" + "go.uber.org/dig" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/handler" + "github.com/Tencent/WeKnora/internal/middleware" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// RouterParams 路由参数 +type RouterParams struct { + dig.In + + Config *config.Config + UserService interfaces.UserService + KBService interfaces.KnowledgeBaseService + KnowledgeService interfaces.KnowledgeService + ChunkService interfaces.ChunkService + SessionService interfaces.SessionService + MessageService interfaces.MessageService + ModelService interfaces.ModelService + EvaluationService interfaces.EvaluationService + KBHandler *handler.KnowledgeBaseHandler + KnowledgeHandler *handler.KnowledgeHandler + TenantHandler *handler.TenantHandler + TenantService interfaces.TenantService + ChunkHandler *handler.ChunkHandler + SessionHandler *handler.SessionHandler + MessageHandler *handler.MessageHandler + ModelHandler *handler.ModelHandler + EvaluationHandler *handler.EvaluationHandler + AuthHandler *handler.AuthHandler + InitializationHandler *handler.InitializationHandler + SystemHandler *handler.SystemHandler +} + +// NewRouter 创建新的路由 +func NewRouter(params RouterParams) *gin.Engine { + r := gin.New() + + // CORS 中间件应放在最前面 + r.Use(cors.New(cors.Config{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}, + AllowHeaders: []string{"Origin", "Content-Type", "Accept", "Authorization", "X-API-Key", "X-Request-ID"}, + ExposeHeaders: []string{"Content-Length", "Access-Control-Allow-Origin"}, + AllowCredentials: true, + MaxAge: 12 * time.Hour, + })) + + // 其他中间件 + r.Use(middleware.RequestID()) + r.Use(middleware.Logger()) + r.Use(middleware.Recovery()) + r.Use(middleware.ErrorHandler()) + r.Use(middleware.Auth(params.TenantService, params.UserService, params.Config)) + + // 添加OpenTelemetry追踪中间件 + r.Use(middleware.TracingMiddleware()) + + // 健康检查 + r.GET("/health", func(c *gin.Context) { + c.JSON(200, gin.H{"status": "ok"}) + }) + + // 需要认证的API路由 + v1 := r.Group("/api/v1") + { + RegisterAuthRoutes(v1, params.AuthHandler) + RegisterTenantRoutes(v1, params.TenantHandler) + RegisterKnowledgeBaseRoutes(v1, params.KBHandler) + RegisterKnowledgeRoutes(v1, params.KnowledgeHandler) + RegisterChunkRoutes(v1, params.ChunkHandler) + RegisterSessionRoutes(v1, params.SessionHandler) + RegisterChatRoutes(v1, params.SessionHandler) + RegisterMessageRoutes(v1, params.MessageHandler) + RegisterModelRoutes(v1, params.ModelHandler) + RegisterEvaluationRoutes(v1, params.EvaluationHandler) + RegisterInitializationRoutes(v1, params.InitializationHandler) + RegisterSystemRoutes(v1, params.SystemHandler) + } + + return r +} + +// RegisterChunkRoutes 注册分块相关的路由 +func RegisterChunkRoutes(r *gin.RouterGroup, handler *handler.ChunkHandler) { + // 分块路由组 + chunks := r.Group("/chunks") + { + // 获取分块列表 + chunks.GET("/:knowledge_id", handler.ListKnowledgeChunks) + // 删除分块 + chunks.DELETE("/:knowledge_id/:id", handler.DeleteChunk) + // 删除知识下的所有分块 + chunks.DELETE("/:knowledge_id", handler.DeleteChunksByKnowledgeID) + // 更新分块信息 + chunks.PUT("/:knowledge_id/:id", handler.UpdateChunk) + } +} + +// RegisterKnowledgeRoutes 注册知识相关的路由 +func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandler) { + // 知识库下的知识路由组 + kb := r.Group("/knowledge-bases/:id/knowledge") + { + // 从文件创建知识 + kb.POST("/file", handler.CreateKnowledgeFromFile) + // 从URL创建知识 + kb.POST("/url", handler.CreateKnowledgeFromURL) + // 获取知识库下的知识列表 + kb.GET("", handler.ListKnowledge) + } + + // 知识路由组 + k := r.Group("/knowledge") + { + // 批量获取知识 + k.GET("/batch", handler.GetKnowledgeBatch) + // 获取知识详情 + k.GET("/:id", handler.GetKnowledge) + // 删除知识 + k.DELETE("/:id", handler.DeleteKnowledge) + // 更新知识 + k.PUT("/:id", handler.UpdateKnowledge) + // 获取知识文件 + k.GET("/:id/download", handler.DownloadKnowledgeFile) + // 更新图像分块信息 + k.PUT("/image/:id/:chunk_id", handler.UpdateImageInfo) + } +} + +// RegisterKnowledgeBaseRoutes 注册知识库相关的路由 +func RegisterKnowledgeBaseRoutes(r *gin.RouterGroup, handler *handler.KnowledgeBaseHandler) { + // 知识库路由组 + kb := r.Group("/knowledge-bases") + { + // 创建知识库 + kb.POST("", handler.CreateKnowledgeBase) + // 获取知识库列表 + kb.GET("", handler.ListKnowledgeBases) + // 获取知识库详情 + kb.GET("/:id", handler.GetKnowledgeBase) + // 更新知识库 + kb.PUT("/:id", handler.UpdateKnowledgeBase) + // 删除知识库 + kb.DELETE("/:id", handler.DeleteKnowledgeBase) + // 混合搜索 + kb.GET("/:id/hybrid-search", handler.HybridSearch) + // 拷贝知识库 + kb.POST("/copy", handler.CopyKnowledgeBase) + } +} + +// RegisterMessageRoutes 注册消息相关的路由 +func RegisterMessageRoutes(r *gin.RouterGroup, handler *handler.MessageHandler) { + // 消息路由组 + messages := r.Group("/messages") + { + // 加载更早的消息，用于向上滚动加载 + messages.GET("/:session_id/load", handler.LoadMessages) + // 删除消息 + messages.DELETE("/:session_id/:id", handler.DeleteMessage) + } +} + +// RegisterSessionRoutes 注册路由 +func RegisterSessionRoutes(r *gin.RouterGroup, handler *handler.SessionHandler) { + sessions := r.Group("/sessions") + { + sessions.POST("", handler.CreateSession) + sessions.GET("/:id", handler.GetSession) + sessions.GET("", handler.GetSessionsByTenant) + sessions.PUT("/:id", handler.UpdateSession) + sessions.DELETE("/:id", handler.DeleteSession) + sessions.POST("/:session_id/generate_title", handler.GenerateTitle) + // 继续接收活跃流 + sessions.GET("/continue-stream/:session_id", handler.ContinueStream) + } +} + +// RegisterChatRoutes 注册路由 +func RegisterChatRoutes(r *gin.RouterGroup, handler *handler.SessionHandler) { + knowledgeChat := r.Group("/knowledge-chat") + { + knowledgeChat.POST("/:session_id", handler.KnowledgeQA) + } + + // 新增知识检索接口，不需要session_id + knowledgeSearch := r.Group("/knowledge-search") + { + knowledgeSearch.POST("", handler.SearchKnowledge) + } +} + +// RegisterTenantRoutes 注册租户相关的路由 +func RegisterTenantRoutes(r *gin.RouterGroup, handler *handler.TenantHandler) { + // 租户路由组 + tenantRoutes := r.Group("/tenants") + { + tenantRoutes.POST("", handler.CreateTenant) + tenantRoutes.GET("/:id", handler.GetTenant) + tenantRoutes.PUT("/:id", handler.UpdateTenant) + tenantRoutes.DELETE("/:id", handler.DeleteTenant) + tenantRoutes.GET("", handler.ListTenants) + } +} + +// RegisterModelRoutes 注册模型相关的路由 +func RegisterModelRoutes(r *gin.RouterGroup, handler *handler.ModelHandler) { + // 模型路由组 + models := r.Group("/models") + { + // 创建模型 + models.POST("", handler.CreateModel) + // 获取模型列表 + models.GET("", handler.ListModels) + // 获取单个模型 + models.GET("/:id", handler.GetModel) + // 更新模型 + models.PUT("/:id", handler.UpdateModel) + // 删除模型 + models.DELETE("/:id", handler.DeleteModel) + } +} + +func RegisterEvaluationRoutes(r *gin.RouterGroup, handler *handler.EvaluationHandler) { + evaluationRoutes := r.Group("/evaluation") + { + evaluationRoutes.POST("/", handler.Evaluation) + evaluationRoutes.GET("/", handler.GetEvaluationResult) + } +} + +// RegisterAuthRoutes registers authentication routes +func RegisterAuthRoutes(r *gin.RouterGroup, handler *handler.AuthHandler) { + r.POST("/auth/register", handler.Register) + r.POST("/auth/login", handler.Login) + r.POST("/auth/refresh", handler.RefreshToken) + r.GET("/auth/validate", handler.ValidateToken) + r.POST("/auth/logout", handler.Logout) + r.GET("/auth/me", handler.GetCurrentUser) + r.POST("/auth/change-password", handler.ChangePassword) +} + +func RegisterInitializationRoutes(r *gin.RouterGroup, handler *handler.InitializationHandler) { + // 初始化接口 + r.GET("/initialization/config/:kbId", handler.GetCurrentConfigByKB) + r.POST("/initialization/initialize/:kbId", handler.InitializeByKB) + + // Ollama相关接口 + r.GET("/initialization/ollama/status", handler.CheckOllamaStatus) + r.GET("/initialization/ollama/models", handler.ListOllamaModels) + r.POST("/initialization/ollama/models/check", handler.CheckOllamaModels) + r.POST("/initialization/ollama/models/download", handler.DownloadOllamaModel) + r.GET("/initialization/ollama/download/progress/:taskId", handler.GetDownloadProgress) + r.GET("/initialization/ollama/download/tasks", handler.ListDownloadTasks) + + // 远程API相关接口 + r.POST("/initialization/remote/check", handler.CheckRemoteModel) + r.POST("/initialization/embedding/test", handler.TestEmbeddingModel) + r.POST("/initialization/rerank/check", handler.CheckRerankModel) + r.POST("/initialization/multimodal/test", handler.TestMultimodalFunction) + + r.POST("/initialization/extract/text-relation", handler.ExtractTextRelations) + r.POST("/initialization/extract/fabri-tag", handler.FabriTag) + r.POST("/initialization/extract/fabri-text", handler.FabriText) +} + +// RegisterSystemRoutes registers system information routes +func RegisterSystemRoutes(r *gin.RouterGroup, handler *handler.SystemHandler) { + systemRoutes := r.Group("/system") + { + systemRoutes.GET("/info", handler.GetSystemInfo) + } +} diff --git a/internal/router/task.go b/internal/router/task.go new file mode 100644 index 0000000..547c4f1 --- /dev/null +++ b/internal/router/task.go @@ -0,0 +1,66 @@ +package router + +import ( + "log" + "os" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "go.uber.org/dig" +) + +type AsynqTaskParams struct { + dig.In + + Server *asynq.Server + Extracter interfaces.Extracter +} + +func getAsynqRedisClientOpt() *asynq.RedisClientOpt { + opt := &asynq.RedisClientOpt{ + Addr: os.Getenv("REDIS_ADDR"), + Password: os.Getenv("REDIS_PASSWORD"), + ReadTimeout: 100 * time.Millisecond, + WriteTimeout: 200 * time.Millisecond, + DB: 0, + } + return opt +} + +func NewAsyncqClient() *asynq.Client { + opt := getAsynqRedisClientOpt() + client := asynq.NewClient(opt) + return client +} + +func NewAsynqServer() *asynq.Server { + opt := getAsynqRedisClientOpt() + srv := asynq.NewServer( + opt, + asynq.Config{ + Queues: map[string]int{ + "critical": 6, // Highest priority queue + "default": 3, // Default priority queue + "low": 1, // Lowest priority queue + }, + }, + ) + return srv +} + +func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { + // Create a new mux and register all handlers + mux := asynq.NewServeMux() + + mux.HandleFunc(types.TypeChunkExtract, params.Extracter.Extract) + + go func() { + // Start the server + if err := params.Server.Run(mux); err != nil { + log.Fatalf("could not run server: %v", err) + } + }() + return mux +} diff --git a/internal/runtime/container.go b/internal/runtime/container.go new file mode 100644 index 0000000..00fceb5 --- /dev/null +++ b/internal/runtime/container.go @@ -0,0 +1,23 @@ +// Package runtime 提供应用程序运行时的依赖注入容器 +// 该包使用 uber 的 dig 库来管理依赖项注入 +package runtime + +import ( + "go.uber.org/dig" +) + +// container 是应用程序的全局依赖注入容器 +// 所有服务和组件都通过它进行注册和解析 +var container *dig.Container + +// init 初始化依赖注入容器 +// 在程序启动时自动调用 +func init() { + container = dig.New() +} + +// GetContainer 返回全局依赖注入容器的引用 +// 供其他包使用以注册或获取服务 +func GetContainer() *dig.Container { + return container +} diff --git a/internal/stream/factory.go b/internal/stream/factory.go new file mode 100644 index 0000000..7205441 --- /dev/null +++ b/internal/stream/factory.go @@ -0,0 +1,36 @@ +package stream + +import ( + "os" + "strconv" + "time" + + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// 流管理器类型 +const ( + TypeMemory = "memory" + TypeRedis = "redis" +) + +// NewStreamManager 创建流管理器 +func NewStreamManager() (interfaces.StreamManager, error) { + switch os.Getenv("STREAM_MANAGER_TYPE") { + case TypeRedis: + db, err := strconv.Atoi(os.Getenv("REDIS_DB")) + if err != nil { + db = 0 + } + ttl := time.Hour // 默认1小时 + return NewRedisStreamManager( + os.Getenv("REDIS_ADDR"), + os.Getenv("REDIS_PASSWORD"), + db, + os.Getenv("REDIS_PREFIX"), + ttl, + ) + default: + return NewMemoryStreamManager(), nil + } +} diff --git a/internal/stream/memory_manager.go b/internal/stream/memory_manager.go new file mode 100644 index 0000000..3210eea --- /dev/null +++ b/internal/stream/memory_manager.go @@ -0,0 +1,123 @@ +package stream + +import ( + "context" + "sync" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// 内存流信息 +type memoryStreamInfo struct { + sessionID string + requestID string + query string + content string + knowledgeReferences types.References + lastUpdated time.Time + isCompleted bool +} + +// MemoryStreamManager 基于内存的流管理器实现 +type MemoryStreamManager struct { + // 会话ID -> 请求ID -> 流数据 + activeStreams map[string]map[string]*memoryStreamInfo + mu sync.RWMutex +} + +// NewMemoryStreamManager 创建一个新的内存流管理器 +func NewMemoryStreamManager() *MemoryStreamManager { + return &MemoryStreamManager{ + activeStreams: make(map[string]map[string]*memoryStreamInfo), + } +} + +// RegisterStream 注册一个新的流 +func (m *MemoryStreamManager) RegisterStream(ctx context.Context, sessionID, requestID, query string) error { + m.mu.Lock() + defer m.mu.Unlock() + + info := &memoryStreamInfo{ + sessionID: sessionID, + requestID: requestID, + query: query, + lastUpdated: time.Now(), + } + + if _, exists := m.activeStreams[sessionID]; !exists { + m.activeStreams[sessionID] = make(map[string]*memoryStreamInfo) + } + + m.activeStreams[sessionID][requestID] = info + return nil +} + +// UpdateStream 更新流内容 +func (m *MemoryStreamManager) UpdateStream(ctx context.Context, + sessionID, requestID string, content string, references types.References, +) error { + m.mu.Lock() + defer m.mu.Unlock() + + if sessionMap, exists := m.activeStreams[sessionID]; exists { + if stream, found := sessionMap[requestID]; found { + stream.content += content + if len(references) > 0 { + stream.knowledgeReferences = references + } + stream.lastUpdated = time.Now() + } + } + return nil +} + +// CompleteStream 完成流 +func (m *MemoryStreamManager) CompleteStream(ctx context.Context, sessionID, requestID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + if sessionMap, exists := m.activeStreams[sessionID]; exists { + if stream, found := sessionMap[requestID]; found { + stream.isCompleted = true + // 30s 后删除流 + go func() { + time.Sleep(30 * time.Second) + m.mu.Lock() + defer m.mu.Unlock() + delete(sessionMap, requestID) + if len(sessionMap) == 0 { + delete(m.activeStreams, sessionID) + } + }() + } + } + return nil +} + +// GetStream 获取特定流 +func (m *MemoryStreamManager) GetStream(ctx context.Context, + sessionID, requestID string, +) (*interfaces.StreamInfo, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + if sessionMap, exists := m.activeStreams[sessionID]; exists { + if stream, found := sessionMap[requestID]; found { + return &interfaces.StreamInfo{ + SessionID: stream.sessionID, + RequestID: stream.requestID, + Query: stream.query, + Content: stream.content, + KnowledgeReferences: stream.knowledgeReferences, + LastUpdated: stream.lastUpdated, + IsCompleted: stream.isCompleted, + }, nil + } + } + return nil, nil +} + +// 确保实现了接口 +var _ interfaces.StreamManager = (*MemoryStreamManager)(nil) diff --git a/internal/stream/redis_manager.go b/internal/stream/redis_manager.go new file mode 100644 index 0000000..ec4e138 --- /dev/null +++ b/internal/stream/redis_manager.go @@ -0,0 +1,192 @@ +package stream + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/redis/go-redis/v9" +) + +// redisStreamInfo Redis存储的流信息 +type redisStreamInfo struct { + SessionID string `json:"session_id"` + RequestID string `json:"request_id"` + Query string `json:"query"` + Content string `json:"content"` + KnowledgeReferences types.References `json:"knowledge_references"` + LastUpdated time.Time `json:"last_updated"` + IsCompleted bool `json:"is_completed"` +} + +// RedisStreamManager 基于Redis的流管理器实现 +type RedisStreamManager struct { + client *redis.Client + ttl time.Duration // 流数据在Redis中的过期时间 + prefix string // Redis键前缀 +} + +// NewRedisStreamManager 创建一个新的Redis流管理器 +func NewRedisStreamManager(redisAddr, redisPassword string, + redisDB int, prefix string, ttl time.Duration, +) (*RedisStreamManager, error) { + client := redis.NewClient(&redis.Options{ + Addr: redisAddr, + Password: redisPassword, + DB: redisDB, + }) + + // 验证连接 + _, err := client.Ping(context.Background()).Result() + if err != nil { + return nil, fmt.Errorf("连接Redis失败: %w", err) + } + + if ttl == 0 { + ttl = 24 * time.Hour // 默认TTL为24小时 + } + + if prefix == "" { + prefix = "stream:" // 默认前缀 + } + + return &RedisStreamManager{ + client: client, + ttl: ttl, + prefix: prefix, + }, nil +} + +// 构建Redis键 +func (r *RedisStreamManager) buildKey(sessionID, requestID string) string { + return fmt.Sprintf("%s:%s:%s", r.prefix, sessionID, requestID) +} + +// RegisterStream 注册一个新的流 +func (r *RedisStreamManager) RegisterStream(ctx context.Context, sessionID, requestID, query string) error { + info := &redisStreamInfo{ + SessionID: sessionID, + RequestID: requestID, + Query: query, + LastUpdated: time.Now(), + } + + data, err := json.Marshal(info) + if err != nil { + return fmt.Errorf("序列化流信息失败: %w", err) + } + + key := r.buildKey(sessionID, requestID) + return r.client.Set(ctx, key, data, r.ttl).Err() +} + +// UpdateStream 更新流内容 +func (r *RedisStreamManager) UpdateStream(ctx context.Context, sessionID, requestID string, content string, references types.References) error { + key := r.buildKey(sessionID, requestID) + + // 获取当前数据 + data, err := r.client.Get(ctx, key).Bytes() + if err != nil { + if err == redis.Nil { + return nil // 键不存在，可能已过期 + } + return fmt.Errorf("获取流数据失败: %w", err) + } + + var info redisStreamInfo + if err := json.Unmarshal(data, &info); err != nil { + return fmt.Errorf("解析流数据失败: %w", err) + } + + // 更新数据 + info.Content += content + if len(references) > 0 { + info.KnowledgeReferences = references + } + info.LastUpdated = time.Now() + + // 保存回Redis + updatedData, err := json.Marshal(info) + if err != nil { + return fmt.Errorf("序列化更新的流信息失败: %w", err) + } + + return r.client.Set(ctx, key, updatedData, r.ttl).Err() +} + +// CompleteStream 完成流 +func (r *RedisStreamManager) CompleteStream(ctx context.Context, sessionID, requestID string) error { + key := r.buildKey(sessionID, requestID) + + // 获取当前数据 + data, err := r.client.Get(ctx, key).Bytes() + if err != nil { + if err == redis.Nil { + return nil // 键不存在，可能已过期 + } + return fmt.Errorf("获取流数据失败: %w", err) + } + + var info redisStreamInfo + if err := json.Unmarshal(data, &info); err != nil { + return fmt.Errorf("解析流数据失败: %w", err) + } + + // 标记为完成 + info.IsCompleted = true + info.LastUpdated = time.Now() + + // 保存回Redis + updatedData, err := json.Marshal(info) + if err != nil { + return fmt.Errorf("序列化更新的流信息失败: %w", err) + } + + // 30s 后删除流 + go func() { + time.Sleep(30 * time.Second) + r.client.Del(ctx, key) + }() + return r.client.Set(ctx, key, updatedData, r.ttl).Err() +} + +// GetStream 获取特定流 +func (r *RedisStreamManager) GetStream(ctx context.Context, sessionID, requestID string) (*interfaces.StreamInfo, error) { + key := r.buildKey(sessionID, requestID) + + // 获取数据 + data, err := r.client.Get(ctx, key).Bytes() + if err != nil { + if err == redis.Nil { + return nil, nil // 键不存在 + } + return nil, fmt.Errorf("获取流数据失败: %w", err) + } + + var info redisStreamInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, fmt.Errorf("解析流数据失败: %w", err) + } + + // 转换为接口结构 + return &interfaces.StreamInfo{ + SessionID: info.SessionID, + RequestID: info.RequestID, + Query: info.Query, + Content: info.Content, + KnowledgeReferences: info.KnowledgeReferences, + LastUpdated: info.LastUpdated, + IsCompleted: info.IsCompleted, + }, nil +} + +// Close 关闭Redis连接 +func (r *RedisStreamManager) Close() error { + return r.client.Close() +} + +// 确保实现了接口 +var _ interfaces.StreamManager = (*RedisStreamManager)(nil) diff --git a/internal/tracing/init.go b/internal/tracing/init.go new file mode 100644 index 0000000..0d32245 --- /dev/null +++ b/internal/tracing/init.go @@ -0,0 +1,105 @@ +package tracing + +import ( + "context" + "log" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.24.0" + "go.opentelemetry.io/otel/trace" +) + +const ( + AppName = "WeKnoraApp" +) + +type Tracer struct { + Cleanup func(context.Context) error +} + +var tracer trace.Tracer + +// InitTracer initializes OpenTelemetry tracer +func InitTracer() (*Tracer, error) { + // Create resource description + labels := []attribute.KeyValue{ + semconv.TelemetrySDKLanguageGo, + semconv.ServiceNameKey.String(AppName), + } + res := resource.NewWithAttributes(semconv.SchemaURL, labels...) + var err error + + // First try to create OTLP exporter (can connect to Jaeger, Zipkin, etc.) + var traceExporter sdktrace.SpanExporter + if endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); endpoint != "" { + // Use gRPC exporter + client := otlptracegrpc.NewClient( + otlptracegrpc.WithEndpoint(endpoint), + otlptracegrpc.WithInsecure(), + ) + traceExporter, err = otlptrace.New(context.Background(), client) + if err != nil { + return nil, err + } + } else { + // If no OTLP endpoint is set, default to standard output + traceExporter, err = stdouttrace.New() + if err != nil { + return nil, err + } + } + + // Create batch SpanProcessor + bsp := sdktrace.NewBatchSpanProcessor(traceExporter) + + sampler := sdktrace.AlwaysSample() + + // Create and register TracerProvider + tp := sdktrace.NewTracerProvider( + sdktrace.WithSampler(sampler), + sdktrace.WithResource(res), + sdktrace.WithSpanProcessor(bsp), + ) + otel.SetTracerProvider(tp) + + // Set global propagator + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + + // Create Tracer for project use + tracer = tp.Tracer(AppName) + + // Return cleanup function + return &Tracer{ + Cleanup: func(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := tp.Shutdown(ctx); err != nil { + log.Printf("Error shutting down tracer provider: %v", err) + return err + } + return nil + }, + }, nil +} + +// GetTracer gets global Tracer +func GetTracer() trace.Tracer { + return tracer +} + +// Create context with span +func ContextWithSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return GetTracer().Start(ctx, name, opts...) +} diff --git a/internal/types/chat.go b/internal/types/chat.go new file mode 100644 index 0000000..3229cd3 --- /dev/null +++ b/internal/types/chat.go @@ -0,0 +1,64 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" +) + +// ChatResponse chat response +type ChatResponse struct { + Content string `json:"content"` + // Usage information + Usage struct { + // Prompt tokens + PromptTokens int `json:"prompt_tokens"` + // Completion tokens + CompletionTokens int `json:"completion_tokens"` + // Total tokens + TotalTokens int `json:"total_tokens"` + } `json:"usage"` +} + +// Response type +type ResponseType string + +const ( + // Answer response type + ResponseTypeAnswer ResponseType = "answer" + // References response type + ResponseTypeReferences ResponseType = "references" +) + +// StreamResponse stream response +type StreamResponse struct { + // Unique identifier + ID string `json:"id"` + // Response type + ResponseType ResponseType `json:"response_type"` + // Current fragment content + Content string `json:"content"` + // Whether the response is complete + Done bool `json:"done"` + // Knowledge references + KnowledgeReferences References `json:"knowledge_references"` +} + +// References references +type References []*SearchResult + +// Value implements the driver.Valuer interface, used to convert References to database values +func (c References) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database values to References +func (c *References) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} diff --git a/internal/types/chat_manage.go b/internal/types/chat_manage.go new file mode 100644 index 0000000..fff243d --- /dev/null +++ b/internal/types/chat_manage.go @@ -0,0 +1,118 @@ +package types + +// ChatManage represents the configuration and state for a chat session +// including query processing, search parameters, and model configurations +type ChatManage struct { + SessionID string `json:"session_id"` // Unique identifier for the chat session + Query string `json:"query,omitempty"` // Original user query + ProcessedQuery string `json:"processed_query,omitempty"` // Query after preprocessing + RewriteQuery string `json:"rewrite_query,omitempty"` // Query after rewriting for better retrieval + History []*History `json:"history,omitempty"` // Chat history for context + + KnowledgeBaseID string `json:"knowledge_base_id"` // ID of the knowledge base to search against + VectorThreshold float64 `json:"vector_threshold"` // Minimum score threshold for vector search results + KeywordThreshold float64 `json:"keyword_threshold"` // Minimum score threshold for keyword search results + EmbeddingTopK int `json:"embedding_top_k"` // Number of top results to retrieve from embedding search + VectorDatabase string `json:"vector_database"` // Vector database type/name to use + + RerankModelID string `json:"rerank_model_id"` // Model ID for reranking search results + RerankTopK int `json:"rerank_top_k"` // Number of top results after reranking + RerankThreshold float64 `json:"rerank_threshold"` // Minimum score threshold for reranked results + + ChatModelID string `json:"chat_model_id"` // Model ID for chat completion + SummaryConfig SummaryConfig `json:"summary_config"` // Configuration for summary generation + FallbackStrategy FallbackStrategy `json:"fallback_strategy"` // Strategy when no relevant results are found + FallbackResponse string `json:"fallback_response"` // Default response when fallback occurs + + // Internal fields for pipeline data processing + SearchResult []*SearchResult `json:"-"` // Results from search phase + RerankResult []*SearchResult `json:"-"` // Results after reranking + MergeResult []*SearchResult `json:"-"` // Final merged results after all processing + Entity []string `json:"-"` // List of identified entities + GraphResult *GraphData `json:"-"` // Graph data from search phase + UserContent string `json:"-"` // Processed user content + ChatResponse *ChatResponse `json:"-"` // Final response from chat model + ResponseChan <-chan StreamResponse `json:"-"` // Channel for streaming responses +} + +// Clone creates a deep copy of the ChatManage object +func (c *ChatManage) Clone() *ChatManage { + return &ChatManage{ + Query: c.Query, + ProcessedQuery: c.ProcessedQuery, + RewriteQuery: c.RewriteQuery, + SessionID: c.SessionID, + KnowledgeBaseID: c.KnowledgeBaseID, + VectorThreshold: c.VectorThreshold, + KeywordThreshold: c.KeywordThreshold, + EmbeddingTopK: c.EmbeddingTopK, + VectorDatabase: c.VectorDatabase, + RerankModelID: c.RerankModelID, + RerankTopK: c.RerankTopK, + RerankThreshold: c.RerankThreshold, + ChatModelID: c.ChatModelID, + SummaryConfig: SummaryConfig{ + MaxTokens: c.SummaryConfig.MaxTokens, + RepeatPenalty: c.SummaryConfig.RepeatPenalty, + TopK: c.SummaryConfig.TopK, + TopP: c.SummaryConfig.TopP, + FrequencyPenalty: c.SummaryConfig.FrequencyPenalty, + PresencePenalty: c.SummaryConfig.PresencePenalty, + Prompt: c.SummaryConfig.Prompt, + ContextTemplate: c.SummaryConfig.ContextTemplate, + NoMatchPrefix: c.SummaryConfig.NoMatchPrefix, + Temperature: c.SummaryConfig.Temperature, + Seed: c.SummaryConfig.Seed, + MaxCompletionTokens: c.SummaryConfig.MaxCompletionTokens, + }, + FallbackStrategy: c.FallbackStrategy, + FallbackResponse: c.FallbackResponse, + } +} + +// EventType represents different stages in the RAG (Retrieval Augmented Generation) pipeline +type EventType string + +const ( + PREPROCESS_QUERY EventType = "preprocess_query" // Query preprocessing stage + REWRITE_QUERY EventType = "rewrite_query" // Query rewriting for better retrieval + CHUNK_SEARCH EventType = "chunk_search" // Search for relevant chunks + ENTITY_SEARCH EventType = "entity_search" // Search for relevant entities + CHUNK_RERANK EventType = "chunk_rerank" // Rerank search results + CHUNK_MERGE EventType = "chunk_merge" // Merge similar chunks + INTO_CHAT_MESSAGE EventType = "into_chat_message" // Convert chunks into chat messages + CHAT_COMPLETION EventType = "chat_completion" // Generate chat completion + CHAT_COMPLETION_STREAM EventType = "chat_completion_stream" // Stream chat completion + STREAM_FILTER EventType = "stream_filter" // Filter streaming output + FILTER_TOP_K EventType = "filter_top_k" // Keep only top K results +) + +// Pipline defines the sequence of events for different chat modes +var Pipline = map[string][]EventType{ + "chat": { // Simple chat without retrieval + CHAT_COMPLETION, + }, + "chat_stream": { // Streaming chat without retrieval + CHAT_COMPLETION_STREAM, + STREAM_FILTER, + }, + "rag": { // Retrieval Augmented Generation + CHUNK_SEARCH, + CHUNK_RERANK, + CHUNK_MERGE, + INTO_CHAT_MESSAGE, + CHAT_COMPLETION, + }, + "rag_stream": { // Streaming Retrieval Augmented Generation + REWRITE_QUERY, + PREPROCESS_QUERY, + CHUNK_SEARCH, + ENTITY_SEARCH, + CHUNK_RERANK, + CHUNK_MERGE, + FILTER_TOP_K, + INTO_CHAT_MESSAGE, + CHAT_COMPLETION_STREAM, + STREAM_FILTER, + }, +} diff --git a/internal/types/chunk.go b/internal/types/chunk.go new file mode 100644 index 0000000..a8448f6 --- /dev/null +++ b/internal/types/chunk.go @@ -0,0 +1,90 @@ +// Package types defines data structures and types used throughout the system +// These types are shared across different service modules to ensure data consistency +package types + +import ( + "time" + + "gorm.io/gorm" +) + +// ChunkType 定义了不同类型的 Chunk +type ChunkType string + +const ( + // ChunkTypeText 表示普通的文本 Chunk + ChunkTypeText ChunkType = "text" + // ChunkTypeImageOCR 表示图片 OCR 文本的 Chunk + ChunkTypeImageOCR ChunkType = "image_ocr" + // ChunkTypeImageCaption 表示图片描述的 Chunk + ChunkTypeImageCaption ChunkType = "image_caption" + // ChunkTypeSummary 表示摘要类型的 Chunk + ChunkTypeSummary = "summary" + // ChunkTypeEntity 表示实体类型的 Chunk + ChunkTypeEntity ChunkType = "entity" + // ChunkTypeRelationship 表示关系类型的 Chunk + ChunkTypeRelationship ChunkType = "relationship" +) + +// ImageInfo 表示与 Chunk 关联的图片信息 +type ImageInfo struct { + // 图片URL（COS） + URL string `json:"url" gorm:"type:text"` + // 原始图片URL + OriginalURL string `json:"original_url" gorm:"type:text"` + // 图片在文本中的开始位置 + StartPos int `json:"start_pos"` + // 图片在文本中的结束位置 + EndPos int `json:"end_pos"` + // 图片描述 + Caption string `json:"caption"` + // 图片OCR文本 + OCRText string `json:"ocr_text"` +} + +// Chunk represents a document chunk +// Chunks are meaningful text segments extracted from original documents +// and are the basic units of knowledge base retrieval +// Each chunk contains a portion of the original content +// and maintains its positional relationship with the original text +// Chunks can be independently embedded as vectors and retrieved, supporting precise content localization +type Chunk struct { + // Unique identifier of the chunk, using UUID format + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + // Tenant ID, used for multi-tenant isolation + TenantID uint `json:"tenant_id"` + // ID of the parent knowledge, associated with the Knowledge model + KnowledgeID string `json:"knowledge_id"` + // ID of the knowledge base, for quick location + KnowledgeBaseID string `json:"knowledge_base_id"` + // Actual text content of the chunk + Content string `json:"content"` + // Index position of the chunk in the original document + ChunkIndex int `json:"chunk_index"` + // Whether the chunk is enabled, can be used to temporarily disable certain chunks + IsEnabled bool `json:"is_enabled" gorm:"default:true"` + // Starting character position in the original text + StartAt int `json:"start_at" ` + // Ending character position in the original text + EndAt int `json:"end_at"` + // Previous chunk ID + PreChunkID string `json:"pre_chunk_id"` + // Next chunk ID + NextChunkID string `json:"next_chunk_id"` + // Chunk 类型，用于区分不同类型的 Chunk + ChunkType ChunkType `json:"chunk_type" gorm:"type:varchar(20);default:'text'"` + // 父 Chunk ID，用于关联图片 Chunk 和原始文本 Chunk + ParentChunkID string `json:"parent_chunk_id" gorm:"type:varchar(36);index"` + // 关系 Chunk ID，用于关联关系 Chunk 和原始文本 Chunk + RelationChunks JSON `json:"relation_chunks" gorm:"type:json"` + // 间接关系 Chunk ID，用于关联间接关系 Chunk 和原始文本 Chunk + IndirectRelationChunks JSON `json:"indirect_relation_chunks" gorm:"type:json"` + // 图片信息，存储为 JSON + ImageInfo string `json:"image_info" gorm:"type:text"` + // Chunk creation time + CreatedAt time.Time `json:"created_at"` + // Chunk last update time + UpdatedAt time.Time `json:"updated_at"` + // Soft delete marker, supports data recovery + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` +} diff --git a/internal/types/cleanup.go b/internal/types/cleanup.go new file mode 100644 index 0000000..6fcb066 --- /dev/null +++ b/internal/types/cleanup.go @@ -0,0 +1,4 @@ +package types + +// CleanupFunc represents the resource cleanup function +type CleanupFunc func() error diff --git a/internal/types/const.go b/internal/types/const.go new file mode 100644 index 0000000..1588868 --- /dev/null +++ b/internal/types/const.go @@ -0,0 +1,20 @@ +package types + +// ContextKey defines a type for context keys to avoid string collision +type ContextKey string + +const ( + // TenantIDContextKey is the context key for tenant ID + TenantIDContextKey ContextKey = "TenantID" + // TenantInfoContextKey is the context key for tenant information + TenantInfoContextKey ContextKey = "TenantInfo" + // RequestIDContextKey is the context key for request ID + RequestIDContextKey ContextKey = "RequestID" + // LoggerContextKey is the context key for logger + LoggerContextKey ContextKey = "Logger" +) + +// String returns the string representation of the context key +func (c ContextKey) String() string { + return string(c) +} diff --git a/internal/types/dataset.go b/internal/types/dataset.go new file mode 100644 index 0000000..d4fc7df --- /dev/null +++ b/internal/types/dataset.go @@ -0,0 +1,11 @@ +package types + +// QAPair represents a complete QA example with question, related passages and answer +type QAPair struct { + QID int // Question ID + Question string // Question text + PIDs []int // Related passage IDs + Passages []string // Passage texts + AID int // Answer ID + Answer string // Answer text +} diff --git a/internal/types/embedding.go b/internal/types/embedding.go new file mode 100644 index 0000000..19fc810 --- /dev/null +++ b/internal/types/embedding.go @@ -0,0 +1,34 @@ +package types + +// SourceType represents the type of content source +type SourceType int + +const ( + ChunkSourceType SourceType = iota // Source is a text chunk + PassageSourceType // Source is a passage + SummarySourceType // Source is a summary +) + +// MatchType represents the type of matching algorithm +type MatchType int + +const ( + MatchTypeEmbedding MatchType = iota + MatchTypeKeywords + MatchTypeNearByChunk + MatchTypeHistory + MatchTypeParentChunk // 父Chunk匹配类型 + MatchTypeRelationChunk // 关系Chunk匹配类型 + MatchTypeGraph +) + +// IndexInfo contains information about indexed content +type IndexInfo struct { + ID string // Unique identifier + Content string // Content text + SourceID string // ID of the source document + SourceType SourceType // Type of the source + ChunkID string // ID of the text chunk + KnowledgeID string // ID of the knowledge + KnowledgeBaseID string // ID of the knowledge base +} diff --git a/internal/types/errors.go b/internal/types/errors.go new file mode 100644 index 0000000..c9b0dfc --- /dev/null +++ b/internal/types/errors.go @@ -0,0 +1,46 @@ +package types + +import "fmt" + +// StorageQuotaExceededError represents the storage quota exceeded error +type StorageQuotaExceededError struct { + Message string +} + +// Error implements the error interface +func (e *StorageQuotaExceededError) Error() string { + return e.Message +} + +// NewStorageQuotaExceededError creates a storage quota exceeded error +func NewStorageQuotaExceededError() *StorageQuotaExceededError { + return &StorageQuotaExceededError{ + Message: "Storage quota exceeded", + } +} + +// DuplicateKnowledgeError duplicate knowledge error, contains the existing knowledge object +type DuplicateKnowledgeError struct { + Message string + Knowledge *Knowledge +} + +func (e *DuplicateKnowledgeError) Error() string { + return e.Message +} + +// NewDuplicateFileError creates a duplicate file error +func NewDuplicateFileError(knowledge *Knowledge) *DuplicateKnowledgeError { + return &DuplicateKnowledgeError{ + Message: fmt.Sprintf("File already exists: %s", knowledge.FileName), + Knowledge: knowledge, + } +} + +// NewDuplicateURLError creates a duplicate URL error +func NewDuplicateURLError(knowledge *Knowledge) *DuplicateKnowledgeError { + return &DuplicateKnowledgeError{ + Message: fmt.Sprintf("URL already exists: %s", knowledge.Source), + Knowledge: knowledge, + } +} diff --git a/internal/types/evaluation.go b/internal/types/evaluation.go new file mode 100644 index 0000000..5e477c6 --- /dev/null +++ b/internal/types/evaluation.go @@ -0,0 +1,99 @@ +package types + +import ( + "encoding/json" + "time" + + "github.com/yanyiwu/gojieba" +) + +// Jieba is a global instance of Chinese text segmentation tool +var Jieba *gojieba.Jieba = gojieba.NewJieba() + +// EvaluationStatue represents the status of an evaluation task +type EvaluationStatue int + +const ( + EvaluationStatuePending EvaluationStatue = iota // Task is waiting to start + EvaluationStatueRunning // Task is in progress + EvaluationStatueSuccess // Task completed successfully + EvaluationStatueFailed // Task failed +) + +// EvaluationTask contains information about an evaluation task +type EvaluationTask struct { + ID string `json:"id"` // Unique task ID + TenantID uint `json:"tenant_id"` // Tenant/Organization ID + DatasetID string `json:"dataset_id"` // Dataset ID for evaluation + + StartTime time.Time `json:"start_time"` // Task start time + Status EvaluationStatue `json:"status"` // Current task status + ErrMsg string `json:"err_msg,omitempty"` // Error message if failed + + Total int `json:"total,omitempty"` // Total items to evaluate + Finished int `json:"finished,omitempty"` // Completed items count +} + +// EvaluationDetail contains detailed evaluation information +type EvaluationDetail struct { + Task *EvaluationTask `json:"task"` // Evaluation task info + Params *ChatManage `json:"params"` // Evaluation parameters + Metric *MetricResult `json:"metric,omitempty"` // Evaluation metrics +} + +// String returns JSON representation of EvaluationTask +func (e *EvaluationTask) String() string { + b, _ := json.Marshal(e) + return string(b) +} + +// MetricInput contains input data for metric calculation +type MetricInput struct { + RetrievalGT [][]int // Ground truth for retrieval + RetrievalIDs []int // Retrieved IDs + + GeneratedTexts string // Generated text for evaluation + GeneratedGT string // Ground truth text for comparison +} + +// MetricResult contains evaluation metrics +type MetricResult struct { + RetrievalMetrics RetrievalMetrics `json:"retrieval_metrics"` // Retrieval performance metrics + GenerationMetrics GenerationMetrics `json:"generation_metrics"` // Text generation quality metrics +} + +// RetrievalMetrics contains metrics for retrieval evaluation +type RetrievalMetrics struct { + Precision float64 `json:"precision"` // Precision score + Recall float64 `json:"recall"` // Recall score + + NDCG3 float64 `json:"ndcg3"` // Normalized Discounted Cumulative Gain at 3 + NDCG10 float64 `json:"ndcg10"` // Normalized Discounted Cumulative Gain at 10 + MRR float64 `json:"mrr"` // Mean Reciprocal Rank + MAP float64 `json:"map"` // Mean Average Precision +} + +// GenerationMetrics contains metrics for text generation evaluation +type GenerationMetrics struct { + BLEU1 float64 `json:"bleu1"` // BLEU-1 score + BLEU2 float64 `json:"bleu2"` // BLEU-2 score + BLEU4 float64 `json:"bleu4"` // BLEU-4 score + + ROUGE1 float64 `json:"rouge1"` // ROUGE-1 score + ROUGE2 float64 `json:"rouge2"` // ROUGE-2 score + ROUGEL float64 `json:"rougel"` // ROUGE-L score +} + +// EvalState represents different stages of evaluation process +type EvalState int + +const ( + StateBegin EvalState = iota // Evaluation started + StateAfterQaPairs // After loading QA pairs + StateAfterDataset // After processing dataset + StateAfterEmbedding // After generating embeddings + StateAfterVectorSearch // After vector search + StateAfterRerank // After reranking + StateAfterComplete // After completion + StateEnd // Evaluation ended +) diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go new file mode 100644 index 0000000..2bffdaf --- /dev/null +++ b/internal/types/extract_graph.go @@ -0,0 +1,51 @@ +package types + +const ( + TypeChunkExtract = "chunk:extract" +) + +type ExtractChunkPayload struct { + TenantID uint `json:"tenant_id"` + ChunkID string `json:"chunk_id"` + ModelID string `json:"model_id"` +} + +type PromptTemplateStructured struct { + Description string `json:"description"` + Tags []string `json:"tags"` + Examples []GraphData `json:"examples"` +} + +type GraphNode struct { + Name string `json:"name,omitempty"` + Chunks []string `json:"chunks,omitempty"` + Attributes []string `json:"attributes,omitempty"` +} + +type GraphRelation struct { + Node1 string `json:"node1,omitempty"` + Node2 string `json:"node2,omitempty"` + Type string `json:"type,omitempty"` +} + +type GraphData struct { + Text string `json:"text,omitempty"` + Node []*GraphNode `json:"node,omitempty"` + Relation []*GraphRelation `json:"relation,omitempty"` +} + +type NameSpace struct { + KnowledgeBase string `json:"knowledge_base"` + Knowledge string `json:"knowledge"` +} + +func (n NameSpace) Labels() []string { + res := make([]string, 0) + if n.KnowledgeBase != "" { + res = append(res, n.KnowledgeBase) + } + if n.Knowledge != "" { + res = append(res, n.Knowledge) + } + return res +} diff --git a/internal/types/graph.go b/internal/types/graph.go new file mode 100644 index 0000000..e3f750a --- /dev/null +++ b/internal/types/graph.go @@ -0,0 +1,53 @@ +// Package types defines the core data structures and interfaces used throughout the WeKnora system. +package types + +import "context" + +// Entity represents a node in the knowledge graph extracted from document chunks. +// Each entity corresponds to a meaningful concept, person, place or thing identified in the text. +type Entity struct { + ID string // Unique identifier for the entity + ChunkIDs []string // References to document chunks where this entity appears + Frequency int `json:"-"` // Number of occurrences in the corpus + Degree int `json:"-"` // Number of connections to other entities + Title string `json:"title"` // Display name of the entity + Type string `json:"type"` // Classification of the entity (e.g., person, concept, organization) + Description string `json:"description"` // Brief explanation or context about the entity +} + +// Relationship represents a connection between two entities in the knowledge graph. +// It captures the semantic connection between entities identified in the document chunks. +type Relationship struct { + ID string `json:"-"` // Unique identifier for the relationship + ChunkIDs []string `json:"-"` // References to document chunks where this relationship is established + CombinedDegree int `json:"-"` // Sum of degrees of the connected entities, used for ranking + Weight float64 `json:"-"` // Strength of the relationship based on textual evidence + Source string `json:"source"` // ID of the entity where the relationship starts + Target string `json:"target"` // ID of the entity where the relationship ends + Description string `json:"description"` // Description of how these entities are related + Strength int `json:"strength"` // Normalized measure of relationship importance (1-10) +} + +// GraphBuilder defines the interface for building and querying the knowledge graph. +// It provides methods to construct the graph from document chunks and retrieve related information. +type GraphBuilder interface { + // BuildGraph constructs a knowledge graph from the provided document chunks. + // It extracts entities and relationships, then builds the graph structure. + BuildGraph(ctx context.Context, chunks []*Chunk) error + + // GetRelationChunks retrieves the IDs of chunks directly related to the specified chunk. + // The topK parameter limits the number of results returned, based on relationship strength. + GetRelationChunks(chunkID string, topK int) []string + + // GetIndirectRelationChunks finds chunk IDs that are indirectly connected to the specified chunk. + // These are "second-degree" connections, useful for expanding the context during retrieval. + GetIndirectRelationChunks(chunkID string, topK int) []string + + // GetAllEntities returns all entities currently in the knowledge graph. + // This is primarily used for visualization and diagnostics. + GetAllEntities() []*Entity + + // GetAllRelationships returns all relationships currently in the knowledge graph. + // This is primarily used for visualization and diagnostics. + GetAllRelationships() []*Relationship +} diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go new file mode 100644 index 0000000..81ea87b --- /dev/null +++ b/internal/types/interfaces/chunk.go @@ -0,0 +1,62 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// ChunkRepository defines the interface for chunk repository operations +type ChunkRepository interface { + // CreateChunks creates chunks + CreateChunks(ctx context.Context, chunks []*types.Chunk) error + // GetChunkByID gets a chunk by id + GetChunkByID(ctx context.Context, tenantID uint, id string) (*types.Chunk, error) + // ListChunksByID lists chunks by ids + ListChunksByID(ctx context.Context, tenantID uint, ids []string) ([]*types.Chunk, error) + // ListChunksByKnowledgeID lists chunks by knowledge id + ListChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) ([]*types.Chunk, error) + // ListPagedChunksByKnowledgeID lists paged chunks by knowledge id + ListPagedChunksByKnowledgeID( + ctx context.Context, + tenantID uint, + knowledgeID string, + page *types.Pagination, + chunk_type []types.ChunkType, + ) ([]*types.Chunk, int64, error) + ListChunkByParentID(ctx context.Context, tenantID uint, parentID string) ([]*types.Chunk, error) + // UpdateChunk updates a chunk + UpdateChunk(ctx context.Context, chunk *types.Chunk) error + // DeleteChunk deletes a chunk + DeleteChunk(ctx context.Context, tenantID uint, id string) error + // DeleteChunksByKnowledgeID deletes chunks by knowledge id + DeleteChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) error + // DeleteByKnowledgeList deletes all chunks for a knowledge list + DeleteByKnowledgeList(ctx context.Context, tenantID uint, knowledgeIDs []string) error +} + +// ChunkService defines the interface for chunk service operations +type ChunkService interface { + // CreateChunks creates chunks + CreateChunks(ctx context.Context, chunks []*types.Chunk) error + // GetChunkByID gets a chunk by id + GetChunkByID(ctx context.Context, knowledgeID string, id string) (*types.Chunk, error) + // ListChunksByKnowledgeID lists chunks by knowledge id + ListChunksByKnowledgeID(ctx context.Context, knowledgeID string) ([]*types.Chunk, error) + // ListPagedChunksByKnowledgeID lists paged chunks by knowledge id + ListPagedChunksByKnowledgeID( + ctx context.Context, + knowledgeID string, + page *types.Pagination, + ) (*types.PageResult, error) + // UpdateChunk updates a chunk + UpdateChunk(ctx context.Context, chunk *types.Chunk) error + // DeleteChunk deletes a chunk + DeleteChunk(ctx context.Context, id string) error + // DeleteChunksByKnowledgeID deletes chunks by knowledge id + DeleteChunksByKnowledgeID(ctx context.Context, knowledgeID string) error + // DeleteByKnowledgeList deletes all chunks for a knowledge list + DeleteByKnowledgeList(ctx context.Context, ids []string) error + // ListChunkByParentID lists chunks by parent id + ListChunkByParentID(ctx context.Context, tenantID uint, parentID string) ([]*types.Chunk, error) +} diff --git a/internal/types/interfaces/evaluation.go b/internal/types/interfaces/evaluation.go new file mode 100644 index 0000000..db89a79 --- /dev/null +++ b/internal/types/interfaces/evaluation.go @@ -0,0 +1,35 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// EvaluationService defines operations for evaluation tasks +type EvaluationService interface { + // Evaluation starts a new evaluation task + Evaluation(ctx context.Context, datasetID string, knowledgeBaseID string, + chatModelID string, rerankModelID string, + ) (*types.EvaluationDetail, error) + // EvaluationResult retrieves evaluation result by task ID + EvaluationResult(ctx context.Context, taskID string) (*types.EvaluationDetail, error) +} + +// Metrics defines interface for computing evaluation metrics +type Metrics interface { + // Compute calculates metric score based on input data + Compute(metricInput *types.MetricInput) float64 +} + +// EvalHook defines interface for evaluation process hooks +type EvalHook interface { + // Handle processes evaluation state change + Handle(ctx context.Context, state types.EvalState, index int, data interface{}) error +} + +// DatasetService defines operations for dataset management +type DatasetService interface { + // GetDatasetByID retrieves QA pairs from dataset by ID + GetDatasetByID(ctx context.Context, datasetID string) ([]*types.QAPair, error) +} diff --git a/internal/types/interfaces/extracter.go b/internal/types/interfaces/extracter.go new file mode 100644 index 0000000..c764ad1 --- /dev/null +++ b/internal/types/interfaces/extracter.go @@ -0,0 +1,11 @@ +package interfaces + +import ( + "context" + + "github.com/hibiken/asynq" +) + +type Extracter interface { + Extract(ctx context.Context, t *asynq.Task) error +} diff --git a/internal/types/interfaces/file.go b/internal/types/interfaces/file.go new file mode 100644 index 0000000..307aaea --- /dev/null +++ b/internal/types/interfaces/file.go @@ -0,0 +1,18 @@ +package interfaces + +import ( + "context" + "io" + "mime/multipart" +) + +// FileService is the interface for file services. +// FileService provides methods to save, retrieve, and delete files. +type FileService interface { + // SaveFile saves a file. + SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint, knowledgeID string) (string, error) + // GetFile retrieves a file. + GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) + // DeleteFile deletes a file. + DeleteFile(ctx context.Context, filePath string) error +} diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go new file mode 100644 index 0000000..db798e6 --- /dev/null +++ b/internal/types/interfaces/knowledge.go @@ -0,0 +1,74 @@ +package interfaces + +import ( + "context" + "io" + "mime/multipart" + + "github.com/Tencent/WeKnora/internal/types" +) + +// KnowledgeService defines the interface for knowledge services. +type KnowledgeService interface { + // CreateKnowledgeFromFile creates knowledge from a file. + CreateKnowledgeFromFile( + ctx context.Context, + kbID string, + file *multipart.FileHeader, + metadata map[string]string, + enableMultimodel *bool, + ) (*types.Knowledge, error) + // CreateKnowledgeFromURL creates knowledge from a URL. + CreateKnowledgeFromURL(ctx context.Context, kbID string, url string, enableMultimodel *bool) (*types.Knowledge, error) + // CreateKnowledgeFromPassage creates knowledge from text passages. + CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error) + // GetKnowledgeByID retrieves knowledge by ID. + GetKnowledgeByID(ctx context.Context, id string) (*types.Knowledge, error) + // GetKnowledgeBatch retrieves a batch of knowledge by IDs. + GetKnowledgeBatch(ctx context.Context, tenantID uint, ids []string) ([]*types.Knowledge, error) + // ListKnowledgeByKnowledgeBaseID lists all knowledge under a knowledge base. + ListKnowledgeByKnowledgeBaseID(ctx context.Context, kbID string) ([]*types.Knowledge, error) + // ListPagedKnowledgeByKnowledgeBaseID lists all knowledge under a knowledge base with pagination. + ListPagedKnowledgeByKnowledgeBaseID( + ctx context.Context, + kbID string, + page *types.Pagination, + ) (*types.PageResult, error) + // DeleteKnowledge deletes knowledge by ID. + DeleteKnowledge(ctx context.Context, id string) error + // GetKnowledgeFile retrieves the file associated with the knowledge. + GetKnowledgeFile(ctx context.Context, id string) (io.ReadCloser, string, error) + // UpdateKnowledge updates knowledge information. + UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error + // CloneKnowledgeBase clones knowledge to another knowledge base. + CloneKnowledgeBase(ctx context.Context, srcID, dstID string) error + // UpdateImageInfo updates image information for a knowledge chunk. + UpdateImageInfo(ctx context.Context, knowledgeID string, chunkID string, imageInfo string) error +} + +// KnowledgeRepository defines the interface for knowledge repositories. +type KnowledgeRepository interface { + CreateKnowledge(ctx context.Context, knowledge *types.Knowledge) error + GetKnowledgeByID(ctx context.Context, tenantID uint, id string) (*types.Knowledge, error) + ListKnowledgeByKnowledgeBaseID(ctx context.Context, tenantID uint, kbID string) ([]*types.Knowledge, error) + ListPagedKnowledgeByKnowledgeBaseID(ctx context.Context, + tenantID uint, kbID string, page *types.Pagination, + ) ([]*types.Knowledge, int64, error) + UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error + DeleteKnowledge(ctx context.Context, tenantID uint, id string) error + DeleteKnowledgeList(ctx context.Context, tenantID uint, ids []string) error + GetKnowledgeBatch(ctx context.Context, tenantID uint, ids []string) ([]*types.Knowledge, error) + // CheckKnowledgeExists checks if knowledge already exists. + // For file types, check by fileHash or (fileName+fileSize). + // For URL types, check by URL. + // Returns whether it exists, the existing knowledge object (if any), and possible error. + CheckKnowledgeExists( + ctx context.Context, + tenantID uint, + kbID string, + params *types.KnowledgeCheckParams, + ) (bool, *types.Knowledge, error) + // AminusB returns the difference set of A and B. + AminusB(ctx context.Context, Atenant uint, A string, Btenant uint, B string) ([]string, error) + UpdateKnowledgeColumn(ctx context.Context, id string, column string, value interface{}) error +} diff --git a/internal/types/interfaces/knowledgebase.go b/internal/types/interfaces/knowledgebase.go new file mode 100644 index 0000000..38bcf34 --- /dev/null +++ b/internal/types/interfaces/knowledgebase.go @@ -0,0 +1,138 @@ +// Package interfaces defines the interface contracts between different system components +// Through interface definitions, business logic can be decoupled from specific implementations, +// improving code testability and maintainability +// Knowledge base related interfaces are used to manage knowledge base resources and their contents +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// KnowledgeBaseService defines the knowledge base service interface +// Provides high-level operations for knowledge base creation, querying, updating, deletion, and content searching +type KnowledgeBaseService interface { + // CreateKnowledgeBase creates a new knowledge base + // Parameters: + // - ctx: Context information, carrying request tracking, user identity, etc. + // - kb: Knowledge base object containing basic information + // Returns: + // - Created knowledge base object (including automatically generated ID) + // - Possible errors such as insufficient permissions, duplicate names, etc. + CreateKnowledgeBase(ctx context.Context, kb *types.KnowledgeBase) (*types.KnowledgeBase, error) + + // GetKnowledgeBaseByID retrieves knowledge base information by ID + // Parameters: + // - ctx: Context information + // - id: Unique identifier of the knowledge base + // Returns: + // - Knowledge base object, if found + // - Possible errors such as not existing, insufficient permissions, etc. + GetKnowledgeBaseByID(ctx context.Context, id string) (*types.KnowledgeBase, error) + + // ListKnowledgeBases lists all knowledge bases under the current tenant + // Parameters: + // - ctx: Context information, containing tenant information + // Returns: + // - List of knowledge base objects + // - Possible errors such as insufficient permissions, etc. + ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) + + // UpdateKnowledgeBase updates knowledge base information + // Parameters: + // - ctx: Context information + // - id: Unique identifier of the knowledge base + // - name: New knowledge base name + // - description: New knowledge base description + // - config: Knowledge base configuration, including chunking strategy, vectorization settings, etc. + // Returns: + // - Updated knowledge base object + // - Possible errors such as not existing, insufficient permissions, etc. + UpdateKnowledgeBase(ctx context.Context, + id string, name string, description string, config *types.KnowledgeBaseConfig, + ) (*types.KnowledgeBase, error) + + // DeleteKnowledgeBase deletes a knowledge base + // Parameters: + // - ctx: Context information + // - id: Unique identifier of the knowledge base + // Returns: + // - Possible errors such as not existing, insufficient permissions, etc. + DeleteKnowledgeBase(ctx context.Context, id string) error + + // HybridSearch performs hybrid search (vector + keywords) in the knowledge base + // Parameters: + // - ctx: Context information + // - id: Unique identifier of the knowledge base + // - params: Search parameters, including query text, thresholds, etc. + // Returns: + // - List of search results, sorted by relevance + // - Possible errors such as not existing, insufficient permissions, search engine errors, etc. + HybridSearch(ctx context.Context, id string, params types.SearchParams) ([]*types.SearchResult, error) + + // CopyKnowledgeBase copies a knowledge base + // Parameters: + // - ctx: Context information + // - sourceID: Source knowledge base ID + // - targetID: Target knowledge base ID + // Returns: + // - Copied knowledge base object + // - Possible errors such as not existing, insufficient permissions, etc. + CopyKnowledgeBase(ctx context.Context, src string, dst string) (*types.KnowledgeBase, *types.KnowledgeBase, error) +} + +// KnowledgeBaseRepository defines the knowledge base repository interface +// Responsible for knowledge base data persistence and retrieval, +// serving as a bridge between the service layer and data storage +type KnowledgeBaseRepository interface { + // CreateKnowledgeBase creates a knowledge base record + // Parameters: + // - ctx: Context information + // - kb: Knowledge base object + // Returns: + // - Possible errors such as database connection failure, unique constraint conflicts, etc. + CreateKnowledgeBase(ctx context.Context, kb *types.KnowledgeBase) error + + // GetKnowledgeBaseByID queries a knowledge base by ID + // Parameters: + // - ctx: Context information + // - id: Knowledge base ID + // Returns: + // - Knowledge base object, if found + // - Possible errors such as record not existing, database errors, etc. + GetKnowledgeBaseByID(ctx context.Context, id string) (*types.KnowledgeBase, error) + + // ListKnowledgeBases lists all knowledge bases in the system + // Parameters: + // - ctx: Context information + // Returns: + // - List of knowledge base objects + // - Possible errors such as database errors, etc. + ListKnowledgeBases(ctx context.Context) ([]*types.KnowledgeBase, error) + + // ListKnowledgeBasesByTenantID lists all knowledge bases for a specific tenant + // Parameters: + // - ctx: Context information + // - tenantID: Tenant ID + // Returns: + // - List of knowledge base objects + // - Possible errors such as database errors, etc. + ListKnowledgeBasesByTenantID(ctx context.Context, tenantID uint) ([]*types.KnowledgeBase, error) + + // UpdateKnowledgeBase updates a knowledge base record + // Parameters: + // - ctx: Context information + // - kb: Knowledge base object containing update information + // Returns: + // - Possible errors such as record not existing, database errors, etc. + UpdateKnowledgeBase(ctx context.Context, kb *types.KnowledgeBase) error + + // DeleteKnowledgeBase deletes a knowledge base record + // Parameters: + // - ctx: Context information + // - id: Knowledge base ID + // Returns: + // - Possible errors such as record not existing, database errors, etc. + DeleteKnowledgeBase(ctx context.Context, id string) error +} diff --git a/internal/types/interfaces/message.go b/internal/types/interfaces/message.go new file mode 100644 index 0000000..ffb1777 --- /dev/null +++ b/internal/types/interfaces/message.go @@ -0,0 +1,41 @@ +package interfaces + +import ( + "context" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +// MessageService defines the message service interface +type MessageService interface { + // CreateMessage creates a message + CreateMessage(ctx context.Context, message *types.Message) (*types.Message, error) + + // GetMessage gets a message + GetMessage(ctx context.Context, sessionID string, id string) (*types.Message, error) + + // GetMessagesBySession gets all messages of a session + GetMessagesBySession(ctx context.Context, sessionID string, page int, pageSize int) ([]*types.Message, error) + + // GetRecentMessagesBySession gets recent messages of a session + GetRecentMessagesBySession(ctx context.Context, sessionID string, limit int) ([]*types.Message, error) + + // GetMessagesBySessionBeforeTime gets messages before a specific time of a session + GetMessagesBySessionBeforeTime( + ctx context.Context, sessionID string, beforeTime time.Time, limit int, + ) ([]*types.Message, error) + + // UpdateMessage updates a message + UpdateMessage(ctx context.Context, message *types.Message) error + + // DeleteMessage deletes a message + DeleteMessage(ctx context.Context, sessionID string, id string) error +} + +// MessageRepository defines the message repository interface +type MessageRepository interface { + MessageService + // GetFirstMessageOfUser gets the first message of a user + GetFirstMessageOfUser(ctx context.Context, sessionID string) (*types.Message, error) +} diff --git a/internal/types/interfaces/model.go b/internal/types/interfaces/model.go new file mode 100644 index 0000000..7742222 --- /dev/null +++ b/internal/types/interfaces/model.go @@ -0,0 +1,49 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/types" +) + +// ModelService defines the model service interface +type ModelService interface { + // CreateModel creates a model + CreateModel(ctx context.Context, model *types.Model) error + // GetModelByID gets a model by ID + GetModelByID(ctx context.Context, id string) (*types.Model, error) + // ListModels lists all models + ListModels(ctx context.Context) ([]*types.Model, error) + // UpdateModel updates a model + UpdateModel(ctx context.Context, model *types.Model) error + // DeleteModel deletes a model + DeleteModel(ctx context.Context, id string) error + // GetEmbeddingModel gets an embedding model + GetEmbeddingModel(ctx context.Context, modelId string) (embedding.Embedder, error) + // GetRerankModel gets a rerank model + GetRerankModel(ctx context.Context, modelId string) (rerank.Reranker, error) + // GetChatModel gets a chat model + GetChatModel(ctx context.Context, modelId string) (chat.Chat, error) +} + +// ModelRepository defines the model repository interface +type ModelRepository interface { + // Create creates a model + Create(ctx context.Context, model *types.Model) error + // GetByID gets a model by ID + GetByID(ctx context.Context, tenantID uint, id string) (*types.Model, error) + // List lists all models + List( + ctx context.Context, + tenantID uint, + modelType types.ModelType, + source types.ModelSource, + ) ([]*types.Model, error) + // Update updates a model + Update(ctx context.Context, model *types.Model) error + // Delete deletes a model + Delete(ctx context.Context, tenantID uint, id string) error +} diff --git a/internal/types/interfaces/resource.go b/internal/types/interfaces/resource.go new file mode 100644 index 0000000..07c6cd8 --- /dev/null +++ b/internal/types/interfaces/resource.go @@ -0,0 +1,19 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// ResourceCleaner defines the resource cleaner interface +type ResourceCleaner interface { + // Register registers a resource cleanup function + Register(cleanup types.CleanupFunc) + + // RegisterWithName registers a resource cleanup function with a name + RegisterWithName(name string, cleanup types.CleanupFunc) + + // Cleanup executes all resource cleanup functions + Cleanup(ctx context.Context) []error +} diff --git a/internal/types/interfaces/retriever.go b/internal/types/interfaces/retriever.go new file mode 100644 index 0000000..7969ab7 --- /dev/null +++ b/internal/types/interfaces/retriever.go @@ -0,0 +1,109 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/types" +) + +// RetrieveEngine defines the retrieve engine interface +type RetrieveEngine interface { + // EngineType gets the retrieve engine type + EngineType() types.RetrieverEngineType + + // Retrieve executes the retrieve + Retrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) + + // Support gets the supported retrieve types + Support() []types.RetrieverType +} + +// RetrieveEngineRepository defines the retrieve engine repository interface +type RetrieveEngineRepository interface { + // Save saves the index info + Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error + + // BatchSave saves the index info list + BatchSave(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) error + + // EstimateStorageSize estimates the storage size + EstimateStorageSize(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) int64 + + // DeleteByChunkIDList deletes the index info by chunk id list + DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error + // 复制索引数据 + // sourceKnowledgeBaseID: 源知识库ID + // sourceToTargetChunkIDMap: 源分块ID到目标分块ID的映射关系 + // targetKnowledgeBaseID: 目标知识库ID + // params: 额外参数，如向量表示等 + CopyIndices( + ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, + ) error + + // DeleteByKnowledgeIDList deletes the index info by knowledge id list + DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error + + // RetrieveEngine retrieves the engine + RetrieveEngine +} + +// RetrieveEngineRegistry defines the retrieve engine registry interface +type RetrieveEngineRegistry interface { + // Register registers the retrieve engine service + Register(indexService RetrieveEngineService) error + // GetRetrieveEngineService gets the retrieve engine service + GetRetrieveEngineService(engineType types.RetrieverEngineType) (RetrieveEngineService, error) + // GetAllRetrieveEngineServices gets all retrieve engine services + GetAllRetrieveEngineServices() []RetrieveEngineService +} + +// RetrieveEngineService defines the retrieve engine service interface +type RetrieveEngineService interface { + // Index indexes the index info + Index(ctx context.Context, + embedder embedding.Embedder, + indexInfo *types.IndexInfo, + retrieverTypes []types.RetrieverType, + ) error + + // BatchIndex indexes the index info list + BatchIndex(ctx context.Context, + embedder embedding.Embedder, + indexInfoList []*types.IndexInfo, + retrieverTypes []types.RetrieverType, + ) error + + // EstimateStorageSize estimates the storage size + EstimateStorageSize(ctx context.Context, + embedder embedding.Embedder, + indexInfoList []*types.IndexInfo, + retrieverTypes []types.RetrieverType, + ) int64 + // CopyIndices 从源知识库复制索引到目标知识库，免去重新计算嵌入向量的开销 + // sourceKnowledgeBaseID: 源知识库ID + // sourceToTargetChunkIDMap: 源分块ID到目标分块ID的映射关系，key为源分块ID，value为目标分块ID + // targetKnowledgeBaseID: 目标知识库ID + CopyIndices( + ctx context.Context, + sourceKnowledgeBaseID string, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, + dimension int, + ) error + + // DeleteByChunkIDList deletes the index info by chunk id list + DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error + + // DeleteByKnowledgeIDList deletes the index info by knowledge id list + DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error + + // RetrieveEngine retrieves the engine + RetrieveEngine +} diff --git a/internal/types/interfaces/retriever_graph.go b/internal/types/interfaces/retriever_graph.go new file mode 100644 index 0000000..04f939d --- /dev/null +++ b/internal/types/interfaces/retriever_graph.go @@ -0,0 +1,13 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +type RetrieveGraphRepository interface { + AddGraph(ctx context.Context, namespace types.NameSpace, graphs []*types.GraphData) error + DelGraph(ctx context.Context, namespace []types.NameSpace) error + SearchNode(ctx context.Context, namespace types.NameSpace, nodes []string) (*types.GraphData, error) +} diff --git a/internal/types/interfaces/session.go b/internal/types/interfaces/session.go new file mode 100644 index 0000000..1f5c5ab --- /dev/null +++ b/internal/types/interfaces/session.go @@ -0,0 +1,49 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// SessionService defines the session service interface +type SessionService interface { + // CreateSession creates a session + CreateSession(ctx context.Context, session *types.Session) (*types.Session, error) + // GetSession gets a session + GetSession(ctx context.Context, id string) (*types.Session, error) + // GetSessionsByTenant gets all sessions of a tenant + GetSessionsByTenant(ctx context.Context) ([]*types.Session, error) + // GetPagedSessionsByTenant gets paged sessions of a tenant + GetPagedSessionsByTenant(ctx context.Context, page *types.Pagination) (*types.PageResult, error) + // UpdateSession updates a session + UpdateSession(ctx context.Context, session *types.Session) error + // DeleteSession deletes a session + DeleteSession(ctx context.Context, id string) error + // GenerateTitle generates a title for the current conversation + GenerateTitle(ctx context.Context, sessionID string, messages []types.Message) (string, error) + // KnowledgeQA performs knowledge-based question answering + KnowledgeQA(ctx context.Context, + sessionID, query string, + ) ([]*types.SearchResult, <-chan types.StreamResponse, error) + // KnowledgeQAByEvent performs knowledge-based question answering by event + KnowledgeQAByEvent(ctx context.Context, chatManage *types.ChatManage, eventList []types.EventType) error + // SearchKnowledge performs knowledge-based search, without summarization + SearchKnowledge(ctx context.Context, knowledgeBaseID, query string) ([]*types.SearchResult, error) +} + +// SessionRepository defines the session repository interface +type SessionRepository interface { + // Create creates a session + Create(ctx context.Context, session *types.Session) (*types.Session, error) + // Get gets a session + Get(ctx context.Context, tenantID uint, id string) (*types.Session, error) + // GetByTenantID gets all sessions of a tenant + GetByTenantID(ctx context.Context, tenantID uint) ([]*types.Session, error) + // GetPagedByTenantID gets paged sessions of a tenant + GetPagedByTenantID(ctx context.Context, tenantID uint, page *types.Pagination) ([]*types.Session, int64, error) + // Update updates a session + Update(ctx context.Context, session *types.Session) error + // Delete deletes a session + Delete(ctx context.Context, tenantID uint, id string) error +} diff --git a/internal/types/interfaces/stream_manager.go b/internal/types/interfaces/stream_manager.go new file mode 100644 index 0000000..d6a26ca --- /dev/null +++ b/internal/types/interfaces/stream_manager.go @@ -0,0 +1,34 @@ +package interfaces + +import ( + "context" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +// StreamInfo stream information +type StreamInfo struct { + SessionID string // session ID + RequestID string // request ID + Query string // query content + Content string // current content + KnowledgeReferences types.References // knowledge references + LastUpdated time.Time // last updated time + IsCompleted bool // whether completed +} + +// StreamManager stream manager interface +type StreamManager interface { + // RegisterStream registers a new stream + RegisterStream(ctx context.Context, sessionID, requestID, query string) error + + // UpdateStream updates the stream content + UpdateStream(ctx context.Context, sessionID, requestID string, content string, references types.References) error + + // CompleteStream completes the stream + CompleteStream(ctx context.Context, sessionID, requestID string) error + + // GetStream gets a specific stream + GetStream(ctx context.Context, sessionID, requestID string) (*StreamInfo, error) +} diff --git a/internal/types/interfaces/tenant.go b/internal/types/interfaces/tenant.go new file mode 100644 index 0000000..df5839e --- /dev/null +++ b/internal/types/interfaces/tenant.go @@ -0,0 +1,41 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// TenantService defines the tenant service interface +type TenantService interface { + // CreateTenant creates a tenant + CreateTenant(ctx context.Context, tenant *types.Tenant) (*types.Tenant, error) + // GetTenantByID gets a tenant by ID + GetTenantByID(ctx context.Context, id uint) (*types.Tenant, error) + // ListTenants lists all tenants + ListTenants(ctx context.Context) ([]*types.Tenant, error) + // UpdateTenant updates a tenant + UpdateTenant(ctx context.Context, tenant *types.Tenant) (*types.Tenant, error) + // DeleteTenant deletes a tenant + DeleteTenant(ctx context.Context, id uint) error + // UpdateAPIKey updates the API key + UpdateAPIKey(ctx context.Context, id uint) (string, error) + // ExtractTenantIDFromAPIKey extracts the tenant ID from the API key + ExtractTenantIDFromAPIKey(apiKey string) (uint, error) +} + +// TenantRepository defines the tenant repository interface +type TenantRepository interface { + // CreateTenant creates a tenant + CreateTenant(ctx context.Context, tenant *types.Tenant) error + // GetTenantByID gets a tenant by ID + GetTenantByID(ctx context.Context, id uint) (*types.Tenant, error) + // ListTenants lists all tenants + ListTenants(ctx context.Context) ([]*types.Tenant, error) + // UpdateTenant updates a tenant + UpdateTenant(ctx context.Context, tenant *types.Tenant) error + // DeleteTenant deletes a tenant + DeleteTenant(ctx context.Context, id uint) error + // AdjustStorageUsed adjusts the storage used for a tenant + AdjustStorageUsed(ctx context.Context, tenantID uint, delta int64) error +} diff --git a/internal/types/interfaces/user.go b/internal/types/interfaces/user.go new file mode 100644 index 0000000..2c892a9 --- /dev/null +++ b/internal/types/interfaces/user.go @@ -0,0 +1,75 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// UserService defines the user service interface +type UserService interface { + // Register creates a new user account + Register(ctx context.Context, req *types.RegisterRequest) (*types.User, error) + // Login authenticates a user and returns tokens + Login(ctx context.Context, req *types.LoginRequest) (*types.LoginResponse, error) + // GetUserByID gets a user by ID + GetUserByID(ctx context.Context, id string) (*types.User, error) + // GetUserByEmail gets a user by email + GetUserByEmail(ctx context.Context, email string) (*types.User, error) + // GetUserByUsername gets a user by username + GetUserByUsername(ctx context.Context, username string) (*types.User, error) + // UpdateUser updates user information + UpdateUser(ctx context.Context, user *types.User) error + // DeleteUser deletes a user + DeleteUser(ctx context.Context, id string) error + // ChangePassword changes user password + ChangePassword(ctx context.Context, userID string, oldPassword, newPassword string) error + // ValidatePassword validates user password + ValidatePassword(ctx context.Context, userID string, password string) error + // GenerateTokens generates access and refresh tokens for user + GenerateTokens(ctx context.Context, user *types.User) (accessToken, refreshToken string, err error) + // ValidateToken validates an access token + ValidateToken(ctx context.Context, token string) (*types.User, error) + // RefreshToken refreshes access token using refresh token + RefreshToken(ctx context.Context, refreshToken string) (accessToken, newRefreshToken string, err error) + // RevokeToken revokes a token + RevokeToken(ctx context.Context, token string) error + // GetCurrentUser gets current user from context + GetCurrentUser(ctx context.Context) (*types.User, error) +} + +// UserRepository defines the user repository interface +type UserRepository interface { + // CreateUser creates a user + CreateUser(ctx context.Context, user *types.User) error + // GetUserByID gets a user by ID + GetUserByID(ctx context.Context, id string) (*types.User, error) + // GetUserByEmail gets a user by email + GetUserByEmail(ctx context.Context, email string) (*types.User, error) + // GetUserByUsername gets a user by username + GetUserByUsername(ctx context.Context, username string) (*types.User, error) + // UpdateUser updates a user + UpdateUser(ctx context.Context, user *types.User) error + // DeleteUser deletes a user + DeleteUser(ctx context.Context, id string) error + // ListUsers lists users with pagination + ListUsers(ctx context.Context, offset, limit int) ([]*types.User, error) +} + +// AuthTokenRepository defines the auth token repository interface +type AuthTokenRepository interface { + // CreateToken creates an auth token + CreateToken(ctx context.Context, token *types.AuthToken) error + // GetTokenByValue gets a token by its value + GetTokenByValue(ctx context.Context, tokenValue string) (*types.AuthToken, error) + // GetTokensByUserID gets all tokens for a user + GetTokensByUserID(ctx context.Context, userID string) ([]*types.AuthToken, error) + // UpdateToken updates a token + UpdateToken(ctx context.Context, token *types.AuthToken) error + // DeleteToken deletes a token + DeleteToken(ctx context.Context, id string) error + // DeleteExpiredTokens deletes all expired tokens + DeleteExpiredTokens(ctx context.Context) error + // RevokeTokensByUserID revokes all tokens for a user + RevokeTokensByUserID(ctx context.Context, userID string) error +} diff --git a/internal/types/json.go b/internal/types/json.go new file mode 100644 index 0000000..e2aae2d --- /dev/null +++ b/internal/types/json.go @@ -0,0 +1,68 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "errors" +) + +// JSON is a custom type that wraps json.RawMessage. +// Used for storing JSON data in the database. +type JSON json.RawMessage + +// Scan implements the sql.Scanner interface. +func (j *JSON) Scan(value interface{}) error { + bytes, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + + result := json.RawMessage{} + err := json.Unmarshal(bytes, &result) + *j = JSON(result) + return err +} + +// Value implements the driver.Valuer interface. +func (j JSON) Value() (driver.Value, error) { + if len(j) == 0 { + return nil, nil + } + return json.RawMessage(j).MarshalJSON() +} + +// MarshalJSON implements the json.Marshaler interface. +func (j JSON) MarshalJSON() ([]byte, error) { + if len(j) == 0 { + return []byte("null"), nil + } + return j, nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (j *JSON) UnmarshalJSON(data []byte) error { + if j == nil { + return errors.New("JSON: UnmarshalJSON on nil pointer") + } + *j = JSON(data) + return nil +} + +// ToString converts JSON to a string. +func (j JSON) ToString() string { + if len(j) == 0 { + return "{}" + } + return string(j) +} + +// Map converts JSON to a map. +func (j JSON) Map() (map[string]interface{}, error) { + if len(j) == 0 { + return map[string]interface{}{}, nil + } + + var m map[string]interface{} + err := json.Unmarshal(j, &m) + return m, err +} diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go new file mode 100644 index 0000000..46b68d1 --- /dev/null +++ b/internal/types/knowledge.go @@ -0,0 +1,92 @@ +package types + +import ( + "fmt" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +// Knowledge represents a knowledge entity in the system. +// It contains metadata about the knowledge source, its processing status, +// and references to the physical file if applicable. +type Knowledge struct { + // Unique identifier of the knowledge + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + // Tenant ID + TenantID uint `json:"tenant_id"` + // ID of the knowledge base + KnowledgeBaseID string `json:"knowledge_base_id"` + // Type of the knowledge + Type string `json:"type"` + // Title of the knowledge + Title string `json:"title"` + // Description of the knowledge + Description string `json:"description"` + // Source of the knowledge + Source string `json:"source"` + // Parse status of the knowledge + ParseStatus string `json:"parse_status"` + // Enable status of the knowledge + EnableStatus string `json:"enable_status"` + // ID of the embedding model + EmbeddingModelID string `json:"embedding_model_id"` + // File name of the knowledge + FileName string `json:"file_name"` + // File type of the knowledge + FileType string `json:"file_type"` + // File size of the knowledge + FileSize int64 `json:"file_size"` + // File hash of the knowledge + FileHash string `json:"file_hash"` + // File path of the knowledge + FilePath string `json:"file_path"` + // Storage size of the knowledge + StorageSize int64 `json:"storage_size"` + // Metadata of the knowledge + Metadata JSON `json:"metadata" gorm:"type:json"` + // Creation time of the knowledge + CreatedAt time.Time `json:"created_at"` + // Last updated time of the knowledge + UpdatedAt time.Time `json:"updated_at"` + // Processed time of the knowledge + ProcessedAt *time.Time `json:"processed_at"` + // Error message of the knowledge + ErrorMessage string `json:"error_message"` + // Deletion time of the knowledge + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` +} + +// GetMetadata returns the metadata as a map[string]string. +func (k *Knowledge) GetMetadata() map[string]string { + metadata := make(map[string]string) + metadataMap, err := k.Metadata.Map() + if err != nil { + return nil + } + for k, v := range metadataMap { + metadata[k] = fmt.Sprintf("%v", v) + } + return metadata +} + +// BeforeCreate hook generates a UUID for new Knowledge entities before they are created. +func (k *Knowledge) BeforeCreate(tx *gorm.DB) (err error) { + k.ID = uuid.New().String() + return nil +} + +// KnowledgeCheckParams defines parameters used to check if knowledge already exists. +type KnowledgeCheckParams struct { + // File parameters + FileName string + FileSize int64 + FileHash string + // URL parameters + URL string + // Text passage parameters + Passages []string + // Knowledge type + Type string +} diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go new file mode 100644 index 0000000..c2a42ec --- /dev/null +++ b/internal/types/knowledgebase.go @@ -0,0 +1,195 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "time" + + "gorm.io/gorm" +) + +const ( + InitDefaultKnowledgeBaseID = "kb-00000001" +) + +// KnowledgeBase represents a knowledge base +type KnowledgeBase struct { + // Unique identifier of the knowledge base + ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"` + // Name of the knowledge base + Name string `yaml:"name" json:"name"` + // Description of the knowledge base + Description string `yaml:"description" json:"description"` + // Tenant ID + TenantID uint `yaml:"tenant_id" json:"tenant_id"` + // Chunking configuration + ChunkingConfig ChunkingConfig `yaml:"chunking_config" json:"chunking_config" gorm:"type:json"` + // Image processing configuration + ImageProcessingConfig ImageProcessingConfig `yaml:"image_processing_config" json:"image_processing_config" gorm:"type:json"` + // ID of the embedding model + EmbeddingModelID string `yaml:"embedding_model_id" json:"embedding_model_id"` + // Summary model ID + SummaryModelID string `yaml:"summary_model_id" json:"summary_model_id"` + // Rerank model ID + RerankModelID string `yaml:"rerank_model_id" json:"rerank_model_id"` + // VLM model ID + VLMModelID string `yaml:"vlm_model_id" json:"vlm_model_id"` + // VLM config + VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"` + // Storage config + StorageConfig StorageConfig `yaml:"cos_config" json:"cos_config" gorm:"column:cos_config;type:json"` + // Extract config + ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"` + // Creation time of the knowledge base + CreatedAt time.Time `yaml:"created_at" json:"created_at"` + // Last updated time of the knowledge base + UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"` + // Deletion time of the knowledge base + DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"` +} + +// KnowledgeBaseConfig represents the knowledge base configuration +type KnowledgeBaseConfig struct { + // Chunking configuration + ChunkingConfig ChunkingConfig `yaml:"chunking_config" json:"chunking_config"` + // Image processing configuration + ImageProcessingConfig ImageProcessingConfig `yaml:"image_processing_config" json:"image_processing_config"` +} + +// ChunkingConfig represents the document splitting configuration +type ChunkingConfig struct { + // Chunk size + ChunkSize int `yaml:"chunk_size" json:"chunk_size"` + // Chunk overlap + ChunkOverlap int `yaml:"chunk_overlap" json:"chunk_overlap"` + // Separators + Separators []string `yaml:"separators" json:"separators"` + // Enable multimodal + EnableMultimodal bool `yaml:"enable_multimodal" json:"enable_multimodal"` +} + +// COSConfig represents the COS configuration +type StorageConfig struct { + // Secret ID + SecretID string `yaml:"secret_id" json:"secret_id"` + // Secret Key + SecretKey string `yaml:"secret_key" json:"secret_key"` + // Region + Region string `yaml:"region" json:"region"` + // Bucket Name + BucketName string `yaml:"bucket_name" json:"bucket_name"` + // App ID + AppID string `yaml:"app_id" json:"app_id"` + // Path Prefix + PathPrefix string `yaml:"path_prefix" json:"path_prefix"` + // Provider + Provider string `yaml:"provider" json:"provider"` +} + +func (c StorageConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +func (c *StorageConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// ImageProcessingConfig represents the image processing configuration +type ImageProcessingConfig struct { + // Model ID + ModelID string `yaml:"model_id" json:"model_id"` +} + +// Value implements the driver.Valuer interface, used to convert ChunkingConfig to database value +func (c ChunkingConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to ChunkingConfig +func (c *ChunkingConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// Value implements the driver.Valuer interface, used to convert ImageProcessingConfig to database value +func (c ImageProcessingConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to ImageProcessingConfig +func (c *ImageProcessingConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// VLMConfig represents the VLM configuration +type VLMConfig struct { + // Model Name + ModelName string `yaml:"model_name" json:"model_name"` + // Base URL + BaseURL string `yaml:"base_url" json:"base_url"` + // API Key + APIKey string `yaml:"api_key" json:"api_key"` + // Interface Type: "ollama" or "openai" + InterfaceType string `yaml:"interface_type" json:"interface_type"` +} + +// Value implements the driver.Valuer interface, used to convert VLMConfig to database value +func (c VLMConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to VLMConfig +func (c *VLMConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +type ExtractConfig struct { + Text string `yaml:"text" json:"text"` + Tags []string `yaml:"tags" json:"tags"` + Nodes []*GraphNode `yaml:"nodes" json:"nodes"` + Relations []*GraphRelation `yaml:"relations" json:"relations"` +} + +// Value implements the driver.Valuer interface, used to convert ExtractConfig to database value +func (e ExtractConfig) Value() (driver.Value, error) { + return json.Marshal(e) +} + +// Scan implements the sql.Scanner interface, used to convert database value to ExtractConfig +func (e *ExtractConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, e) +} diff --git a/internal/types/message.go b/internal/types/message.go new file mode 100644 index 0000000..21409e2 --- /dev/null +++ b/internal/types/message.go @@ -0,0 +1,60 @@ +// Package types defines data structures and types used throughout the system +package types + +import ( + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +// History represents a conversation history entry +// Contains query-answer pairs and associated knowledge references +// Used for tracking conversation context and history +type History struct { + Query string // User query text + Answer string // System response text + CreateAt time.Time // When this history entry was created + KnowledgeReferences References // Knowledge references used in the answer +} + +// Message represents a conversation message +// Each message belongs to a conversation session and can be from either user or system +// Messages can contain references to knowledge chunks used to generate responses +type Message struct { + // Unique identifier for the message + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + // ID of the session this message belongs to + SessionID string `json:"session_id"` + // Request identifier for tracking API requests + RequestID string `json:"request_id"` + // Message text content + Content string `json:"content"` + // Message role: "user", "assistant", "system" + Role string `json:"role"` + // References to knowledge chunks used in the response + KnowledgeReferences References `json:"knowledge_references" gorm:"type:json,column:knowledge_references"` + // Whether message generation is complete + IsCompleted bool `json:"is_completed"` + // Message creation timestamp + CreatedAt time.Time `json:"created_at"` + // Last update timestamp + UpdatedAt time.Time `json:"updated_at"` + // Soft delete timestamp + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` +} + +// BeforeCreate is a GORM hook that runs before creating a new message record +// Automatically generates a UUID for new messages and initializes knowledge references +// Parameters: +// - tx: GORM database transaction +// +// Returns: +// - error: Any error encountered during the hook execution +func (m *Message) BeforeCreate(tx *gorm.DB) (err error) { + m.ID = uuid.New().String() + if m.KnowledgeReferences == nil { + m.KnowledgeReferences = make(References, 0) + } + return nil +} diff --git a/internal/types/model.go b/internal/types/model.go new file mode 100644 index 0000000..d70fada --- /dev/null +++ b/internal/types/model.go @@ -0,0 +1,105 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +// ModelType represents the type of AI model +type ModelType string + +const ( + ModelTypeEmbedding ModelType = "Embedding" // Embedding model + ModelTypeRerank ModelType = "Rerank" // Rerank model + ModelTypeKnowledgeQA ModelType = "KnowledgeQA" // KnowledgeQA model + ModelTypeVLLM ModelType = "VLLM" // VLLM model +) + +type ModelStatus string + +const ( + ModelStatusActive ModelStatus = "active" // Model is active + ModelStatusDownloading ModelStatus = "downloading" // Model is downloading + ModelStatusDownloadFailed ModelStatus = "download_failed" // Model download failed +) + +// ModelSource represents the source of the model +type ModelSource string + +const ( + ModelSourceLocal ModelSource = "local" // Local model + ModelSourceRemote ModelSource = "remote" // Remote model + ModelSourceAliyun ModelSource = "aliyun" // Aliyun DashScope model +) + +type EmbeddingParameters struct { + Dimension int `yaml:"dimension" json:"dimension"` + TruncatePromptTokens int `yaml:"truncate_prompt_tokens" json:"truncate_prompt_tokens"` +} + +type ModelParameters struct { + BaseURL string `yaml:"base_url" json:"base_url"` + APIKey string `yaml:"api_key" json:"api_key"` + EmbeddingParameters EmbeddingParameters `yaml:"embedding_parameters" json:"embedding_parameters"` +} + +// Model represents the AI model +type Model struct { + // Unique identifier of the model + ID string `yaml:"id" json:"id" gorm:"type:varchar(36);primaryKey"` + // Tenant ID + TenantID uint `yaml:"tenant_id" json:"tenant_id"` + // Name of the model + Name string `yaml:"name" json:"name"` + // Type of the model + Type ModelType `yaml:"type" json:"type"` + // Source of the model + Source ModelSource `yaml:"source" json:"source"` + // Description of the model + Description string `yaml:"description" json:"description"` + // Model parameters in JSON format + Parameters ModelParameters `yaml:"parameters" json:"parameters" gorm:"type:json"` + // Whether the model is the default model + IsDefault bool `yaml:"is_default" json:"is_default"` + // Model status, default: active, possible: downloading, download_failed + Status ModelStatus `yaml:"status" json:"status"` + // Creation time of the model + CreatedAt time.Time `yaml:"created_at" json:"created_at"` + // Last updated time of the model + UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"` + // Deletion time of the model + DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"` +} + +// Value implements the driver.Valuer interface, used to convert ModelParameters to database value +func (c ModelParameters) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to ModelParameters +func (c *ModelParameters) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// BeforeCreate is a GORM hook that runs before creating a new model record +// Automatically generates a UUID for new models +// Parameters: +// - tx: GORM database transaction +// +// Returns: +// - error: Any error encountered during the hook execution +func (m *Model) BeforeCreate(tx *gorm.DB) (err error) { + m.ID = uuid.New().String() + return nil +} diff --git a/internal/types/retriever.go b/internal/types/retriever.go new file mode 100644 index 0000000..e4de533 --- /dev/null +++ b/internal/types/retriever.go @@ -0,0 +1,82 @@ +package types + +// RetrieverEngineType represents the type of retriever engine +type RetrieverEngineType string + +// RetrieverEngineType constants +const ( + PostgresRetrieverEngineType RetrieverEngineType = "postgres" + ElasticsearchRetrieverEngineType RetrieverEngineType = "elasticsearch" + InfinityRetrieverEngineType RetrieverEngineType = "infinity" + ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss" +) + +// RetrieverType represents the type of retriever +type RetrieverType string + +// RetrieverType constants +const ( + KeywordsRetrieverType RetrieverType = "keywords" // Keywords retriever + VectorRetrieverType RetrieverType = "vector" // Vector retriever + WebSearchRetrieverType RetrieverType = "websearch" // Web search retriever +) + +// RetrieveParams represents the parameters for retrieval +type RetrieveParams struct { + // Query text + Query string + // Query embedding (used for vector retrieval) + Embedding []float32 + // Knowledge base IDs + KnowledgeBaseIDs []string + // Excluded knowledge IDs + ExcludeKnowledgeIDs []string + // Excluded chunk IDs + ExcludeChunkIDs []string + // Number of results to return + TopK int + // Similarity threshold + Threshold float64 + // Additional parameters, different retrievers may require different parameters + AdditionalParams map[string]interface{} + // Retriever type + RetrieverType RetrieverType // Retriever type +} + +// RetrieverEngineParams represents the parameters for retriever engine +type RetrieverEngineParams struct { + // Retriever engine type + RetrieverEngineType RetrieverEngineType `yaml:"retriever_engine_type" json:"retriever_engine_type"` + // Retriever type + RetrieverType RetrieverType `yaml:"retriever_type" json:"retriever_type"` +} + +// IndexWithScore represents the index with score +type IndexWithScore struct { + // ID + ID string + // Content + Content string + // Source ID + SourceID string + // Source type + SourceType SourceType + // Chunk ID + ChunkID string + // Knowledge ID + KnowledgeID string + // Knowledge base ID + KnowledgeBaseID string + // Score + Score float64 + // Match type + MatchType MatchType +} + +// RetrieveResult represents the result of retrieval +type RetrieveResult struct { + Results []*IndexWithScore // Retrieval results + RetrieverEngineType RetrieverEngineType // Retrieval source type + RetrieverType RetrieverType // Retrieval type + Error error // Retrieval error +} diff --git a/internal/types/search.go b/internal/types/search.go new file mode 100644 index 0000000..7e23d3c --- /dev/null +++ b/internal/types/search.go @@ -0,0 +1,129 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" +) + +// SearchResult represents the search result +type SearchResult struct { + // ID + ID string `gorm:"column:id" json:"id"` + // Content + Content string `gorm:"column:content" json:"content"` + // Knowledge ID + KnowledgeID string `gorm:"column:knowledge_id" json:"knowledge_id"` + // Chunk index + ChunkIndex int `gorm:"column:chunk_index" json:"chunk_index"` + // Knowledge title + KnowledgeTitle string `gorm:"column:knowledge_title" json:"knowledge_title"` + // Start at + StartAt int `gorm:"column:start_at" json:"start_at"` + // End at + EndAt int `gorm:"column:end_at" json:"end_at"` + // Seq + Seq int `gorm:"column:seq" json:"seq"` + // Score + Score float64 `json:"score"` + // Match type + MatchType MatchType `json:"match_type"` + // SubChunkIndex + SubChunkID []string `json:"sub_chunk_id"` + // Metadata + Metadata map[string]string `json:"metadata"` + + // Chunk 类型 + ChunkType string `json:"chunk_type"` + // 父 Chunk ID + ParentChunkID string `json:"parent_chunk_id"` + // 图片信息 (JSON 格式) + ImageInfo string `json:"image_info"` + + // Knowledge file name + // Used for file type knowledge, contains the original file name + KnowledgeFilename string `json:"knowledge_filename"` + + // Knowledge source + // Used to indicate the source of the knowledge, such as "url" + KnowledgeSource string `json:"knowledge_source"` +} + +// SearchParams represents the search parameters +type SearchParams struct { + QueryText string `json:"query_text"` + VectorThreshold float64 `json:"vector_threshold"` + KeywordThreshold float64 `json:"keyword_threshold"` + MatchCount int `json:"match_count"` +} + +// Value implements the driver.Valuer interface, used to convert SearchResult to database value +func (c SearchResult) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to SearchResult +func (c *SearchResult) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// Pagination represents the pagination parameters +type Pagination struct { + // Page + Page int `form:"page" json:"page" binding:"omitempty,min=1"` + // Page size + PageSize int `form:"page_size" json:"page_size" binding:"omitempty,min=1,max=100"` +} + +// GetPage gets the page number, default is 1 +func (p *Pagination) GetPage() int { + if p.Page < 1 { + return 1 + } + return p.Page +} + +// GetPageSize gets the page size, default is 20 +func (p *Pagination) GetPageSize() int { + if p.PageSize < 1 { + return 20 + } + if p.PageSize > 100 { + return 100 + } + return p.PageSize +} + +// Offset gets the offset for database query +func (p *Pagination) Offset() int { + return (p.GetPage() - 1) * p.GetPageSize() +} + +// Limit gets the limit for database query +func (p *Pagination) Limit() int { + return p.GetPageSize() +} + +// PageResult represents the pagination query result +type PageResult struct { + Total int64 `json:"total"` // Total number of records + Page int `json:"page"` // Current page number + PageSize int `json:"page_size"` // Page size + Data interface{} `json:"data"` // Data +} + +// NewPageResult creates a new pagination result +func NewPageResult(total int64, page *Pagination, data interface{}) *PageResult { + return &PageResult{ + Total: total, + Page: page.GetPage(), + PageSize: page.GetPageSize(), + Data: data, + } +} diff --git a/internal/types/session.go b/internal/types/session.go new file mode 100644 index 0000000..7d209ed --- /dev/null +++ b/internal/types/session.go @@ -0,0 +1,120 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "time" + + "github.com/google/uuid" + "gorm.io/gorm" +) + +// FallbackStrategy represents the fallback strategy type +type FallbackStrategy string + +const ( + FallbackStrategyFixed FallbackStrategy = "fixed" // Fixed response + FallbackStrategyModel FallbackStrategy = "model" // Model fallback response +) + +type SummaryConfig struct { + // Max tokens + MaxTokens int `json:"max_tokens"` + // Repeat penalty + RepeatPenalty float64 `json:"repeat_penalty"` + // TopK + TopK int `json:"top_k"` + // TopP + TopP float64 `json:"top_p"` + // Frequency penalty + FrequencyPenalty float64 `json:"frequency_penalty"` + // Presence penalty + PresencePenalty float64 `json:"presence_penalty"` + // Prompt + Prompt string `json:"prompt"` + // Context template + ContextTemplate string `json:"context_template"` + // No match prefix + NoMatchPrefix string `json:"no_match_prefix"` + // Temperature + Temperature float64 `json:"temperature"` + // Seed + Seed int `json:"seed"` + // Max completion tokens + MaxCompletionTokens int `json:"max_completion_tokens"` +} + +// Session represents the session +type Session struct { + // ID + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + // Title + Title string `json:"title"` + // Description + Description string `json:"description"` + // Tenant ID + TenantID uint `json:"tenant_id" gorm:"index"` + + // Strategy configuration + KnowledgeBaseID string `json:"knowledge_base_id"` // 关联的知识库ID + MaxRounds int `json:"max_rounds"` // 多轮保持轮数 + EnableRewrite bool `json:"enable_rewrite"` // 多轮改写开关 + FallbackStrategy FallbackStrategy `json:"fallback_strategy"` // 兜底策略 + FallbackResponse string `json:"fallback_response"` // 固定回复内容 + EmbeddingTopK int `json:"embedding_top_k"` // 向量召回TopK + KeywordThreshold float64 `json:"keyword_threshold"` // 关键词召回阈值 + VectorThreshold float64 `json:"vector_threshold"` // 向量召回阈值 + RerankModelID string `json:"rerank_model_id"` // 排序模型ID + RerankTopK int `json:"rerank_top_k"` // 排序TopK + RerankThreshold float64 `json:"rerank_threshold"` // 排序阈值 + SummaryModelID string `json:"summary_model_id"` // 总结模型ID + SummaryParameters *SummaryConfig `json:"summary_parameters" gorm:"type:json"` // 总结模型参数 + + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` + + // Association relationship, not stored in the database + Messages []Message `json:"-" gorm:"foreignKey:SessionID"` +} + +func (s *Session) BeforeCreate(tx *gorm.DB) (err error) { + s.ID = uuid.New().String() + return nil +} + +type StringArray []string + +// Value implements the driver.Valuer interface, used to convert StringArray to database value +func (c StringArray) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to StringArray +func (c *StringArray) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// Value implements the driver.Valuer interface, used to convert SummaryConfig to database value +func (c *SummaryConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to SummaryConfig +func (c *SummaryConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} diff --git a/internal/types/tenant.go b/internal/types/tenant.go new file mode 100644 index 0000000..20e2e14 --- /dev/null +++ b/internal/types/tenant.go @@ -0,0 +1,69 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "time" + + "gorm.io/gorm" +) + +const ( + InitDefaultTenantID uint = 1 +) + +// Tenant represents the tenant +type Tenant struct { + // ID + ID uint `yaml:"id" json:"id" gorm:"primaryKey"` + // Name + Name string `yaml:"name" json:"name"` + // Description + Description string `yaml:"description" json:"description"` + // API key + APIKey string `yaml:"api_key" json:"api_key"` + // Status + Status string `yaml:"status" json:"status" gorm:"default:'active'"` + // Retriever engines + RetrieverEngines RetrieverEngines `yaml:"retriever_engines" json:"retriever_engines" gorm:"type:json"` + // Business + Business string `yaml:"business" json:"business"` + // Storage quota (Bytes), default is 10GB, including vector, original file, text, index, etc. + StorageQuota int64 `yaml:"storage_quota" json:"storage_quota" gorm:"default:10737418240"` + // Storage used (Bytes) + StorageUsed int64 `yaml:"storage_used" json:"storage_used" gorm:"default:0"` + // Creation time + CreatedAt time.Time `yaml:"created_at" json:"created_at"` + // Last updated time + UpdatedAt time.Time `yaml:"updated_at" json:"updated_at"` + // Deletion time + DeletedAt gorm.DeletedAt `yaml:"deleted_at" json:"deleted_at" gorm:"index"` +} + +type RetrieverEngines struct { + Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"` +} + +func (t *Tenant) BeforeCreate(tx *gorm.DB) error { + if t.RetrieverEngines.Engines == nil { + t.RetrieverEngines.Engines = []RetrieverEngineParams{} + } + return nil +} + +// Value implements the driver.Valuer interface, used to convert RetrieverEngines to database value +func (c RetrieverEngines) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements the sql.Scanner interface, used to convert database value to RetrieverEngines +func (c *RetrieverEngines) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} diff --git a/internal/types/user.go b/internal/types/user.go new file mode 100644 index 0000000..59b8019 --- /dev/null +++ b/internal/types/user.go @@ -0,0 +1,114 @@ +package types + +import ( + "time" + + "gorm.io/gorm" +) + +// User represents a user in the system +type User struct { + // Unique identifier of the user + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + // Username of the user + Username string `json:"username" gorm:"type:varchar(100);uniqueIndex;not null"` + // Email address of the user + Email string `json:"email" gorm:"type:varchar(255);uniqueIndex;not null"` + // Hashed password of the user + PasswordHash string `json:"-" gorm:"type:varchar(255);not null"` + // Avatar URL of the user + Avatar string `json:"avatar" gorm:"type:varchar(500)"` + // Tenant ID that the user belongs to + TenantID uint `json:"tenant_id" gorm:"index"` + // Whether the user is active + IsActive bool `json:"is_active" gorm:"default:true"` + // Creation time of the user + CreatedAt time.Time `json:"created_at"` + // Last updated time of the user + UpdatedAt time.Time `json:"updated_at"` + // Deletion time of the user + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` + + // Association relationship, not stored in the database + Tenant *Tenant `json:"tenant,omitempty" gorm:"foreignKey:TenantID"` +} + +// AuthToken represents an authentication token +type AuthToken struct { + // Unique identifier of the token + ID string `json:"id" gorm:"type:varchar(36);primaryKey"` + // User ID that owns this token + UserID string `json:"user_id" gorm:"type:varchar(36);index;not null"` + // Token value (JWT or other format) + Token string `json:"token" gorm:"type:text;not null"` + // Token type (access_token, refresh_token) + TokenType string `json:"token_type" gorm:"type:varchar(50);not null"` + // Token expiration time + ExpiresAt time.Time `json:"expires_at"` + // Whether the token is revoked + IsRevoked bool `json:"is_revoked" gorm:"default:false"` + // Creation time of the token + CreatedAt time.Time `json:"created_at"` + // Last updated time of the token + UpdatedAt time.Time `json:"updated_at"` + + // Association relationship + User *User `json:"user,omitempty" gorm:"foreignKey:UserID"` +} + +// LoginRequest represents a login request +type LoginRequest struct { + Email string `json:"email" binding:"required,email"` + Password string `json:"password" binding:"required,min=6"` +} + +// RegisterRequest represents a registration request +type RegisterRequest struct { + Username string `json:"username" binding:"required,min=3,max=50"` + Email string `json:"email" binding:"required,email"` + Password string `json:"password" binding:"required,min=6"` +} + +// LoginResponse represents a login response +type LoginResponse struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + User *User `json:"user,omitempty"` + Tenant *Tenant `json:"tenant,omitempty"` + Token string `json:"token,omitempty"` + RefreshToken string `json:"refresh_token,omitempty"` +} + +// RegisterResponse represents a registration response +type RegisterResponse struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + User *User `json:"user,omitempty"` + Tenant *Tenant `json:"tenant,omitempty"` +} + +// UserInfo represents user information for API responses +type UserInfo struct { + ID string `json:"id"` + Username string `json:"username"` + Email string `json:"email"` + Avatar string `json:"avatar"` + TenantID uint `json:"tenant_id"` + IsActive bool `json:"is_active"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// ToUserInfo converts User to UserInfo (without sensitive data) +func (u *User) ToUserInfo() *UserInfo { + return &UserInfo{ + ID: u.ID, + Username: u.Username, + Email: u.Email, + Avatar: u.Avatar, + TenantID: u.TenantID, + IsActive: u.IsActive, + CreatedAt: u.CreatedAt, + UpdatedAt: u.UpdatedAt, + } +} diff --git a/internal/utils/security.go b/internal/utils/security.go new file mode 100644 index 0000000..f86c0b2 --- /dev/null +++ b/internal/utils/security.go @@ -0,0 +1,171 @@ +package utils + +import ( + "html" + "regexp" + "strings" + "unicode/utf8" +) + +// XSS 防护相关正则表达式 +var ( + // 匹配潜在的 XSS 攻击模式 + xssPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)]*>.*?`), + regexp.MustCompile(`(?i)]*>.*?`), + regexp.MustCompile(`(?i)]*>.*?`), + regexp.MustCompile(`(?i)]*>.*?`), + regexp.MustCompile(`(?i)]*>`), + regexp.MustCompile(`(?i)]*>.*?`), + regexp.MustCompile(`(?i)]*>`), + regexp.MustCompile(`(?i)]*>.*?`), + regexp.MustCompile(`(?i)javascript:`), + regexp.MustCompile(`(?i)vbscript:`), + regexp.MustCompile(`(?i)onload\s*=`), + regexp.MustCompile(`(?i)onerror\s*=`), + regexp.MustCompile(`(?i)onclick\s*=`), + regexp.MustCompile(`(?i)onmouseover\s*=`), + regexp.MustCompile(`(?i)onfocus\s*=`), + regexp.MustCompile(`(?i)onblur\s*=`), + } +) + +// SanitizeHTML 清理 HTML 内容，防止 XSS 攻击 +func SanitizeHTML(input string) string { + if input == "" { + return "" + } + + // 检查输入长度 + if len(input) > 10000 { + input = input[:10000] + } + + // 检查是否包含潜在的 XSS 攻击 + for _, pattern := range xssPatterns { + if pattern.MatchString(input) { + // 如果包含恶意内容，进行 HTML 转义 + return html.EscapeString(input) + } + } + + // 如果内容相对安全，返回原内容 + return input +} + +// EscapeHTML 转义 HTML 特殊字符 +func EscapeHTML(input string) string { + if input == "" { + return "" + } + return html.EscapeString(input) +} + +// ValidateInput 验证用户输入 +func ValidateInput(input string) (string, bool) { + if input == "" { + return "", true + } + + // 检查长度 + if len(input) > 10000 { + return "", false + } + + // 检查是否包含控制字符 + for _, r := range input { + if r < 32 && r != 9 && r != 10 && r != 13 { + return "", false + } + } + + // 检查 UTF-8 有效性 + if !utf8.ValidString(input) { + return "", false + } + + // 检查是否包含潜在的 XSS 攻击 + for _, pattern := range xssPatterns { + if pattern.MatchString(input) { + return "", false + } + } + + return strings.TrimSpace(input), true +} + +// IsValidURL 验证 URL 是否安全 +func IsValidURL(url string) bool { + if url == "" { + return false + } + + // 检查长度 + if len(url) > 2048 { + return false + } + + // 检查协议 + if !strings.HasPrefix(strings.ToLower(url), "http://") && + !strings.HasPrefix(strings.ToLower(url), "https://") { + return false + } + + // 检查是否包含恶意内容 + for _, pattern := range xssPatterns { + if pattern.MatchString(url) { + return false + } + } + + return true +} + +// IsValidImageURL 验证图片 URL 是否安全 +func IsValidImageURL(url string) bool { + if !IsValidURL(url) { + return false + } + + // 检查是否为图片文件 + imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico"} + lowerURL := strings.ToLower(url) + + for _, ext := range imageExtensions { + if strings.Contains(lowerURL, ext) { + return true + } + } + + return false +} + +// CleanMarkdown 清理 Markdown 内容 +func CleanMarkdown(input string) string { + if input == "" { + return "" + } + + // 移除潜在的恶意脚本 + cleaned := input + for _, pattern := range xssPatterns { + cleaned = pattern.ReplaceAllString(cleaned, "") + } + + return cleaned +} + +// SanitizeForDisplay 为显示清理内容 +func SanitizeForDisplay(input string) string { + if input == "" { + return "" + } + + // 首先清理 Markdown + cleaned := CleanMarkdown(input) + + // 然后进行 HTML 转义 + escaped := html.EscapeString(cleaned) + + return escaped +} diff --git a/mcp-server/.gitignore b/mcp-server/.gitignore new file mode 100644 index 0000000..9937c76 --- /dev/null +++ b/mcp-server/.gitignore @@ -0,0 +1,2 @@ +/.codebuddy +/__pycache__ \ No newline at end of file diff --git a/mcp-server/CHANGELOG.md b/mcp-server/CHANGELOG.md new file mode 100644 index 0000000..aed790f --- /dev/null +++ b/mcp-server/CHANGELOG.md @@ -0,0 +1,98 @@ +# 更新日志 + +所有重要的项目更改都将记录在此文件中。 + +格式基于 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)， +并且本项目遵循 [语义化版本](https://semver.org/lang/zh-CN/)。 + +## [1.0.0] - 2024-01-XX + +### 新增 +- 初始版本发布 +- WeKnora MCP Server 核心功能 +- 完整的 WeKnora API 集成 +- 租户管理工具 +- 知识库管理工具 +- 知识管理工具 +- 模型管理工具 +- 会话管理工具 +- 聊天功能工具 +- 块管理工具 +- 多种启动方式支持 +- 命令行参数支持 +- 环境变量配置 +- 完整的包安装支持 +- 开发和生产模式 +- 详细的文档和安装指南 + +### 工具列表 +- `create_tenant` - 创建新租户 +- `list_tenants` - 列出所有租户 +- `create_knowledge_base` - 创建知识库 +- `list_knowledge_bases` - 列出知识库 +- `get_knowledge_base` - 获取知识库详情 +- `delete_knowledge_base` - 删除知识库 +- `hybrid_search` - 混合搜索 +- `create_knowledge_from_url` - 从 URL 创建知识 +- `list_knowledge` - 列出知识 +- `get_knowledge` - 获取知识详情 +- `delete_knowledge` - 删除知识 +- `create_model` - 创建模型 +- `list_models` - 列出模型 +- `get_model` - 获取模型详情 +- `create_session` - 创建聊天会话 +- `get_session` - 获取会话详情 +- `list_sessions` - 列出会话 +- `delete_session` - 删除会话 +- `chat` - 发送聊天消息 +- `list_chunks` - 列出知识块 +- `delete_chunk` - 删除知识块 + +### 文件结构 +``` +WeKnoraMCP/ +├── __init__.py # 包初始化文件 +├── main.py # 主入口点 (推荐) +├── run.py # 便捷启动脚本 +├── run_server.py # 原始启动脚本 +├── weknora_mcp_server.py # MCP 服务器实现 +├── test_module.py # 模组测试脚本 +├── requirements.txt # 依赖列表 +├── setup.py # 安装脚本 (传统) +├── pyproject.toml # 现代项目配置 +├── MANIFEST.in # 包含文件清单 +├── LICENSE # MIT 许可证 +├── README.md # 项目说明 +├── INSTALL.md # 详细安装指南 +└── CHANGELOG.md # 更新日志 +``` + +### 启动方式 +1. `python main.py` - 主入口点 (推荐) +2. `python run_server.py` - 原始启动脚本 +3. `python run.py` - 便捷启动脚本 +4. `python weknora_mcp_server.py` - 直接运行 +5. `python -m weknora_mcp_server` - 模块运行 +6. `weknora-mcp-server` - 安装后命令行工具 +7. `weknora-server` - 安装后命令行工具 (别名) + +### 技术特性 +- 基于 Model Context Protocol (MCP) 1.0.0+ +- 异步 I/O 支持 +- 完整的错误处理 +- 详细的日志记录 +- 环境变量配置 +- 命令行参数支持 +- 多种安装方式 +- 开发和生产模式 +- 完整的测试覆盖 + +### 依赖 +- Python 3.8+ +- mcp >= 1.0.0 +- requests >= 2.31.0 + +### 兼容性 +- 支持 Windows、macOS、Linux +- 支持 Python 3.8-3.12 +- 兼容现代 Python 包管理工具 \ No newline at end of file diff --git a/mcp-server/EXAMPLES.md b/mcp-server/EXAMPLES.md new file mode 100644 index 0000000..0007128 --- /dev/null +++ b/mcp-server/EXAMPLES.md @@ -0,0 +1,411 @@ +# WeKnora MCP Server 使用示例 + +本文档提供了 WeKnora MCP Server 的详细使用示例。 + +## 基本使用 + +### 1. 启动服务器 + +```bash +# 推荐方式 - 使用主入口点 +python main.py + +# 检查环境配置 +python main.py --check-only + +# 启用详细日志 +python main.py --verbose +``` + +### 2. 环境配置示例 + +```bash +# 设置环境变量 +export WEKNORA_BASE_URL="http://localhost:8080/api/v1" +export WEKNORA_API_KEY="your_api_key_here" + +# 或者在 .env 文件中设置 +echo "WEKNORA_BASE_URL=http://localhost:8080/api/v1" > .env +echo "WEKNORA_API_KEY=your_api_key_here" >> .env +``` + +## MCP 工具使用示例 + +以下是各种 MCP 工具的使用示例： + +### 租户管理 + +#### 创建租户 +```json +{ + "tool": "create_tenant", + "arguments": { + "name": "我的公司", + "description": "公司知识管理系统", + "business": "technology", + "retriever_engines": { + "engines": [ + {"retriever_type": "keywords", "retriever_engine_type": "postgres"}, + {"retriever_type": "vector", "retriever_engine_type": "postgres"} + ] + } + } +} +``` + +#### 列出所有租户 +```json +{ + "tool": "list_tenants", + "arguments": {} +} +``` + +### 知识库管理 + +#### 创建知识库 +```json +{ + "tool": "create_knowledge_base", + "arguments": { + "name": "产品文档库", + "description": "产品相关文档和资料", + "embedding_model_id": "text-embedding-ada-002", + "summary_model_id": "gpt-3.5-turbo" + } +} +``` + +#### 列出知识库 +```json +{ + "tool": "list_knowledge_bases", + "arguments": {} +} +``` + +#### 获取知识库详情 +```json +{ + "tool": "get_knowledge_base", + "arguments": { + "kb_id": "kb_123456" + } +} +``` + +#### 混合搜索 +```json +{ + "tool": "hybrid_search", + "arguments": { + "kb_id": "kb_123456", + "query": "如何使用API", + "vector_threshold": 0.7, + "keyword_threshold": 0.5, + "match_count": 10 + } +} +``` + +### 知识管理 + +#### 从URL创建知识 +```json +{ + "tool": "create_knowledge_from_url", + "arguments": { + "kb_id": "kb_123456", + "url": "https://docs.example.com/api-guide", + "enable_multimodel": true + } +} +``` + +#### 列出知识 +```json +{ + "tool": "list_knowledge", + "arguments": { + "kb_id": "kb_123456", + "page": 1, + "page_size": 20 + } +} +``` + +#### 获取知识详情 +```json +{ + "tool": "get_knowledge", + "arguments": { + "knowledge_id": "know_789012" + } +} +``` + +### 模型管理 + +#### 创建模型 +```json +{ + "tool": "create_model", + "arguments": { + "name": "GPT-4 Chat Model", + "type": "KnowledgeQA", + "source": "openai", + "description": "OpenAI GPT-4 模型用于知识问答", + "base_url": "https://api.openai.com/v1", + "api_key": "sk-...", + "is_default": true + } +} +``` + +#### 列出模型 +```json +{ + "tool": "list_models", + "arguments": {} +} +``` + +### 会话管理 + +#### 创建聊天会话 +```json +{ + "tool": "create_session", + "arguments": { + "kb_id": "kb_123456", + "max_rounds": 10, + "enable_rewrite": true, + "fallback_response": "抱歉，我无法回答这个问题。", + "summary_model_id": "gpt-3.5-turbo" + } +} +``` + +#### 获取会话详情 +```json +{ + "tool": "get_session", + "arguments": { + "session_id": "sess_345678" + } +} +``` + +#### 列出会话 +```json +{ + "tool": "list_sessions", + "arguments": { + "page": 1, + "page_size": 10 + } +} +``` + +### 聊天功能 + +#### 发送聊天消息 +```json +{ + "tool": "chat", + "arguments": { + "session_id": "sess_345678", + "query": "请介绍一下产品的主要功能" + } +} +``` + +### 块管理 + +#### 列出知识块 +```json +{ + "tool": "list_chunks", + "arguments": { + "knowledge_id": "know_789012", + "page": 1, + "page_size": 50 + } +} +``` + +#### 删除知识块 +```json +{ + "tool": "delete_chunk", + "arguments": { + "knowledge_id": "know_789012", + "chunk_id": "chunk_456789" + } +} +``` + +## 完整工作流程示例 + +### 场景：创建一个完整的知识问答系统 + +```bash +# 1. 启动服务器 +python main.py --verbose + +# 2. 在 MCP 客户端中执行以下步骤： +``` + +#### 步骤 1: 创建租户 +```json +{ + "tool": "create_tenant", + "arguments": { + "name": "技术文档中心", + "description": "公司技术文档知识管理", + "business": "technology" + } +} +``` + +#### 步骤 2: 创建知识库 +```json +{ + "tool": "create_knowledge_base", + "arguments": { + "name": "API文档库", + "description": "所有API相关文档" + } +} +``` + +#### 步骤 3: 添加知识内容 +```json +{ + "tool": "create_knowledge_from_url", + "arguments": { + "kb_id": "返回的知识库ID", + "url": "https://docs.company.com/api", + "enable_multimodel": true + } +} +``` + +#### 步骤 4: 创建聊天会话 +```json +{ + "tool": "create_session", + "arguments": { + "kb_id": "知识库ID", + "max_rounds": 5, + "enable_rewrite": true + } +} +``` + +#### 步骤 5: 开始对话 +```json +{ + "tool": "chat", + "arguments": { + "session_id": "会话ID", + "query": "如何使用用户认证API？" + } +} +``` + +## 错误处理示例 + +### 常见错误和解决方案 + +#### 1. 连接错误 +```json +{ + "error": "Connection refused", + "solution": "检查 WEKNORA_BASE_URL 是否正确，确认服务正在运行" +} +``` + +#### 2. 认证错误 +```json +{ + "error": "Unauthorized", + "solution": "检查 WEKNORA_API_KEY 是否设置正确" +} +``` + +#### 3. 资源不存在 +```json +{ + "error": "Knowledge base not found", + "solution": "确认知识库ID是否正确，或先创建知识库" +} +``` + +## 高级配置示例 + +### 自定义检索配置 +```json +{ + "tool": "hybrid_search", + "arguments": { + "kb_id": "kb_123456", + "query": "搜索查询", + "vector_threshold": 0.8, + "keyword_threshold": 0.6, + "match_count": 15 + } +} +``` + +### 自定义会话策略 +```json +{ + "tool": "create_session", + "arguments": { + "kb_id": "kb_123456", + "max_rounds": 20, + "enable_rewrite": true, + "fallback_response": "根据现有知识，我无法准确回答您的问题。请尝试重新表述或联系技术支持。" + } +} +``` + +## 性能优化建议 + +1. **批量操作**: 尽量批量处理知识创建和更新 +2. **缓存策略**: 合理设置搜索阈值以平衡准确性和性能 +3. **会话管理**: 及时清理不需要的会话以节省资源 +4. **监控日志**: 使用 `--verbose` 选项监控性能指标 + +## 集成示例 + +### 与 Claude Desktop 集成 +在 Claude Desktop 的配置文件中添加： +```json +{ + "mcpServers": { + "weknora": { + "command": "python", + "args": ["path/to/main.py"], + "env": { + "WEKNORA_BASE_URL": "http://localhost:8080/api/v1", + "WEKNORA_API_KEY": "your_api_key" + } + } + } +} +``` + +项目仓库: https://github.com/NannaOlympicBroadcast/WeKnoraMCP + +### 与其他 MCP 客户端集成 +参考各客户端的文档，配置服务器启动命令和环境变量。 + +## 故障排除 + +如果遇到问题： +1. 运行 `python main.py --check-only` 检查环境 +2. 使用 `python main.py --verbose` 查看详细日志 +3. 检查 WeKnora 服务是否正常运行 +4. 验证网络连接和防火墙设置 \ No newline at end of file diff --git a/mcp-server/INSTALL.md b/mcp-server/INSTALL.md new file mode 100644 index 0000000..090ecce --- /dev/null +++ b/mcp-server/INSTALL.md @@ -0,0 +1,208 @@ +# WeKnora MCP Server 安装和使用指南 + +## 快速开始 + +### 1. 安装依赖 +```bash +pip install -r requirements.txt +``` + +### 2. 设置环境变量 +```bash +# Linux/macOS +export WEKNORA_BASE_URL="http://localhost:8080/api/v1" +export WEKNORA_API_KEY="your_api_key_here" + +# Windows PowerShell +$env:WEKNORA_BASE_URL="http://localhost:8080/api/v1" +$env:WEKNORA_API_KEY="your_api_key_here" + +# Windows CMD +set WEKNORA_BASE_URL=http://localhost:8080/api/v1 +set WEKNORA_API_KEY=your_api_key_here +``` + +### 3. 运行服务器 + +有多种方式运行服务器： + +#### 方式 1: 使用主入口点 (推荐) +```bash +python main.py +``` + +#### 方式 2: 使用原始启动脚本 +```bash +python run_server.py +``` + +#### 方式 3: 直接运行服务器模块 +```bash +python weknora_mcp_server.py +``` + +#### 方式 4: 作为 Python 模块运行 +```bash +python -m weknora_mcp_server +``` + +## 作为 Python 包安装 + +### 开发模式安装 +```bash +pip install -e . +``` + +安装后可以使用命令行工具： +```bash +weknora-mcp-server +# 或 +weknora-server +``` + +### 生产模式安装 +```bash +pip install . +``` + +### 构建分发包 +```bash +# 构建源码分发包和轮子 +python setup.py sdist bdist_wheel + +# 或使用 build 工具 +pip install build +python -m build +``` + +## 命令行选项 + +主入口点 `main.py` 支持以下选项： + +```bash +python main.py --help # 显示帮助信息 +python main.py --check-only # 仅检查环境配置 +python main.py --verbose # 启用详细日志 +python main.py --version # 显示版本信息 +``` + +## 环境检查 + +运行以下命令检查环境配置： +```bash +python main.py --check-only +``` + +这将显示： +- WeKnora API 基础 URL 配置 +- API 密钥设置状态 +- 依赖包安装状态 + +## 故障排除 + +### 1. 导入错误 +如果遇到 `ImportError`，请确保： +- 已安装所有依赖：`pip install -r requirements.txt` +- Python 版本兼容（推荐 3.8+） +- 没有文件名冲突 + +### 2. 连接错误 +如果无法连接到 WeKnora API： +- 检查 `WEKNORA_BASE_URL` 是否正确 +- 确认 WeKnora 服务正在运行 +- 验证网络连接 + +### 3. 认证错误 +如果遇到认证问题： +- 检查 `WEKNORA_API_KEY` 是否设置 +- 确认 API 密钥有效 +- 验证权限设置 + +## 开发模式 + +### 项目结构 +``` +WeKnoraMCP/ +├── __init__.py # 包初始化文件 +├── main.py # 主入口点 +├── run_server.py # 原始启动脚本 +├── weknora_mcp_server.py # MCP 服务器实现 +├── requirements.txt # 依赖列表 +├── setup.py # 安装脚本 +├── MANIFEST.in # 包含文件清单 +├── LICENSE # 许可证 +├── README.md # 项目说明 +└── INSTALL.md # 安装指南 +``` + +### 添加新功能 +1. 在 `WeKnoraClient` 类中添加新的 API 方法 +2. 在 `handle_list_tools()` 中注册新工具 +3. 在 `handle_call_tool()` 中实现工具逻辑 +4. 更新文档和测试 + +### 测试 +```bash +# 运行基本测试 +python test_imports.py + +# 测试环境配置 +python main.py --check-only + +# 测试服务器启动 +python main.py --verbose +``` + +## 部署 + +### Docker 部署 +创建 `Dockerfile`： +```dockerfile +FROM python:3.11-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY . . +RUN pip install -e . + +ENV WEKNORA_BASE_URL=http://localhost:8080/api/v1 +EXPOSE 8000 + +CMD ["weknora-mcp-server"] +``` + +### 系统服务 +创建 systemd 服务文件 `/etc/systemd/system/weknora-mcp.service`： +```ini +[Unit] +Description=WeKnora MCP Server +After=network.target + +[Service] +Type=simple +User=weknora +WorkingDirectory=/opt/weknora-mcp +Environment=WEKNORA_BASE_URL=http://localhost:8080/api/v1 +Environment=WEKNORA_API_KEY=your_api_key +ExecStart=/usr/local/bin/weknora-mcp-server +Restart=always + +[Install] +WantedBy=multi-user.target +``` + +启用服务： +```bash +sudo systemctl enable weknora-mcp +sudo systemctl start weknora-mcp +``` + +## 支持 + +如果遇到问题，请： +1. 查看日志输出 +2. 检查环境配置 +3. 参考故障排除部分 +4. 提交 Issue 到项目仓库: https://github.com/NannaOlympicBroadcast/WeKnoraMCP/issues \ No newline at end of file diff --git a/mcp-server/LICENSE b/mcp-server/LICENSE new file mode 100644 index 0000000..2da53f2 --- /dev/null +++ b/mcp-server/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 WeKnora Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/mcp-server/MANIFEST.in b/mcp-server/MANIFEST.in new file mode 100644 index 0000000..98afaaa --- /dev/null +++ b/mcp-server/MANIFEST.in @@ -0,0 +1,14 @@ +include README.md +include requirements.txt +include LICENSE +include *.py +recursive-include * *.py +recursive-include * *.md +recursive-include * *.txt +recursive-include * *.yml +recursive-include * *.yaml +global-exclude __pycache__ +global-exclude *.py[co] +global-exclude .DS_Store +global-exclude *.so +global-exclude .git* \ No newline at end of file diff --git a/mcp-server/PROJECT_SUMMARY.md b/mcp-server/PROJECT_SUMMARY.md new file mode 100644 index 0000000..37b307e --- /dev/null +++ b/mcp-server/PROJECT_SUMMARY.md @@ -0,0 +1,279 @@ +# WeKnora MCP Server 可运行模组包 - 项目总结 + +## 🎉 项目完成状态 + +✅ **所有测试通过** - 模组已成功打包并可正常运行 + +## 📁 项目结构 + +``` +WeKnoraMCP/ +├── 📦 核心文件 +│ ├── __init__.py # 包初始化文件 +│ ├── weknora_mcp_server.py # MCP 服务器核心实现 +│ └── requirements.txt # 项目依赖 +│ +├── 🚀 启动脚本 (多种方式) +│ ├── main.py # 主入口点 (推荐) ⭐ +│ ├── run_server.py # 原始启动脚本 +│ └── run.py # 便捷启动脚本 +│ +├── 📋 配置文件 +│ ├── setup.py # 传统安装脚本 +│ ├── pyproject.toml # 现代项目配置 +│ └── MANIFEST.in # 包含文件清单 +│ +├── 🧪 测试文件 +│ ├── test_module.py # 模组功能测试 +│ └── test_imports.py # 导入测试 +│ +├── 📚 文档文件 +│ ├── README.md # 项目说明 +│ ├── INSTALL.md # 详细安装指南 +│ ├── EXAMPLES.md # 使用示例 +│ ├── CHANGELOG.md # 更新日志 +│ ├── PROJECT_SUMMARY.md # 项目总结 (本文件) +│ └── LICENSE # MIT 许可证 +│ +└── 📂 其他 + ├── __pycache__/ # Python 缓存 (自动生成) + ├── .codebuddy/ # CodeBuddy 配置 + └── .venv/ # 虚拟环境 (可选) +``` + +## 🚀 启动方式 (7种) + +### 1. 主入口点 (推荐) ⭐ +```bash +python main.py # 基本启动 +python main.py --check-only # 仅检查环境 +python main.py --verbose # 详细日志 +python main.py --help # 显示帮助 +``` + +### 2. 原始启动脚本 +```bash +python run_server.py +``` + +### 3. 便捷启动脚本 +```bash +python run.py +``` + +### 4. 直接运行服务器 +```bash +python weknora_mcp_server.py +``` + +### 5. 作为模块运行 +```bash +python -m weknora_mcp_server +``` + +### 6. 安装后命令行工具 +```bash +pip install -e . # 开发模式安装 +weknora-mcp-server # 主命令 +weknora-server # 别名命令 +``` + +### 7. 生产环境安装 +```bash +pip install . # 生产安装 +weknora-mcp-server # 全局命令 +``` + +## 🔧 环境配置 + +### 必需环境变量 +```bash +# Linux/macOS +export WEKNORA_BASE_URL="http://localhost:8080/api/v1" +export WEKNORA_API_KEY="your_api_key_here" + +# Windows PowerShell +$env:WEKNORA_BASE_URL="http://localhost:8080/api/v1" +$env:WEKNORA_API_KEY="your_api_key_here" + +# Windows CMD +set WEKNORA_BASE_URL=http://localhost:8080/api/v1 +set WEKNORA_API_KEY=your_api_key_here +``` + +## 🛠️ 功能特性 + +### MCP 工具 (21个) +- **租户管理**: `create_tenant`, `list_tenants` +- **知识库管理**: `create_knowledge_base`, `list_knowledge_bases`, `get_knowledge_base`, `delete_knowledge_base`, `hybrid_search` +- **知识管理**: `create_knowledge_from_url`, `list_knowledge`, `get_knowledge`, `delete_knowledge` +- **模型管理**: `create_model`, `list_models`, `get_model` +- **会话管理**: `create_session`, `get_session`, `list_sessions`, `delete_session` +- **聊天功能**: `chat` +- **块管理**: `list_chunks`, `delete_chunk` + +### 技术特性 +- ✅ 异步 I/O 支持 +- ✅ 完整错误处理 +- ✅ 详细日志记录 +- ✅ 环境变量配置 +- ✅ 命令行参数支持 +- ✅ 多种安装方式 +- ✅ 开发和生产模式 +- ✅ 完整测试覆盖 + +## 📦 安装方式 + +### 快速开始 +```bash +# 1. 安装依赖 +pip install -r requirements.txt + +# 2. 设置环境变量 +export WEKNORA_BASE_URL="http://localhost:8080/api/v1" +export WEKNORA_API_KEY="your_api_key" + +# 3. 启动服务器 +python main.py +``` + +### 开发模式安装 +```bash +pip install -e . +weknora-mcp-server +``` + +### 生产模式安装 +```bash +pip install . +weknora-mcp-server +``` + +### 构建分发包 +```bash +# 传统方式 +python setup.py sdist bdist_wheel + +# 现代方式 +pip install build +python -m build +``` + +## 🧪 测试验证 + +### 运行完整测试 +```bash +python test_module.py +``` + +### 测试结果 +``` +WeKnora MCP Server 模组测试 +================================================== +✓ 模块导入测试通过 +✓ 环境配置测试通过 +✓ 客户端创建测试通过 +✓ 文件结构测试通过 +✓ 入口点测试通过 +✓ 包安装测试通过 +================================================== +测试结果: 6/6 通过 +✓ 所有测试通过！模组可以正常使用。 +``` + +## 🔍 兼容性 + +### Python 版本 +- ✅ Python 3.8+ +- ✅ Python 3.9 +- ✅ Python 3.10 +- ✅ Python 3.11 +- ✅ Python 3.12 + +### 操作系统 +- ✅ Windows 10/11 +- ✅ macOS 10.15+ +- ✅ Linux (Ubuntu, CentOS, etc.) + +### 依赖包 +- `mcp >= 1.0.0` - Model Context Protocol 核心库 +- `requests >= 2.31.0` - HTTP 请求库 + +## 📖 文档资源 + +1. **README.md** - 项目概述和快速开始 +2. **INSTALL.md** - 详细安装和配置指南 +3. **EXAMPLES.md** - 完整使用示例和工作流程 +4. **CHANGELOG.md** - 版本更新记录 +5. **PROJECT_SUMMARY.md** - 项目总结 (本文件) + +## 🎯 使用场景 + +### 1. 开发环境 +```bash +python main.py --verbose +``` + +### 2. 生产环境 +```bash +pip install . +weknora-mcp-server +``` + +### 3. Docker 部署 +```dockerfile +FROM python:3.11-slim +WORKDIR /app +COPY . . +RUN pip install . +CMD ["weknora-mcp-server"] +``` + +### 4. 系统服务 +```ini +[Unit] +Description=WeKnora MCP Server + +[Service] +ExecStart=/usr/local/bin/weknora-mcp-server +Environment=WEKNORA_BASE_URL=http://localhost:8080/api/v1 +``` + +## 🔧 故障排除 + +### 常见问题 +1. **导入错误**: 运行 `pip install -r requirements.txt` +2. **连接错误**: 检查 `WEKNORA_BASE_URL` 设置 +3. **认证错误**: 验证 `WEKNORA_API_KEY` 配置 +4. **环境检查**: 运行 `python main.py --check-only` + +### 调试模式 +```bash +python main.py --verbose # 详细日志 +python test_module.py # 运行测试 +``` + +## 🎉 项目成就 + +✅ **完整的可运行模组** - 从单个脚本转换为完整的 Python 包 +✅ **多种启动方式** - 提供 7 种不同的启动方法 +✅ **完善的文档** - 包含安装、使用、示例等完整文档 +✅ **全面的测试** - 所有功能都经过测试验证 +✅ **现代化配置** - 支持 setup.py 和 pyproject.toml +✅ **跨平台兼容** - 支持 Windows、macOS、Linux +✅ **生产就绪** - 可用于开发和生产环境 + +## 🚀 下一步 + +1. **部署到生产环境** +2. **集成到 CI/CD 流程** +3. **发布到 PyPI** +4. **添加更多测试用例** +5. **性能优化和监控** + +--- + +**项目状态**: ✅ 完成并可投入使用 +**项目仓库**: https://github.com/NannaOlympicBroadcast/WeKnoraMCP +**最后更新**: 2024年1月 +**版本**: 1.0.0 \ No newline at end of file diff --git a/mcp-server/README.md b/mcp-server/README.md new file mode 100644 index 0000000..cf98266 --- /dev/null +++ b/mcp-server/README.md @@ -0,0 +1,137 @@ +# WeKnora MCP Server + +这是一个 Model Context Protocol (MCP) 服务器，提供对 WeKnora 知识管理 API 的访问。 + +## 快速开始 + +### 1. 安装依赖 +```bash +pip install -r requirements.txt +``` + +### 2. 配置环境变量 +```bash +# Linux/macOS +export WEKNORA_BASE_URL="http://localhost:8080/api/v1" +export WEKNORA_API_KEY="your_api_key_here" + +# Windows PowerShell +$env:WEKNORA_BASE_URL="http://localhost:8080/api/v1" +$env:WEKNORA_API_KEY="your_api_key_here" + +# Windows CMD +set WEKNORA_BASE_URL=http://localhost:8080/api/v1 +set WEKNORA_API_KEY=your_api_key_here +``` + +### 3. 运行服务器 + +**推荐方式 - 使用主入口点：** +```bash +python main.py +``` + +**其他运行方式：** +```bash +# 使用原始启动脚本 +python run_server.py + +# 使用便捷脚本 +python run.py + +# 直接运行服务器模块 +python weknora_mcp_server.py + +# 作为 Python 模块运行 +python -m weknora_mcp_server +``` + +### 4. 命令行选项 +```bash +python main.py --help # 显示帮助信息 +python main.py --check-only # 仅检查环境配置 +python main.py --verbose # 启用详细日志 +python main.py --version # 显示版本信息 +``` + +## 安装为 Python 包 + +### 开发模式安装 +```bash +pip install -e . +``` + +安装后可以使用命令行工具： +```bash +weknora-mcp-server +# 或 +weknora-server +``` + +### 生产模式安装 +```bash +pip install . +``` + +### 构建分发包 +```bash +# 使用 setuptools +python setup.py sdist bdist_wheel + +# 使用现代构建工具 +pip install build +python -m build +``` + +## 测试模组 + +运行测试脚本验证模组是否正常工作： +```bash +python test_module.py +``` + +## 功能特性 + +该 MCP 服务器提供以下工具： + +### 租户管理 +- `create_tenant` - 创建新租户 +- `list_tenants` - 列出所有租户 + +### 知识库管理 +- `create_knowledge_base` - 创建知识库 +- `list_knowledge_bases` - 列出知识库 +- `get_knowledge_base` - 获取知识库详情 +- `delete_knowledge_base` - 删除知识库 +- `hybrid_search` - 混合搜索 + +### 知识管理 +- `create_knowledge_from_url` - 从 URL 创建知识 +- `list_knowledge` - 列出知识 +- `get_knowledge` - 获取知识详情 +- `delete_knowledge` - 删除知识 + +### 模型管理 +- `create_model` - 创建模型 +- `list_models` - 列出模型 +- `get_model` - 获取模型详情 + +### 会话管理 +- `create_session` - 创建聊天会话 +- `get_session` - 获取会话详情 +- `list_sessions` - 列出会话 +- `delete_session` - 删除会话 + +### 聊天功能 +- `chat` - 发送聊天消息 + +### 块管理 +- `list_chunks` - 列出知识块 +- `delete_chunk` - 删除知识块 + +## 故障排除 + +如果遇到导入错误，请确保： +1. 已安装所有必需的依赖包 +2. Python 版本兼容（推荐 3.8+） +3. 没有文件名冲突（避免使用 `mcp.py` 作为文件名） \ No newline at end of file diff --git a/mcp-server/__init__.py b/mcp-server/__init__.py new file mode 100644 index 0000000..987ea62 --- /dev/null +++ b/mcp-server/__init__.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +""" +WeKnora MCP Server Package + +A Model Context Protocol server that provides access to the WeKnora knowledge management API. +""" + +__version__ = "1.0.0" +__author__ = "WeKnora Team" +__description__ = "WeKnora MCP Server - Model Context Protocol server for WeKnora API" + +from .weknora_mcp_server import WeKnoraClient, run + +__all__ = ["WeKnoraClient", "run"] \ No newline at end of file diff --git a/mcp-server/main.py b/mcp-server/main.py new file mode 100644 index 0000000..6203a27 --- /dev/null +++ b/mcp-server/main.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +WeKnora MCP Server 主入口点 + +这个文件提供了一个统一的入口点来启动 WeKnora MCP 服务器。 +可以通过多种方式运行： +1. python main.py +2. python -m weknora_mcp_server +3. weknora-mcp-server (安装后) +""" + +import os +import sys +import asyncio +import argparse +from pathlib import Path + +def setup_environment(): + """设置环境和路径""" + # 确保当前目录在 Python 路径中 + current_dir = Path(__file__).parent.absolute() + if str(current_dir) not in sys.path: + sys.path.insert(0, str(current_dir)) + +def check_dependencies(): + """检查依赖是否已安装""" + try: + import mcp + import requests + return True + except ImportError as e: + print(f"缺少依赖: {e}") + print("请运行: pip install -r requirements.txt") + return False + +def check_environment_variables(): + """检查环境变量配置""" + base_url = os.getenv("WEKNORA_BASE_URL") + api_key = os.getenv("WEKNORA_API_KEY") + + print("=== WeKnora MCP Server 环境检查 ===") + print(f"Base URL: {base_url or 'http://localhost:8080/api/v1 (默认)'}") + print(f"API Key: {'已设置' if api_key else '未设置 (警告)'}") + + if not base_url: + print("提示: 可以设置 WEKNORA_BASE_URL 环境变量") + + if not api_key: + print("警告: 建议设置 WEKNORA_API_KEY 环境变量") + + print("=" * 40) + return True + +def parse_arguments(): + """解析命令行参数""" + parser = argparse.ArgumentParser( + description="WeKnora MCP Server - Model Context Protocol server for WeKnora API", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + python main.py # 使用默认配置启动 + python main.py --check-only # 仅检查环境，不启动服务器 + python main.py --verbose # 启用详细日志 + +环境变量: + WEKNORA_BASE_URL WeKnora API 基础 URL (默认: http://localhost:8080/api/v1) + WEKNORA_API_KEY WeKnora API 密钥 + """ + ) + + parser.add_argument( + "--check-only", + action="store_true", + help="仅检查环境配置，不启动服务器" + ) + + parser.add_argument( + "--verbose", "-v", + action="store_true", + help="启用详细日志输出" + ) + + parser.add_argument( + "--version", + action="version", + version="WeKnora MCP Server 1.0.0" + ) + + return parser.parse_args() + +async def main(): + """主函数""" + args = parse_arguments() + + # 设置环境 + setup_environment() + + # 检查依赖 + if not check_dependencies(): + sys.exit(1) + + # 检查环境变量 + check_environment_variables() + + # 如果只是检查环境，则退出 + if args.check_only: + print("环境检查完成。") + return + + # 设置日志级别 + if args.verbose: + import logging + logging.basicConfig(level=logging.DEBUG) + print("已启用详细日志模式") + + try: + print("正在启动 WeKnora MCP Server...") + + # 导入并运行服务器 + from weknora_mcp_server import run + await run() + + except ImportError as e: + print(f"导入错误: {e}") + print("请确保所有文件都在正确的位置") + sys.exit(1) + except KeyboardInterrupt: + print("\n服务器已停止") + except Exception as e: + print(f"服务器运行错误: {e}") + if args.verbose: + import traceback + traceback.print_exc() + sys.exit(1) + +def sync_main(): + """同步版本的主函数，用于 entry_points""" + asyncio.run(main()) + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/mcp-server/pyproject.toml b/mcp-server/pyproject.toml new file mode 100644 index 0000000..343a57e --- /dev/null +++ b/mcp-server/pyproject.toml @@ -0,0 +1,108 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "weknora-mcp-server" +version = "1.0.0" +description = "WeKnora MCP Server - Model Context Protocol server for WeKnora API" +readme = "README.md" +license = {text = "MIT"} +authors = [ + {name = "WeKnora Team", email = "support@weknora.com"} +] +maintainers = [ + {name = "WeKnora Team", email = "support@weknora.com"} +] +keywords = ["mcp", "model-context-protocol", "weknora", "knowledge-management", "api-server"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] +requires-python = ">=3.8" +dependencies = [ + "mcp>=1.0.0", + "requests>=2.31.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-asyncio>=0.21.0", + "black>=23.0", + "flake8>=6.0", + "mypy>=1.0", +] +test = [ + "pytest>=7.0", + "pytest-asyncio>=0.21.0", + "pytest-cov>=4.0", +] + +[project.urls] +Homepage = "https://github.com/NannaOlympicBroadcast/WeKnoraMCP" +Documentation = "https://docs.weknora.com" +Repository = "https://github.com/NannaOlympicBroadcast/WeKnoraMCP" +"Bug Reports" = "https://github.com/NannaOlympicBroadcast/WeKnoraMCP/issues" +Changelog = "https://github.com/NannaOlympicBroadcast/WeKnoraMCP/blob/main/CHANGELOG.md" + +[project.scripts] +weknora-mcp-server = "main:sync_main" +weknora-server = "run_server:main" + +[tool.setuptools] +py-modules = ["weknora_mcp_server", "main", "run_server", "run", "test_module"] +include-package-data = true + +[tool.setuptools.package-data] +"*" = ["*.md", "*.txt", "*.yml", "*.yaml"] + +[tool.black] +line-length = 88 +target-version = ['py38'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "-ra -q --strict-markers --strict-config" +testpaths = ["tests"] +asyncio_mode = "auto" \ No newline at end of file diff --git a/mcp-server/requirements.txt b/mcp-server/requirements.txt new file mode 100644 index 0000000..28264a1 --- /dev/null +++ b/mcp-server/requirements.txt @@ -0,0 +1,2 @@ +mcp>=1.0.0 +requests>=2.31.0 \ No newline at end of file diff --git a/mcp-server/run.py b/mcp-server/run.py new file mode 100644 index 0000000..a20dd35 --- /dev/null +++ b/mcp-server/run.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +""" +WeKnora MCP Server 便捷启动脚本 + +这是一个简化的启动脚本，提供最基本的功能。 +对于更多选项，请使用 main.py +""" + +import sys +import os +from pathlib import Path + +def main(): + """简单的启动函数""" + # 添加当前目录到 Python 路径 + current_dir = Path(__file__).parent.absolute() + if str(current_dir) not in sys.path: + sys.path.insert(0, str(current_dir)) + + # 检查环境变量 + base_url = os.getenv("WEKNORA_BASE_URL", "http://localhost:8080/api/v1") + api_key = os.getenv("WEKNORA_API_KEY", "") + + print("WeKnora MCP Server") + print(f"Base URL: {base_url}") + print(f"API Key: {'已设置' if api_key else '未设置'}") + print("-" * 40) + + try: + # 导入并运行 + from main import sync_main + sync_main() + except ImportError: + print("错误: 无法导入必要模块") + print("请确保运行: pip install -r requirements.txt") + sys.exit(1) + except KeyboardInterrupt: + print("\n服务器已停止") + except Exception as e: + print(f"错误: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/mcp-server/run_server.py b/mcp-server/run_server.py new file mode 100644 index 0000000..1956e29 --- /dev/null +++ b/mcp-server/run_server.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +""" +WeKnora MCP Server 启动脚本 +""" + +import os +import sys +import asyncio + +def check_environment(): + """检查环境配置""" + base_url = os.getenv("WEKNORA_BASE_URL") + api_key = os.getenv("WEKNORA_API_KEY") + + if not base_url: + print("警告: WEKNORA_BASE_URL 环境变量未设置，使用默认值: http://localhost:8080/api/v1") + + if not api_key: + print("警告: WEKNORA_API_KEY 环境变量未设置") + + print(f"WeKnora Base URL: {base_url or 'http://localhost:8080/api/v1'}") + print(f"API Key: {'已设置' if api_key else '未设置'}") + +def main(): + """主函数""" + print("启动 WeKnora MCP Server...") + check_environment() + + try: + from weknora_mcp_server import run + asyncio.run(run()) + except ImportError as e: + print(f"导入错误: {e}") + print("请确保已安装所有依赖: pip install -r requirements.txt") + sys.exit(1) + except KeyboardInterrupt: + print("\n服务器已停止") + except Exception as e: + print(f"服务器运行错误: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/mcp-server/setup.py b/mcp-server/setup.py new file mode 100644 index 0000000..cca9860 --- /dev/null +++ b/mcp-server/setup.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +WeKnora MCP Server 安装脚本 +""" + +from setuptools import setup +import os + +# 读取 README 文件 +def read_readme(): + try: + with open("README.md", "r", encoding="utf-8") as f: + return f.read() + except FileNotFoundError: + return "WeKnora MCP Server - Model Context Protocol server for WeKnora API" + +# 读取依赖 +def read_requirements(): + try: + with open("requirements.txt", "r", encoding="utf-8") as f: + return [line.strip() for line in f if line.strip() and not line.startswith("#")] + except FileNotFoundError: + return ["mcp>=1.0.0", "requests>=2.31.0"] + +setup( + name="weknora-mcp-server", + version="1.0.0", + author="WeKnora Team", + author_email="support@weknora.com", + description="WeKnora MCP Server - Model Context Protocol server for WeKnora API", + long_description=read_readme(), + long_description_content_type="text/markdown", + url="https://github.com/NannaOlympicBroadcast/WeKnoraMCP", + py_modules=["weknora_mcp_server", "main", "run_server", "run", "test_module"], + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + ], + python_requires=">=3.8", + install_requires=read_requirements(), + entry_points={ + "console_scripts": [ + "weknora-mcp-server=main:sync_main", + "weknora-server=run_server:main", + ], + }, + include_package_data=True, + data_files=[ + ("", ["README.md", "requirements.txt", "LICENSE"]), + ], + keywords="mcp model-context-protocol weknora knowledge-management api-server", +) \ No newline at end of file diff --git a/mcp-server/test_imports.py b/mcp-server/test_imports.py new file mode 100644 index 0000000..0317a88 --- /dev/null +++ b/mcp-server/test_imports.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +""" +测试 MCP 导入 +""" + +try: + import mcp.types as types + print("✓ mcp.types 导入成功") +except ImportError as e: + print(f"✗ mcp.types 导入失败: {e}") + +try: + from mcp.server import Server, NotificationOptions + print("✓ mcp.server 导入成功") +except ImportError as e: + print(f"✗ mcp.server 导入失败: {e}") + +try: + import mcp.server.stdio + print("✓ mcp.server.stdio 导入成功") +except ImportError as e: + print(f"✗ mcp.server.stdio 导入失败: {e}") + +try: + from mcp.server.models import InitializationOptions + print("✓ InitializationOptions 从 mcp.server.models 导入成功") +except ImportError: + try: + from mcp import InitializationOptions + print("✓ InitializationOptions 从 mcp 导入成功") + except ImportError as e: + print(f"✗ InitializationOptions 导入失败: {e}") + +# 检查 MCP 包结构 +import mcp +print(f"\nMCP 包版本: {getattr(mcp, '__version__', '未知')}") +print(f"MCP 包路径: {mcp.__file__}") +print(f"MCP 包内容: {dir(mcp)}") \ No newline at end of file diff --git a/mcp-server/test_module.py b/mcp-server/test_module.py new file mode 100644 index 0000000..3912e87 --- /dev/null +++ b/mcp-server/test_module.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +""" +WeKnora MCP Server 模组测试脚本 + +测试模组的各种启动方式和功能 +""" + +import os +import sys +import subprocess +import importlib.util +from pathlib import Path + +def test_imports(): + """测试模块导入""" + print("=== 测试模块导入 ===") + + try: + # 测试基础依赖 + import mcp + print("✓ mcp 模块导入成功") + + import requests + print("✓ requests 模块导入成功") + + # 测试主模块 + import weknora_mcp_server + print("✓ weknora_mcp_server 模块导入成功") + + # 测试包导入 + from weknora_mcp_server import WeKnoraClient, run + print("✓ WeKnoraClient 和 run 函数导入成功") + + # 测试主入口点 + import main + print("✓ main 模块导入成功") + + return True + + except ImportError as e: + print(f"✗ 导入失败: {e}") + return False + +def test_environment(): + """测试环境配置""" + print("\n=== 测试环境配置 ===") + + base_url = os.getenv("WEKNORA_BASE_URL") + api_key = os.getenv("WEKNORA_API_KEY") + + print(f"WEKNORA_BASE_URL: {base_url or '未设置 (将使用默认值)'}") + print(f"WEKNORA_API_KEY: {'已设置' if api_key else '未设置'}") + + if not base_url: + print("提示: 可以设置环境变量 WEKNORA_BASE_URL") + + if not api_key: + print("提示: 建议设置环境变量 WEKNORA_API_KEY") + + return True + +def test_client_creation(): + """测试客户端创建""" + print("\n=== 测试客户端创建 ===") + + try: + from weknora_mcp_server import WeKnoraClient + + base_url = os.getenv("WEKNORA_BASE_URL", "http://localhost:8080/api/v1") + api_key = os.getenv("WEKNORA_API_KEY", "test_key") + + client = WeKnoraClient(base_url, api_key) + print("✓ WeKnoraClient 创建成功") + + # 检查客户端属性 + assert client.base_url == base_url + assert client.api_key == api_key + print("✓ 客户端配置正确") + + return True + + except Exception as e: + print(f"✗ 客户端创建失败: {e}") + return False + +def test_file_structure(): + """测试文件结构""" + print("\n=== 测试文件结构 ===") + + required_files = [ + "__init__.py", + "main.py", + "run_server.py", + "weknora_mcp_server.py", + "requirements.txt", + "setup.py", + "pyproject.toml", + "README.md", + "INSTALL.md", + "LICENSE", + "MANIFEST.in" + ] + + missing_files = [] + for file in required_files: + if Path(file).exists(): + print(f"✓ {file}") + else: + print(f"✗ {file} (缺失)") + missing_files.append(file) + + if missing_files: + print(f"缺失文件: {missing_files}") + return False + + print("✓ 所有必需文件都存在") + return True + +def test_entry_points(): + """测试入口点""" + print("\n=== 测试入口点 ===") + + # 测试 main.py 的帮助选项 + try: + result = subprocess.run( + [sys.executable, "main.py", "--help"], + capture_output=True, + text=True, + timeout=10 + ) + if result.returncode == 0: + print("✓ main.py --help 工作正常") + else: + print(f"✗ main.py --help 失败: {result.stderr}") + return False + except subprocess.TimeoutExpired: + print("✗ main.py --help 超时") + return False + except Exception as e: + print(f"✗ main.py --help 错误: {e}") + return False + + # 测试环境检查 + try: + result = subprocess.run( + [sys.executable, "main.py", "--check-only"], + capture_output=True, + text=True, + timeout=10 + ) + if result.returncode == 0: + print("✓ main.py --check-only 工作正常") + else: + print(f"✗ main.py --check-only 失败: {result.stderr}") + return False + except subprocess.TimeoutExpired: + print("✗ main.py --check-only 超时") + return False + except Exception as e: + print(f"✗ main.py --check-only 错误: {e}") + return False + + return True + +def test_package_installation(): + """测试包安装 (开发模式)""" + print("\n=== 测试包安装 ===") + + try: + # 检查是否可以以开发模式安装 + result = subprocess.run( + [sys.executable, "setup.py", "check"], + capture_output=True, + text=True, + timeout=30 + ) + + if result.returncode == 0: + print("✓ setup.py 检查通过") + else: + print(f"✗ setup.py 检查失败: {result.stderr}") + return False + + except subprocess.TimeoutExpired: + print("✗ setup.py 检查超时") + return False + except Exception as e: + print(f"✗ setup.py 检查错误: {e}") + return False + + return True + +def main(): + """运行所有测试""" + print("WeKnora MCP Server 模组测试") + print("=" * 50) + + tests = [ + ("模块导入", test_imports), + ("环境配置", test_environment), + ("客户端创建", test_client_creation), + ("文件结构", test_file_structure), + ("入口点", test_entry_points), + ("包安装", test_package_installation), + ] + + passed = 0 + total = len(tests) + + for test_name, test_func in tests: + try: + if test_func(): + passed += 1 + else: + print(f"测试失败: {test_name}") + except Exception as e: + print(f"测试异常: {test_name} - {e}") + + print("\n" + "=" * 50) + print(f"测试结果: {passed}/{total} 通过") + + if passed == total: + print("✓ 所有测试通过！模组可以正常使用。") + return True + else: + print("✗ 部分测试失败，请检查上述错误。") + return False + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1) \ No newline at end of file diff --git a/mcp-server/weknora_mcp_server.py b/mcp-server/weknora_mcp_server.py new file mode 100644 index 0000000..2e2cc48 --- /dev/null +++ b/mcp-server/weknora_mcp_server.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +""" +WeKnora MCP Server + +A Model Context Protocol server that provides access to the WeKnora knowledge management API. +""" + +import os +import json +import logging +from typing import Dict, List, Any, Optional +from datetime import datetime +import requests +from requests.exceptions import RequestException +import mcp.server.stdio +import mcp.types as types +from mcp.server import NotificationOptions, Server +from mcp.server.models import InitializationOptions + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Configuration +WEKNORA_BASE_URL = os.getenv("WEKNORA_BASE_URL", "http://localhost:8080/api/v1") +WEKNORA_API_KEY = os.getenv("WEKNORA_API_KEY", "") + +class WeKnoraClient: + """Client for interacting with WeKnora API""" + + def __init__(self, base_url: str, api_key: str): + self.base_url = base_url + self.api_key = api_key + self.session = requests.Session() + self.session.headers.update({ + "X-API-Key": api_key, + "Content-Type": "application/json" + }) + + def _request(self, method: str, endpoint: str, **kwargs) -> Dict[str, Any]: + """Make a request to the WeKnora API""" + url = f"{self.base_url}{endpoint}" + try: + response = self.session.request(method, url, **kwargs) + response.raise_for_status() + return response.json() + except RequestException as e: + logger.error(f"API request failed: {e}") + raise + + # Tenant Management + def create_tenant(self, name: str, description: str, business: str, retriever_engines: Dict) -> Dict: + """Create a new tenant""" + data = { + "name": name, + "description": description, + "business": business, + "retriever_engines": retriever_engines + } + return self._request("POST", "/tenants", json=data) + + def get_tenant(self, tenant_id: str) -> Dict: + """Get tenant information""" + return self._request("GET", f"/tenants/{tenant_id}") + + def list_tenants(self) -> Dict: + """List all tenants""" + return self._request("GET", "/tenants") + + # Knowledge Base Management + def create_knowledge_base(self, name: str, description: str, config: Dict) -> Dict: + """Create a new knowledge base""" + data = { + "name": name, + "description": description, + **config + } + return self._request("POST", "/knowledge-bases", json=data) + + def list_knowledge_bases(self) -> Dict: + """List all knowledge bases""" + return self._request("GET", "/knowledge-bases") + + def get_knowledge_base(self, kb_id: str) -> Dict: + """Get knowledge base details""" + return self._request("GET", f"/knowledge-bases/{kb_id}") + + def update_knowledge_base(self, kb_id: str, updates: Dict) -> Dict: + """Update knowledge base""" + return self._request("PUT", f"/knowledge-bases/{kb_id}", json=updates) + + def delete_knowledge_base(self, kb_id: str) -> Dict: + """Delete knowledge base""" + return self._request("DELETE", f"/knowledge-bases/{kb_id}") + + def hybrid_search(self, kb_id: str, query: str, config: Dict) -> Dict: + """Perform hybrid search in knowledge base""" + data = { + "query_text": query, + **config + } + return self._request("GET", f"/knowledge-bases/{kb_id}/hybrid-search", json=data) + + # Knowledge Management + def create_knowledge_from_file(self, kb_id: str, file_path: str, enable_multimodel: bool = True) -> Dict: + """Create knowledge from file""" + with open(file_path, 'rb') as f: + files = {'file': f} + data = {'enable_multimodel': str(enable_multimodel).lower()} + # Temporarily remove Content-Type for multipart request + headers = self.session.headers.copy() + del headers['Content-Type'] + response = requests.post( + f"{self.base_url}/knowledge-bases/{kb_id}/knowledge/file", + headers=headers, + files=files, + data=data + ) + response.raise_for_status() + return response.json() + + def create_knowledge_from_url(self, kb_id: str, url: str, enable_multimodel: bool = True) -> Dict: + """Create knowledge from URL""" + data = { + "url": url, + "enable_multimodel": enable_multimodel + } + return self._request("POST", f"/knowledge-bases/{kb_id}/knowledge/url", json=data) + + def list_knowledge(self, kb_id: str, page: int = 1, page_size: int = 20) -> Dict: + """List knowledge in a knowledge base""" + params = {"page": page, "page_size": page_size} + return self._request("GET", f"/knowledge-bases/{kb_id}/knowledge", params=params) + + def get_knowledge(self, knowledge_id: str) -> Dict: + """Get knowledge details""" + return self._request("GET", f"/knowledge/{knowledge_id}") + + def delete_knowledge(self, knowledge_id: str) -> Dict: + """Delete knowledge""" + return self._request("DELETE", f"/knowledge/{knowledge_id}") + + # Model Management + def create_model(self, name: str, model_type: str, source: str, description: str, parameters: Dict, is_default: bool = False) -> Dict: + """Create a new model""" + data = { + "name": name, + "type": model_type, + "source": source, + "description": description, + "parameters": parameters, + "is_default": is_default + } + return self._request("POST", "/models", json=data) + + def list_models(self) -> Dict: + """List all models""" + return self._request("GET", "/models") + + def get_model(self, model_id: str) -> Dict: + """Get model details""" + return self._request("GET", f"/models/{model_id}") + + # Session Management + def create_session(self, kb_id: str, strategy: Dict) -> Dict: + """Create a new chat session""" + data = { + "knowledge_base_id": kb_id, + "session_strategy": strategy + } + return self._request("POST", "/sessions", json=data) + + def get_session(self, session_id: str) -> Dict: + """Get session details""" + return self._request("GET", f"/sessions/{session_id}") + + def list_sessions(self, page: int = 1, page_size: int = 20) -> Dict: + """List sessions""" + params = {"page": page, "page_size": page_size} + return self._request("GET", "/sessions", params=params) + + def delete_session(self, session_id: str) -> Dict: + """Delete session""" + return self._request("DELETE", f"/sessions/{session_id}") + + # Chat Functionality + def chat(self, session_id: str, query: str) -> Dict: + """Send a chat message""" + data = {"query": query} + # Note: This returns SSE stream, simplified here + return self._request("POST", f"/knowledge-chat/{session_id}", json=data) + + # Chunk Management + def list_chunks(self, knowledge_id: str, page: int = 1, page_size: int = 20) -> Dict: + """List chunks of knowledge""" + params = {"page": page, "page_size": page_size} + return self._request("GET", f"/chunks/{knowledge_id}", params=params) + + def delete_chunk(self, knowledge_id: str, chunk_id: str) -> Dict: + """Delete a chunk""" + return self._request("DELETE", f"/chunks/{knowledge_id}/{chunk_id}") + +# Initialize MCP server +app = Server("weknora-server") +client = WeKnoraClient(WEKNORA_BASE_URL, WEKNORA_API_KEY) + +# Tool definitions +@app.list_tools() +async def handle_list_tools() -> list[types.Tool]: + """List all available WeKnora tools""" + return [ + # Tenant Management + types.Tool( + name="create_tenant", + description="Create a new tenant in WeKnora", + inputSchema={ + "type": "object", + "properties": { + "name": {"type": "string", "description": "Tenant name"}, + "description": {"type": "string", "description": "Tenant description"}, + "business": {"type": "string", "description": "Business type"}, + "retriever_engines": { + "type": "object", + "description": "Retriever engine configuration", + "properties": { + "engines": { + "type": "array", + "items": { + "type": "object", + "properties": { + "retriever_type": {"type": "string"}, + "retriever_engine_type": {"type": "string"} + } + } + } + } + } + }, + "required": ["name", "description", "business"] + } + ), + types.Tool( + name="list_tenants", + description="List all tenants", + inputSchema={"type": "object", "properties": {}} + ), + + # Knowledge Base Management + types.Tool( + name="create_knowledge_base", + description="Create a new knowledge base", + inputSchema={ + "type": "object", + "properties": { + "name": {"type": "string", "description": "Knowledge base name"}, + "description": {"type": "string", "description": "Knowledge base description"}, + "embedding_model_id": {"type": "string", "description": "Embedding model ID"}, + "summary_model_id": {"type": "string", "description": "Summary model ID"} + }, + "required": ["name", "description"] + } + ), + types.Tool( + name="list_knowledge_bases", + description="List all knowledge bases", + inputSchema={"type": "object", "properties": {}} + ), + types.Tool( + name="get_knowledge_base", + description="Get knowledge base details", + inputSchema={ + "type": "object", + "properties": { + "kb_id": {"type": "string", "description": "Knowledge base ID"} + }, + "required": ["kb_id"] + } + ), + types.Tool( + name="delete_knowledge_base", + description="Delete a knowledge base", + inputSchema={ + "type": "object", + "properties": { + "kb_id": {"type": "string", "description": "Knowledge base ID"} + }, + "required": ["kb_id"] + } + ), + types.Tool( + name="hybrid_search", + description="Perform hybrid search in knowledge base", + inputSchema={ + "type": "object", + "properties": { + "kb_id": {"type": "string", "description": "Knowledge base ID"}, + "query": {"type": "string", "description": "Search query"}, + "vector_threshold": {"type": "number", "description": "Vector similarity threshold", "default": 0.5}, + "keyword_threshold": {"type": "number", "description": "Keyword match threshold", "default": 0.3}, + "match_count": {"type": "integer", "description": "Number of results to return", "default": 5} + }, + "required": ["kb_id", "query"] + } + ), + + # Knowledge Management + types.Tool( + name="create_knowledge_from_file", + description="Create knowledge from a local file on the server filesystem", + inputSchema={ + "type": "object", + "properties": { + "kb_id": {"type": "string", "description": "Knowledge base ID"}, + "file_path": {"type": "string", "description": "Absolute path to the local file on the server"}, + "enable_multimodel": {"type": "boolean", "description": "Enable multimodal processing", "default": True} + }, + "required": ["kb_id", "file_path"] + } + ), + types.Tool( + name="create_knowledge_from_url", + description="Create knowledge from URL", + inputSchema={ + "type": "object", + "properties": { + "kb_id": {"type": "string", "description": "Knowledge base ID"}, + "url": {"type": "string", "description": "URL to create knowledge from"}, + "enable_multimodel": {"type": "boolean", "description": "Enable multimodal processing", "default": True} + }, + "required": ["kb_id", "url"] + } + ), + types.Tool( + name="list_knowledge", + description="List knowledge in a knowledge base", + inputSchema={ + "type": "object", + "properties": { + "kb_id": {"type": "string", "description": "Knowledge base ID"}, + "page": {"type": "integer", "description": "Page number", "default": 1}, + "page_size": {"type": "integer", "description": "Page size", "default": 20} + }, + "required": ["kb_id"] + } + ), + types.Tool( + name="get_knowledge", + description="Get knowledge details", + inputSchema={ + "type": "object", + "properties": { + "knowledge_id": {"type": "string", "description": "Knowledge ID"} + }, + "required": ["knowledge_id"] + } + ), + types.Tool( + name="delete_knowledge", + description="Delete knowledge", + inputSchema={ + "type": "object", + "properties": { + "knowledge_id": {"type": "string", "description": "Knowledge ID"} + }, + "required": ["knowledge_id"] + } + ), + + # Model Management + types.Tool( + name="create_model", + description="Create a new model", + inputSchema={ + "type": "object", + "properties": { + "name": {"type": "string", "description": "Model name"}, + "type": {"type": "string", "description": "Model type (KnowledgeQA, Embedding, Rerank)"}, + "source": {"type": "string", "description": "Model source", "default": "local"}, + "description": {"type": "string", "description": "Model description"}, + "base_url": {"type": "string", "description": "Model API base URL", "default": ""}, + "api_key": {"type": "string", "description": "Model API key", "default": ""}, + "is_default": {"type": "boolean", "description": "Set as default model", "default": False} + }, + "required": ["name", "type", "description"] + } + ), + types.Tool( + name="list_models", + description="List all models", + inputSchema={"type": "object", "properties": {}} + ), + types.Tool( + name="get_model", + description="Get model details", + inputSchema={ + "type": "object", + "properties": { + "model_id": {"type": "string", "description": "Model ID"} + }, + "required": ["model_id"] + } + ), + + # Session Management + types.Tool( + name="create_session", + description="Create a new chat session", + inputSchema={ + "type": "object", + "properties": { + "kb_id": {"type": "string", "description": "Knowledge base ID"}, + "max_rounds": {"type": "integer", "description": "Maximum conversation rounds", "default": 5}, + "enable_rewrite": {"type": "boolean", "description": "Enable query rewriting", "default": True}, + "fallback_response": {"type": "string", "description": "Fallback response", "default": "Sorry, I cannot answer this question."}, + "summary_model_id": {"type": "string", "description": "Summary model ID"} + }, + "required": ["kb_id"] + } + ), + types.Tool( + name="get_session", + description="Get session details", + inputSchema={ + "type": "object", + "properties": { + "session_id": {"type": "string", "description": "Session ID"} + }, + "required": ["session_id"] + } + ), + types.Tool( + name="list_sessions", + description="List chat sessions", + inputSchema={ + "type": "object", + "properties": { + "page": {"type": "integer", "description": "Page number", "default": 1}, + "page_size": {"type": "integer", "description": "Page size", "default": 20} + } + } + ), + types.Tool( + name="delete_session", + description="Delete a session", + inputSchema={ + "type": "object", + "properties": { + "session_id": {"type": "string", "description": "Session ID"} + }, + "required": ["session_id"] + } + ), + + # Chat Functionality + types.Tool( + name="chat", + description="Send a chat message to a session", + inputSchema={ + "type": "object", + "properties": { + "session_id": {"type": "string", "description": "Session ID"}, + "query": {"type": "string", "description": "User query"} + }, + "required": ["session_id", "query"] + } + ), + + # Chunk Management + types.Tool( + name="list_chunks", + description="List chunks of knowledge", + inputSchema={ + "type": "object", + "properties": { + "knowledge_id": {"type": "string", "description": "Knowledge ID"}, + "page": {"type": "integer", "description": "Page number", "default": 1}, + "page_size": {"type": "integer", "description": "Page size", "default": 20} + }, + "required": ["knowledge_id"] + } + ), + types.Tool( + name="delete_chunk", + description="Delete a chunk", + inputSchema={ + "type": "object", + "properties": { + "knowledge_id": {"type": "string", "description": "Knowledge ID"}, + "chunk_id": {"type": "string", "description": "Chunk ID"} + }, + "required": ["knowledge_id", "chunk_id"] + } + ) + ] + +@app.call_tool() +async def handle_call_tool( + name: str, arguments: dict | None +) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]: + """Handle tool execution""" + + try: + args = arguments or {} + + # Tenant Management + if name == "create_tenant": + result = client.create_tenant( + args["name"], + args["description"], + args["business"], + args.get("retriever_engines", { + "engines": [ + {"retriever_type": "keywords", "retriever_engine_type": "postgres"}, + {"retriever_type": "vector", "retriever_engine_type": "postgres"} + ] + }) + ) + elif name == "list_tenants": + result = client.list_tenants() + + # Knowledge Base Management + elif name == "create_knowledge_base": + config = { + "chunking_config": args.get("chunking_config", { + "chunk_size": 1000, + "chunk_overlap": 200, + "separators": ["."], + "enable_multimodal": True + }), + "embedding_model_id": args.get("embedding_model_id", ""), + "summary_model_id": args.get("summary_model_id", "") + } + result = client.create_knowledge_base( + args["name"], + args["description"], + config + ) + elif name == "list_knowledge_bases": + result = client.list_knowledge_bases() + elif name == "get_knowledge_base": + result = client.get_knowledge_base(args["kb_id"]) + elif name == "delete_knowledge_base": + result = client.delete_knowledge_base(args["kb_id"]) + elif name == "hybrid_search": + config = { + "vector_threshold": args.get("vector_threshold", 0.5), + "keyword_threshold": args.get("keyword_threshold", 0.3), + "match_count": args.get("match_count", 5) + } + result = client.hybrid_search(args["kb_id"], args["query"], config) + + # Knowledge Management + elif name == "create_knowledge_from_file": + result = client.create_knowledge_from_file( + args["kb_id"], + args["file_path"], + args.get("enable_multimodel", True) + ) + elif name == "create_knowledge_from_url": + result = client.create_knowledge_from_url( + args["kb_id"], + args["url"], + args.get("enable_multimodel", True) + ) + elif name == "list_knowledge": + result = client.list_knowledge( + args["kb_id"], + args.get("page", 1), + args.get("page_size", 20) + ) + elif name == "get_knowledge": + result = client.get_knowledge(args["knowledge_id"]) + elif name == "delete_knowledge": + result = client.delete_knowledge(args["knowledge_id"]) + + # Model Management + elif name == "create_model": + parameters = { + "base_url": args.get("base_url", ""), + "api_key": args.get("api_key", "") + } + result = client.create_model( + args["name"], + args["type"], + args.get("source", "local"), + args["description"], + parameters, + args.get("is_default", False) + ) + elif name == "list_models": + result = client.list_models() + elif name == "get_model": + result = client.get_model(args["model_id"]) + + # Session Management + elif name == "create_session": + strategy = { + "max_rounds": args.get("max_rounds", 5), + "enable_rewrite": args.get("enable_rewrite", True), + "fallback_strategy": "FIXED_RESPONSE", + "fallback_response": args.get("fallback_response", "Sorry, I cannot answer this question."), + "embedding_top_k": 10, + "keyword_threshold": 0.5, + "vector_threshold": 0.7, + "summary_model_id": args.get("summary_model_id", "") + } + result = client.create_session(args["kb_id"], strategy) + elif name == "get_session": + result = client.get_session(args["session_id"]) + elif name == "list_sessions": + result = client.list_sessions( + args.get("page", 1), + args.get("page_size", 20) + ) + elif name == "delete_session": + result = client.delete_session(args["session_id"]) + + # Chat Functionality + elif name == "chat": + result = client.chat(args["session_id"], args["query"]) + + # Chunk Management + elif name == "list_chunks": + result = client.list_chunks( + args["knowledge_id"], + args.get("page", 1), + args.get("page_size", 20) + ) + elif name == "delete_chunk": + result = client.delete_chunk(args["knowledge_id"], args["chunk_id"]) + + else: + return [types.TextContent( + type="text", + text=f"Unknown tool: {name}" + )] + + return [types.TextContent( + type="text", + text=json.dumps(result, indent=2, ensure_ascii=False) + )] + + except Exception as e: + logger.error(f"Tool execution failed: {e}") + return [types.TextContent( + type="text", + text=f"Error executing {name}: {str(e)}" + )] + +async def run(): + """Run the MCP server""" + async with mcp.server.stdio.stdio_server() as (read_stream, write_stream): + await app.run( + read_stream, + write_stream, + InitializationOptions( + server_name="weknora-server", + server_version="1.0.0", + capabilities=app.get_capabilities( + notification_options=NotificationOptions(), + experimental_capabilities={}, + ), + ), + ) + +def main(): + """主函数入口点，用于 console_scripts""" + import asyncio + asyncio.run(run()) + +if __name__ == "__main__": + main() diff --git a/migrations/mysql/00-init-db.sql b/migrations/mysql/00-init-db.sql new file mode 100644 index 0000000..357f488 --- /dev/null +++ b/migrations/mysql/00-init-db.sql @@ -0,0 +1,153 @@ +DROP TABLE IF EXISTS tenants; +DROP TABLE IF EXISTS models; +DROP TABLE IF EXISTS knowledge_bases; +DROP TABLE IF EXISTS knowledges; +DROP TABLE IF EXISTS sessions; +DROP TABLE IF EXISTS messages; +DROP TABLE IF EXISTS chunks; + +CREATE TABLE tenants ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + api_key VARCHAR(64) NOT NULL, + retriever_engines JSON NOT NULL, + status VARCHAR(50) DEFAULT 'active', + business VARCHAR(255) NOT NULL, + storage_quota BIGINT NOT NULL DEFAULT 10737418240, + storage_used BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 AUTO_INCREMENT=10000; + +CREATE TABLE models ( + id VARCHAR(64) PRIMARY KEY, + tenant_id INT NOT NULL, + name VARCHAR(255) NOT NULL, + type VARCHAR(50) NOT NULL, + source VARCHAR(50) NOT NULL, + description TEXT, + parameters JSON NOT NULL, + is_default BOOLEAN NOT NULL DEFAULT FALSE, + status VARCHAR(50) NOT NULL DEFAULT 'active', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE INDEX idx_models_tenant_source_type ON models(tenant_id, source, type); + +CREATE TABLE knowledge_bases ( + id VARCHAR(36) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + tenant_id INT NOT NULL, + chunking_config JSON NOT NULL, + image_processing_config JSON NOT NULL, + embedding_model_id VARCHAR(64) NOT NULL, + summary_model_id VARCHAR(64) NOT NULL, + rerank_model_id VARCHAR(64) NOT NULL, + vlm_model_id VARCHAR(64) NOT NULL, + cos_config JSON NOT NULL, + vlm_config JSON NOT NULL, + extract_config JSON NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE INDEX idx_knowledge_bases_tenant_name ON knowledge_bases(tenant_id, name); + +CREATE TABLE knowledges ( + id VARCHAR(36) PRIMARY KEY, + tenant_id INT NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + type VARCHAR(50) NOT NULL, + title VARCHAR(255) NOT NULL, + description TEXT, + source VARCHAR(128) NOT NULL, + parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', + enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', + embedding_model_id VARCHAR(64), + file_name VARCHAR(255), + file_type VARCHAR(50), + file_size BIGINT, + file_path TEXT, + file_hash VARCHAR(64), + storage_size BIGINT NOT NULL DEFAULT 0, + metadata JSON, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL, + processed_at TIMESTAMP, + error_message TEXT +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE INDEX idx_knowledges_tenant_id ON knowledges(tenant_id, knowledge_base_id); + +CREATE TABLE sessions ( + id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + title VARCHAR(255), + description TEXT, + knowledge_base_id VARCHAR(36), + max_rounds INT NOT NULL DEFAULT 5, + enable_rewrite BOOLEAN NOT NULL DEFAULT TRUE, + fallback_strategy VARCHAR(255) NOT NULL DEFAULT 'fixed', + fallback_response VARCHAR(255) NOT NULL DEFAULT '很抱歉，我暂时无法回答这个问题。', + keyword_threshold FLOAT NOT NULL DEFAULT 0.5, + vector_threshold FLOAT NOT NULL DEFAULT 0.5, + rerank_model_id VARCHAR(64), + embedding_top_k INTEGER NOT NULL DEFAULT 10, + rerank_top_k INTEGER NOT NULL DEFAULT 10, + rerank_threshold FLOAT NOT NULL DEFAULT 0.65, + summary_model_id VARCHAR(64), + summary_parameters JSON NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE INDEX idx_sessions_tenant_id ON sessions(tenant_id); + +CREATE TABLE messages ( + id VARCHAR(36) PRIMARY KEY, + request_id VARCHAR(36) NOT NULL, + session_id VARCHAR(36) NOT NULL, + role VARCHAR(50) NOT NULL, + content TEXT NOT NULL, + knowledge_references JSON NOT NULL, + is_completed BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE INDEX idx_messages_session_role ON messages(session_id, role); + +CREATE TABLE chunks ( + id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + knowledge_id VARCHAR(36) NOT NULL, + content TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + start_at INTEGER NOT NULL, + end_at INTEGER NOT NULL, + pre_chunk_id VARCHAR(36), + next_chunk_id VARCHAR(36), + chunk_type VARCHAR(20) NOT NULL DEFAULT 'text', + parent_chunk_id VARCHAR(36), + image_info TEXT, + relation_chunks JSON, + indirect_relation_chunks JSON, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + deleted_at TIMESTAMP NULL DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE INDEX idx_chunks_tenant_knowledge ON chunks(tenant_id, knowledge_id); +CREATE INDEX idx_chunks_parent_id ON chunks(parent_chunk_id); +CREATE INDEX idx_chunks_chunk_type ON chunks(chunk_type); diff --git a/migrations/paradedb/00-init-db.sql b/migrations/paradedb/00-init-db.sql new file mode 100644 index 0000000..403f188 --- /dev/null +++ b/migrations/paradedb/00-init-db.sql @@ -0,0 +1,205 @@ +-- Create extensions +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS pg_search; + + +-- Create tenant table +CREATE TABLE IF NOT EXISTS tenants ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + api_key VARCHAR(64) NOT NULL, + retriever_engines JSONB NOT NULL DEFAULT '[]', + status VARCHAR(50) DEFAULT 'active', + business VARCHAR(255) NOT NULL, + storage_quota BIGINT NOT NULL DEFAULT 10737418240, -- 默认10GB配额(Bytes) + storage_used BIGINT NOT NULL DEFAULT 0, -- 已使用的存储空间(Bytes) + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Set the starting value for tenants id sequence +ALTER SEQUENCE tenants_id_seq RESTART WITH 10000; + +-- Add indexes +CREATE INDEX IF NOT EXISTS idx_tenants_api_key ON tenants(api_key); +CREATE INDEX IF NOT EXISTS idx_tenants_status ON tenants(status); + +-- Create model table +CREATE TABLE IF NOT EXISTS models ( + id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + name VARCHAR(255) NOT NULL, + type VARCHAR(50) NOT NULL, + source VARCHAR(50) NOT NULL, + description TEXT, + parameters JSONB NOT NULL, + is_default BOOLEAN NOT NULL DEFAULT false, + status VARCHAR(50) NOT NULL DEFAULT 'active', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Add indexes for models +CREATE INDEX IF NOT EXISTS idx_models_type ON models(type); +CREATE INDEX IF NOT EXISTS idx_models_source ON models(source); + +-- Create knowledge_base table +CREATE TABLE IF NOT EXISTS knowledge_bases ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + name VARCHAR(255) NOT NULL, + description TEXT, + tenant_id INTEGER NOT NULL, + chunking_config JSONB NOT NULL DEFAULT '{"chunk_size": 512, "chunk_overlap": 50, "split_markers": ["\n\n", "\n", "。"], "keep_separator": true}', + image_processing_config JSONB NOT NULL DEFAULT '{"enable_multimodal": false, "model_id": ""}', + embedding_model_id VARCHAR(64) NOT NULL, + summary_model_id VARCHAR(64) NOT NULL, + rerank_model_id VARCHAR(64) NOT NULL, + vlm_model_id VARCHAR(64) NOT NULL, + cos_config JSONB NOT NULL DEFAULT '{}', + vlm_config JSONB NOT NULL DEFAULT '{}', + extract_config JSONB NULL DEFAULT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Add indexes for knowledge_bases +CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_id ON knowledge_bases(tenant_id); + +-- Create knowledge table +CREATE TABLE IF NOT EXISTS knowledges ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + type VARCHAR(50) NOT NULL, + title VARCHAR(255) NOT NULL, + description TEXT, + source VARCHAR(128) NOT NULL, + parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', + enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', + embedding_model_id VARCHAR(64), + file_name VARCHAR(255), + file_type VARCHAR(50), + file_size BIGINT, + file_path TEXT, + file_hash VARCHAR(64), + storage_size BIGINT NOT NULL DEFAULT 0, -- 存储大小(Byte) + metadata JSONB, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + processed_at TIMESTAMP WITH TIME ZONE, + error_message TEXT, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Add indexes for knowledge +CREATE INDEX IF NOT EXISTS idx_knowledges_tenant_id ON knowledges(tenant_id); +CREATE INDEX IF NOT EXISTS idx_knowledges_base_id ON knowledges(knowledge_base_id); +CREATE INDEX IF NOT EXISTS idx_knowledges_parse_status ON knowledges(parse_status); +CREATE INDEX IF NOT EXISTS idx_knowledges_enable_status ON knowledges(enable_status); + +-- Create session table +CREATE TABLE IF NOT EXISTS sessions ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + title VARCHAR(255), + description TEXT, + knowledge_base_id VARCHAR(36), + max_rounds INTEGER NOT NULL DEFAULT 5, + enable_rewrite BOOLEAN NOT NULL DEFAULT true, + fallback_strategy VARCHAR(255) NOT NULL DEFAULT 'fixed', + fallback_response TEXT NOT NULL DEFAULT '很抱歉，我暂时无法回答这个问题。', + keyword_threshold FLOAT NOT NULL DEFAULT 0.5, + vector_threshold FLOAT NOT NULL DEFAULT 0.5, + rerank_model_id VARCHAR(64), + embedding_top_k INTEGER NOT NULL DEFAULT 10, + rerank_top_k INTEGER NOT NULL DEFAULT 10, + rerank_threshold FLOAT NOT NULL DEFAULT 0.65, + summary_model_id VARCHAR(64), + summary_parameters JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Create Index for sessions +CREATE INDEX IF NOT EXISTS idx_sessions_tenant_id ON sessions(tenant_id); + + +-- Create message table +CREATE TABLE IF NOT EXISTS messages ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + request_id VARCHAR(36) NOT NULL, + session_id VARCHAR(36) NOT NULL, + role VARCHAR(50) NOT NULL, + content TEXT NOT NULL, + knowledge_references JSONB NOT NULL DEFAULT '[]', + is_completed BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Create Index for messages +CREATE INDEX IF NOT EXISTS idx_messages_session_id ON messages(session_id); + + +CREATE TABLE IF NOT EXISTS chunks ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + knowledge_id VARCHAR(36) NOT NULL, + content TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT true, + start_at INTEGER NOT NULL, + end_at INTEGER NOT NULL, + pre_chunk_id VARCHAR(36), + next_chunk_id VARCHAR(36), + chunk_type VARCHAR(20) NOT NULL DEFAULT 'text', + parent_chunk_id VARCHAR(36), + image_info TEXT, + relation_chunks JSONB, + indirect_relation_chunks JSONB, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_chunks_tenant_kg ON chunks(tenant_id, knowledge_id); +CREATE INDEX IF NOT EXISTS idx_chunks_parent_id ON chunks(parent_chunk_id); +CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type); + +CREATE TABLE IF NOT EXISTS embeddings ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + + source_id VARCHAR(64) NOT NULL, + source_type INTEGER NOT NULL, + chunk_id VARCHAR(64), + knowledge_id VARCHAR(64), + knowledge_base_id VARCHAR(64), + content TEXT, + dimension INTEGER NOT NULL, + embedding halfvec +); + +CREATE UNIQUE INDEX IF NOT EXISTS embeddings_unique_source ON embeddings(source_id, source_type); +CREATE INDEX IF NOT EXISTS embeddings_search_idx ON embeddings +USING bm25 (id, knowledge_base_id, content, knowledge_id, chunk_id) +WITH ( + key_field = 'id', + text_fields = '{ + "content": { + "tokenizer": {"type": "chinese_lindera"} + } + }' +); +CREATE INDEX ON embeddings USING hnsw ((embedding::halfvec(3584)) halfvec_cosine_ops) WITH (m = 16, ef_construction = 64) WHERE (dimension = 3584); +CREATE INDEX ON embeddings USING hnsw ((embedding::halfvec(798)) halfvec_cosine_ops) WITH (m = 16, ef_construction = 64) WHERE (dimension = 798); \ No newline at end of file diff --git a/migrations/paradedb/01-migrate-to-paradedb.sql b/migrations/paradedb/01-migrate-to-paradedb.sql new file mode 100644 index 0000000..2ad51b6 --- /dev/null +++ b/migrations/paradedb/01-migrate-to-paradedb.sql @@ -0,0 +1,69 @@ +-- 迁移脚本：从PostgreSQL迁移到ParadeDB +-- 注意：在执行此脚本前，请确保已经备份了数据 + +-- 1. 导出数据（在PostgreSQL中执行） +-- pg_dump -U postgres -h localhost -p 5432 -d your_database > backup.sql + +-- 2. 导入数据（在ParadeDB中执行） +-- psql -U postgres -h localhost -p 5432 -d your_database < backup.sql + +-- 3. 验证数据 + + +-- Insert some sample data +-- INSERT INTO tenants (id, name, description, status, api_key) +-- VALUES +-- (1, 'Demo Tenant', 'This is a demo tenant for testing', 'active', 'sk-00000001abcdefg123456') +-- ON CONFLICT DO NOTHING; + +-- SELECT setval('tenants_id_seq', (SELECT MAX(id) FROM tenants)); + + +-- -- Create knowledge base +-- INSERT INTO knowledge_bases (id, name, description, tenant_id, chunking_config, image_processing_config, embedding_model_id) +-- VALUES +-- ('kb-00000001', 'Default Knowledge Base', 'Default knowledge base for testing', 1, '{"chunk_size": 512, "chunk_overlap": 50, "separators": ["\n\n", "\n", "。"], "keep_separator": true}', '{"enable_multimodal": false, "model_id": ""}', 'model-embedding-00000001'), +-- ('kb-00000002', 'Test Knowledge Base', 'Test knowledge base for development', 1, '{"chunk_size": 512, "chunk_overlap": 50, "separators": ["\n\n", "\n", "。"], "keep_separator": true}', '{"enable_multimodal": false, "model_id": ""}', 'model-embedding-00000001'), +-- ('kb-00000003', 'Test Knowledge Base 2', 'Test knowledge base for development 2', 1, '{"chunk_size": 512, "chunk_overlap": 50, "separators": ["\n\n", "\n", "。"], "keep_separator": true}', '{"enable_multimodal": false, "model_id": ""}', 'model-embedding-00000001') +-- ON CONFLICT DO NOTHING; + + +SELECT COUNT(*) FROM tenants; +SELECT COUNT(*) FROM models; +SELECT COUNT(*) FROM knowledge_bases; +SELECT COUNT(*) FROM knowledges; + + +-- 测试中文全文搜索 + +-- 创建文档表 +CREATE TABLE chinese_documents ( + id SERIAL PRIMARY KEY, + title TEXT, + content TEXT, + published_date DATE +); + +-- 在表上创建 BM25 索引，使用结巴分词器支持中文 +CREATE INDEX idx_documents_bm25 ON chinese_documents +USING bm25 (id, content) +WITH ( + key_field = 'id', + text_fields = '{ + "content": { + "tokenizer": {"type": "chinese_lindera"} + } + }' +); + +INSERT INTO chinese_documents (title, content, published_date) +VALUES +('人工智能的发展', '人工智能技术正在快速发展，影响了我们生活的方方面面。大语言模型是最近的一个重要突破。', '2023-01-15'), +('机器学习基础', '机器学习是人工智能的一个重要分支，包括监督学习、无监督学习和强化学习等方法。', '2023-02-20'), +('深度学习应用', '深度学习在图像识别、自然语言处理和语音识别等领域有广泛应用。', '2023-03-10'), +('自然语言处理技术', '自然语言处理允许计算机理解、解释和生成人类语言，是人工智能的核心技术之一。', '2023-04-05'), +('计算机视觉入门', '计算机视觉让机器能够"看到"并理解视觉世界，广泛应用于安防、医疗等领域。', '2023-05-12'); + +INSERT INTO chinese_documents (title, content, published_date) +VALUES +('hello world', 'hello world', '2023-05-12'); diff --git a/packages/.gitkeep b/packages/.gitkeep new file mode 100644 index 0000000..33e963e --- /dev/null +++ b/packages/.gitkeep @@ -0,0 +1 @@ +# 临时创建的 packages 目录 diff --git a/rerank_server_demo.py b/rerank_server_demo.py new file mode 100644 index 0000000..866acf9 --- /dev/null +++ b/rerank_server_demo.py @@ -0,0 +1,102 @@ +import torch +import uvicorn +from fastapi import FastAPI +from pydantic import BaseModel, Field +from transformers import AutoModelForSequenceClassification, AutoTokenizer +from typing import List + +# --- 1. 定义API的请求和响应数据结构 --- + +# 请求体结构保持不变 +class RerankRequest(BaseModel): + query: str + documents: List[str] + +# --- 修改开始：定义测试用的响应结构，字段名为 "score" --- + +# DocumentInfo 结构保持不变 +class DocumentInfo(BaseModel): + text: str + +# 将原来的 GoRankResult 修改为 TestRankResult +# 核心改动：将 "relevance_score" 字段重命名为 "score" +class TestRankResult(BaseModel): + index: int + document: DocumentInfo + score: float # <--- 【关键修改点】字段名已从 relevance_score 改为 score + +# 最终响应体结构，其 "results" 列表包含的是 TestRankResult +class TestFinalResponse(BaseModel): + results: List[TestRankResult] + +# --- 修改结束 --- + + +# --- 2. 加载模型 (在服务启动时执行一次) --- +print("正在加载模型，请稍候...") +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(f"使用的设备: {device}") +try: + # 请确保这里的路径是正确的 + model_path = '/data1/home/lwx/work/Download/rerank_model_weight' + tokenizer = AutoTokenizer.from_pretrained(model_path) + model = AutoModelForSequenceClassification.from_pretrained(model_path) + model.to(device) + model.eval() + print("模型加载成功！") +except Exception as e: + print(f"模型加载失败: {e}") + # 在测试环境中，如果模型加载失败，可以考虑退出以避免运行一个无效的服务 + exit() + +# --- 3. 创建FastAPI应用 --- +app = FastAPI( + title="Reranker API (Test Version)", + description="一个返回 'score' 字段以测试Go客户端兼容性的API服务", + version="1.0.1" +) + +# --- 4. 定义API端点 --- +# --- 修改开始：将 response_model 指向新的测试用响应结构 --- +@app.post("/rerank", response_model=TestFinalResponse) # <--- 【关键修改点】response_model 改为 TestFinalResponse +def rerank_endpoint(request: RerankRequest): + # --- 修改结束 --- + + pairs = [[request.query, doc] for doc in request.documents] + + with torch.no_grad(): + inputs = tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=1024).to(device) + scores = model(**inputs, return_dict=True).logits.view(-1, ).float() + + # --- 修改开始：按照测试用的结构来构建结果 --- + results = [] + for i, (text, score_val) in enumerate(zip(request.documents, scores)): + + # 1. 创建嵌套的 document 对象 + doc_info = DocumentInfo(text=text) + + # 2. 创建 TestRankResult 对象 + # 注意字段名：index, document, score + test_result = TestRankResult( + index=i, + document=doc_info, + score=score_val.item() # <--- 【关键修改点】赋值给 "score" 字段 + ) + results.append(test_result) + + # 3. 排序 (key 也要相应修改为 score) + sorted_results = sorted(results, key=lambda x: x.score, reverse=True) + # --- 修改结束 --- + + # 返回一个字典，FastAPI 会根据 response_model (TestFinalResponse) 来验证和序列化它 + # 最终生成的 JSON 会是 {"results": [{"index": ..., "document": ..., "score": ...}]} + return {"results": sorted_results} + +@app.get("/") +def read_root(): + return {"status": "Reranker API (Test Version) is running"} + +# --- 5. 启动服务 --- +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=8000) + \ No newline at end of file diff --git a/scripts/build_images.sh b/scripts/build_images.sh new file mode 100755 index 0000000..0767d85 --- /dev/null +++ b/scripts/build_images.sh @@ -0,0 +1,350 @@ +#!/bin/bash +# 该脚本用于从源码构建WeKnora的所有Docker镜像 + +# 设置颜色 +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' # 无颜色 + +# 获取项目根目录（脚本所在目录的上一级） +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" + +# 版本信息 +VERSION="1.0.0" +SCRIPT_NAME=$(basename "$0") + +# 显示帮助信息 +show_help() { + echo -e "${GREEN}WeKnora 镜像构建脚本 v${VERSION}${NC}" + echo -e "${GREEN}用法:${NC} $0 [选项]" + echo "选项:" + echo " -h, --help 显示帮助信息" + echo " -a, --all 构建所有镜像（默认）" + echo " -p, --app 仅构建应用镜像" + echo " -d, --docreader 仅构建文档读取器镜像" + echo " -f, --frontend 仅构建前端镜像" + echo " -c, --clean 清理所有本地镜像" + echo " -v, --version 显示版本信息" + exit 0 +} + +# 显示版本信息 +show_version() { + echo -e "${GREEN}WeKnora 镜像构建脚本 v${VERSION}${NC}" + exit 0 +} + +# 日志函数 +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +# 检查Docker是否已安装 +check_docker() { + log_info "检查Docker环境..." + + if ! command -v docker &> /dev/null; then + log_error "未安装Docker，请先安装Docker" + return 1 + fi + + # 检查Docker服务运行状态 + if ! docker info &> /dev/null; then + log_error "Docker服务未运行，请启动Docker服务" + return 1 + fi + + log_success "Docker环境检查通过" + return 0 +} + +# 检测平台 +check_platform() { + log_info "检测系统平台信息..." + if [ "$(uname -m)" = "x86_64" ]; then + export PLATFORM="linux/amd64" + export TARGETARCH="amd64" + elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then + export PLATFORM="linux/arm64" + export TARGETARCH="arm64" + else + log_warning "未识别的平台类型：$(uname -m)，将使用默认平台 linux/amd64" + export PLATFORM="linux/amd64" + export TARGETARCH="amd64" + fi + log_info "当前平台：$PLATFORM" + log_info "当前架构：$TARGETARCH" +} + +# 获取版本信息 +get_version_info() { + # 从VERSION文件获取版本号 + if [ -f "VERSION" ]; then + VERSION=$(cat VERSION | tr -d '\n\r') + else + VERSION="unknown" + fi + + # 获取commit ID + if command -v git >/dev/null 2>&1; then + COMMIT_ID=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + else + COMMIT_ID="unknown" + fi + + # 获取构建时间 + BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC') + + # 获取Go版本 + if command -v go >/dev/null 2>&1; then + GO_VERSION=$(go version 2>/dev/null || echo "unknown") + else + GO_VERSION="unknown" + fi + + log_info "版本信息: $VERSION" + log_info "Commit ID: $COMMIT_ID" + log_info "构建时间: $BUILD_TIME" + log_info "Go版本: $GO_VERSION" +} + +# 构建应用镜像 +build_app_image() { + log_info "构建应用镜像 (weknora-app)..." + + cd "$PROJECT_ROOT" + + # 获取版本信息 + get_version_info + + docker build \ + --platform $PLATFORM \ + --build-arg GOPRIVATE_ARG=${GOPRIVATE:-""} \ + --build-arg GOPROXY_ARG=${GOPROXY:-"https://goproxy.cn,direct"} \ + --build-arg GOSUMDB_ARG=${GOSUMDB:-"off"} \ + --build-arg VERSION_ARG="$VERSION" \ + --build-arg COMMIT_ID_ARG="$COMMIT_ID" \ + --build-arg BUILD_TIME_ARG="$BUILD_TIME" \ + --build-arg GO_VERSION_ARG="$GO_VERSION" \ + -f docker/Dockerfile.app \ + -t wechatopenai/weknora-app:latest \ + . + + if [ $? -eq 0 ]; then + log_success "应用镜像构建成功" + return 0 + else + log_error "应用镜像构建失败" + return 1 + fi +} + +# 构建文档读取器镜像 +build_docreader_image() { + log_info "构建文档读取器镜像 (weknora-docreader)..." + + cd "$PROJECT_ROOT" + + docker build \ + --platform $PLATFORM \ + --build-arg PLATFORM=$PLATFORM \ + --build-arg TARGETARCH=$TARGETARCH \ + -f docker/Dockerfile.docreader \ + -t wechatopenai/weknora-docreader:latest \ + . + + if [ $? -eq 0 ]; then + log_success "文档读取器镜像构建成功" + return 0 + else + log_error "文档读取器镜像构建失败" + return 1 + fi +} + +# 构建前端镜像 +build_frontend_image() { + log_info "构建前端镜像 (weknora-ui)..." + + cd "$PROJECT_ROOT" + + docker build \ + --platform $PLATFORM \ + -f frontend/Dockerfile \ + -t wechatopenai/weknora-ui:latest \ + frontend/ + + if [ $? -eq 0 ]; then + log_success "前端镜像构建成功" + return 0 + else + log_error "前端镜像构建失败" + return 1 + fi +} + +# 构建所有镜像 +build_all_images() { + log_info "开始构建所有镜像..." + + local app_result=0 + local docreader_result=0 + local frontend_result=0 + + # 构建应用镜像 + build_app_image + app_result=$? + + # 构建文档读取器镜像 + build_docreader_image + docreader_result=$? + + # 构建前端镜像 + build_frontend_image + frontend_result=$? + + # 显示构建结果 + echo "" + log_info "=== 构建结果 ===" + if [ $app_result -eq 0 ]; then + log_success "✓ 应用镜像构建成功" + else + log_error "✗ 应用镜像构建失败" + fi + + if [ $docreader_result -eq 0 ]; then + log_success "✓ 文档读取器镜像构建成功" + else + log_error "✗ 文档读取器镜像构建失败" + fi + + if [ $frontend_result -eq 0 ]; then + log_success "✓ 前端镜像构建成功" + else + log_error "✗ 前端镜像构建失败" + fi + + if [ $app_result -eq 0 ] && [ $docreader_result -eq 0 ] && [ $frontend_result -eq 0 ]; then + log_success "所有镜像构建完成！" + return 0 + else + log_error "部分镜像构建失败" + return 1 + fi +} + +# 清理本地镜像 +clean_images() { + log_info "清理本地WeKnora镜像..." + + # 停止相关容器 + log_info "停止相关容器..." + docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-app:latest" 2>/dev/null) 2>/dev/null || true + docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-docreader:latest" 2>/dev/null) 2>/dev/null || true + docker stop $(docker ps -q --filter "ancestor=wechatopenai/weknora-ui:latest" 2>/dev/null) 2>/dev/null || true + + # 删除相关容器 + log_info "删除相关容器..." + docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-app:latest" 2>/dev/null) 2>/dev/null || true + docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-docreader:latest" 2>/dev/null) 2>/dev/null || true + docker rm $(docker ps -aq --filter "ancestor=wechatopenai/weknora-ui:latest" 2>/dev/null) 2>/dev/null || true + + # 删除镜像 + log_info "删除本地镜像..." + docker rmi wechatopenai/weknora-app:latest 2>/dev/null || true + docker rmi wechatopenai/weknora-docreader:latest 2>/dev/null || true + docker rmi wechatopenai/weknora-ui:latest 2>/dev/null || true + + docker image prune -f + + log_success "镜像清理完成" + return 0 +} + +# 解析命令行参数 +BUILD_ALL=false +BUILD_APP=false +BUILD_DOCREADER=false +BUILD_FRONTEND=false +CLEAN_IMAGES=false + +# 没有参数时默认构建所有镜像 +if [ $# -eq 0 ]; then + BUILD_ALL=true +fi + +while [ "$1" != "" ]; do + case $1 in + -h | --help ) show_help + ;; + -a | --all ) BUILD_ALL=true + ;; + -p | --app ) BUILD_APP=true + ;; + -d | --docreader ) BUILD_DOCREADER=true + ;; + -f | --frontend ) BUILD_FRONTEND=true + ;; + -c | --clean ) CLEAN_IMAGES=true + ;; + -v | --version ) show_version + ;; + * ) log_error "未知选项: $1" + show_help + ;; + esac + shift +done + +# 检查Docker环境 +check_docker +if [ $? -ne 0 ]; then + exit 1 +fi + +# 检测平台 +check_platform + +# 执行清理操作 +if [ "$CLEAN_IMAGES" = true ]; then + clean_images + exit $? +fi + +# 执行构建操作 +if [ "$BUILD_ALL" = true ]; then + build_all_images + exit $? +fi + +if [ "$BUILD_APP" = true ]; then + build_app_image + exit $? +fi + +if [ "$BUILD_DOCREADER" = true ]; then + build_docreader_image + exit $? +fi + +if [ "$BUILD_FRONTEND" = true ]; then + build_frontend_image + exit $? +fi + +exit 0 \ No newline at end of file diff --git a/scripts/get_version.sh b/scripts/get_version.sh new file mode 100755 index 0000000..12b459f --- /dev/null +++ b/scripts/get_version.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# 统一的版本信息获取脚本 +# 支持本地构建和CI构建环境 + +# 设置默认值 +VERSION="unknown" +COMMIT_ID="unknown" +BUILD_TIME="unknown" +GO_VERSION="unknown" + +# 获取版本号 +if [ -f "VERSION" ]; then + VERSION=$(cat VERSION | tr -d '\n\r') +fi + +# 获取commit ID +if [ -n "$GITHUB_SHA" ]; then + # GitHub Actions环境 + COMMIT_ID="${GITHUB_SHA:0:7}" +elif command -v git >/dev/null 2>&1; then + # 本地环境 + COMMIT_ID=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") +fi + +# 获取构建时间 +if [ -n "$GITHUB_ACTIONS" ]; then + # GitHub Actions环境，使用标准时间格式 + BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC') +else + # 本地环境 + BUILD_TIME=$(date -u '+%Y-%m-%d %H:%M:%S UTC') +fi + +# 获取Go版本 +if command -v go >/dev/null 2>&1; then + GO_VERSION=$(go version 2>/dev/null || echo "unknown") +fi + +# 根据参数输出不同格式 +case "${1:-env}" in + "env") + # 输出环境变量格式，对包含空格的值进行转义 + echo "VERSION=$VERSION" + echo "COMMIT_ID=$COMMIT_ID" + echo "BUILD_TIME=\"$BUILD_TIME\"" + echo "GO_VERSION=\"$GO_VERSION\"" + ;; + "json") + # 输出JSON格式 + cat << EOF +{ + "version": "$VERSION", + "commit_id": "$COMMIT_ID", + "build_time": "$BUILD_TIME", + "go_version": "$GO_VERSION" +} +EOF + ;; + "docker-args") + # 输出Docker构建参数格式 + echo "--build-arg VERSION_ARG=$VERSION" + echo "--build-arg COMMIT_ID_ARG=$COMMIT_ID" + echo "--build-arg BUILD_TIME_ARG=$BUILD_TIME" + echo "--build-arg GO_VERSION_ARG=$GO_VERSION" + ;; + "ldflags") + # 输出Go ldflags格式 + echo "-X 'github.com/Tencent/WeKnora/internal/handler.Version=$VERSION' -X 'github.com/Tencent/WeKnora/internal/handler.CommitID=$COMMIT_ID' -X 'github.com/Tencent/WeKnora/internal/handler.BuildTime=$BUILD_TIME' -X 'github.com/Tencent/WeKnora/internal/handler.GoVersion=$GO_VERSION'" + ;; + "info") + # 输出信息格式 + echo "版本信息: $VERSION" + echo "Commit ID: $COMMIT_ID" + echo "构建时间: $BUILD_TIME" + echo "Go版本: $GO_VERSION" + ;; + *) + echo "用法: $0 [env|json|docker-args|ldflags|info]" + echo " env - 输出环境变量格式 (默认)" + echo " json - 输出JSON格式" + echo " docker-args - 输出Docker构建参数格式" + echo " ldflags - 输出Go ldflags格式" + echo " info - 输出信息格式" + exit 1 + ;; +esac diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100644 index 0000000..97e35a2 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e + +# Database connection details (can be overridden by environment variables) +DB_HOST=${DB_HOST:-localhost} +DB_PORT=${DB_PORT:-5432} +DB_USER=${DB_USER:-postgres} +DB_PASSWORD=${DB_PASSWORD:-postgres} +DB_NAME=${DB_NAME:-WeKnora} +MIGRATIONS_DIR="/app/migrations" + +# Check if migrate tool is installed +if ! command -v migrate &> /dev/null; then + echo "Error: migrate tool is not installed" + echo "Install it with: go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest" + exit 1 +fi + +# Construct the database URL +DB_URL="postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable" + +# Execute migration based on command +case "$1" in + up) + echo "Running migrations up..." + echo "DB_URL: ${DB_URL}" + echo "DB_USER: ${DB_USER}" + echo "DB_PASSWORD: ${DB_PASSWORD}" + echo "DB_HOST: ${DB_HOST}" + echo "DB_PORT: ${DB_PORT}" + echo "DB_NAME: ${DB_NAME}" + echo "MIGRATIONS_DIR: ${MIGRATIONS_DIR}" + migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} up + ;; + down) + echo "Running migrations down..." + migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} down + ;; + create) + if [ -z "$2" ]; then + echo "Error: Migration name is required" + echo "Usage: $0 create " + exit 1 + fi + echo "Creating migration files for $2..." + migrate create -ext sql -dir ${MIGRATIONS_DIR} -seq $2 + ;; + *) + echo "Usage: $0 {up|down|create }" + exit 1 + ;; +esac + +echo "Migration command completed successfully" \ No newline at end of file diff --git a/scripts/start_all.sh b/scripts/start_all.sh new file mode 100755 index 0000000..f18f13e --- /dev/null +++ b/scripts/start_all.sh @@ -0,0 +1,729 @@ +#!/bin/bash +# 该脚本用于按需启动/停止Ollama和docker-compose服务 + +# 设置颜色 +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' # 无颜色 + +# 获取项目根目录（脚本所在目录的上一级） +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )" + +# 版本信息 +VERSION="1.0.1" # 版本更新 +SCRIPT_NAME=$(basename "$0") + +# 显示帮助信息 +show_help() { + printf "%b\n" "${GREEN}WeKnora 启动脚本 v${VERSION}${NC}" + printf "%b\n" "${GREEN}用法:${NC} $0 [选项]" + echo "选项:" + echo " -h, --help 显示帮助信息" + echo " -o, --ollama 启动Ollama服务" + echo " -d, --docker 启动Docker容器服务" + echo " -a, --all 启动所有服务（默认）" + echo " -s, --stop 停止所有服务" + echo " -c, --check 检查环境并诊断问题" + echo " -r, --restart 重新构建并重启指定容器" + echo " -l, --list 列出所有正在运行的容器" + echo " -p, --pull 拉取最新的Docker镜像" + echo " --no-pull 启动时不拉取镜像（默认会拉取）" + echo " -v, --version 显示版本信息" + exit 0 +} + +# 显示版本信息 +show_version() { + printf "%b\n" "${GREEN}WeKnora 启动脚本 v${VERSION}${NC}" + exit 0 +} + +# 日志函数 +log_info() { + printf "%b\n" "${BLUE}[INFO]${NC} $1" +} + +log_warning() { + printf "%b\n" "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + printf "%b\n" "${RED}[ERROR]${NC} $1" +} + +log_success() { + printf "%b\n" "${GREEN}[SUCCESS]${NC} $1" +} + +# 选择可用的 Docker Compose 命令（优先 docker compose，其次 docker-compose） +DOCKER_COMPOSE_BIN="" +DOCKER_COMPOSE_SUBCMD="" + +detect_compose_cmd() { + # 优先使用 Docker Compose 插件 + if docker compose version &> /dev/null; then + DOCKER_COMPOSE_BIN="docker" + DOCKER_COMPOSE_SUBCMD="compose" + return 0 + fi + + # 回退到 docker-compose (v1) + if command -v docker-compose &> /dev/null; then + if docker-compose version &> /dev/null; then + DOCKER_COMPOSE_BIN="docker-compose" + DOCKER_COMPOSE_SUBCMD="" + return 0 + fi + fi + + # 都不可用 + return 1 +} + +# 检查并创建.env文件 +check_env_file() { + log_info "检查环境变量配置..." + if [ ! -f "$PROJECT_ROOT/.env" ]; then + log_warning ".env 文件不存在，将从模板创建" + if [ -f "$PROJECT_ROOT/.env.example" ]; then + cp "$PROJECT_ROOT/.env.example" "$PROJECT_ROOT/.env" + log_success "已从 .env.example 创建 .env 文件" + else + log_error "未找到 .env.example 模板文件，无法创建 .env 文件" + return 1 + fi + else + log_info ".env 文件已存在" + fi + + # 检查必要的环境变量是否已设置 + source "$PROJECT_ROOT/.env" + local missing_vars=() + + # 检查基础变量 + if [ -z "$DB_DRIVER" ]; then missing_vars+=("DB_DRIVER"); fi + if [ -z "$STORAGE_TYPE" ]; then missing_vars+=("STORAGE_TYPE"); fi + + return 0 +} + +# 安装Ollama（根据平台不同采用不同方法） +install_ollama() { + # 检查是否为远程服务 + get_ollama_base_url + + if [ $IS_REMOTE -eq 1 ]; then + log_info "检测到远程Ollama服务配置，无需在本地安装Ollama" + return 0 + fi + + log_info "本地Ollama未安装，正在安装..." + + OS=$(uname) + if [ "$OS" = "Darwin" ]; then + # Mac安装方式 + log_info "检测到Mac系统，使用brew安装Ollama..." + if ! command -v brew &> /dev/null; then + # 通过安装包安装 + log_info "Homebrew未安装，使用直接下载方式..." + curl -fsSL https://ollama.com/download/Ollama-darwin.zip -o ollama.zip + unzip ollama.zip + mv ollama /usr/local/bin + rm ollama.zip + else + brew install ollama + fi + else + # Linux安装方式 + log_info "检测到Linux系统，使用安装脚本..." + curl -fsSL https://ollama.com/install.sh | sh + fi + + if [ $? -eq 0 ]; then + log_success "本地Ollama安装完成" + return 0 + else + log_error "本地Ollama安装失败" + return 1 + fi +} + +# 获取Ollama基础URL，检查是否为远程服务 +get_ollama_base_url() { + + check_env_file + + # 从环境变量获取Ollama基础URL + OLLAMA_URL=${OLLAMA_BASE_URL:-"http://host.docker.internal:11434"} + # 提取主机部分 + OLLAMA_HOST=$(echo "$OLLAMA_URL" | sed -E 's|^https?://||' | sed -E 's|:[0-9]+$||' | sed -E 's|/.*$||') + # 提取端口部分 + OLLAMA_PORT=$(echo "$OLLAMA_URL" | grep -oE ':[0-9]+' | grep -oE '[0-9]+' || echo "11434") + # 检查是否为localhost或127.0.0.1 + IS_REMOTE=0 + if [ "$OLLAMA_HOST" = "localhost" ] || [ "$OLLAMA_HOST" = "127.0.0.1" ] || [ "$OLLAMA_HOST" = "host.docker.internal" ]; then + IS_REMOTE=0 # 本地服务 + else + IS_REMOTE=1 # 远程服务 + fi +} + +# 启动Ollama服务 +start_ollama() { + log_info "正在检查Ollama服务..." + # 提取主机和端口 + get_ollama_base_url + log_info "Ollama服务地址: $OLLAMA_URL" + + if [ $IS_REMOTE -eq 1 ]; then + log_info "检测到远程Ollama服务，将直接使用远程服务，不进行本地安装和启动" + # 检查远程服务是否可用 + if curl -s "$OLLAMA_URL/api/tags" &> /dev/null; then + log_success "远程Ollama服务可访问" + return 0 + else + log_warning "远程Ollama服务不可访问，请确认服务地址正确且已启动" + return 1 + fi + fi + + # 以下为本地服务的处理 + # 检查Ollama是否已安装 + if ! command -v ollama &> /dev/null; then + install_ollama + if [ $? -ne 0 ]; then + return 1 + fi + fi + + # 检查Ollama服务是否已运行 + if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then + log_success "本地Ollama服务已经在运行，端口：$OLLAMA_PORT" + else + log_info "启动本地Ollama服务..." + # 注意：官方推荐使用 systemctl 或 launchctl 管理服务，直接后台运行仅用于临时场景 + systemctl restart ollama || (ollama serve > /dev/null 2>&1 < /dev/null &) + + # 等待服务启动 + MAX_RETRIES=30 + COUNT=0 + while [ $COUNT -lt $MAX_RETRIES ]; do + if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then + log_success "本地Ollama服务已成功启动，端口：$OLLAMA_PORT" + break + fi + echo -ne "等待Ollama服务启动... ($COUNT/$MAX_RETRIES)\r" + sleep 1 + COUNT=$((COUNT + 1)) + done + echo "" # 换行 + + if [ $COUNT -eq $MAX_RETRIES ]; then + log_error "本地Ollama服务启动失败" + return 1 + fi + fi + + log_success "本地Ollama服务地址: http://localhost:$OLLAMA_PORT" + return 0 +} + +# 停止Ollama服务 +stop_ollama() { + log_info "正在停止Ollama服务..." + + # 检查是否为远程服务 + get_ollama_base_url + + if [ $IS_REMOTE -eq 1 ]; then + log_info "检测到远程Ollama服务，无需在本地停止" + return 0 + fi + + # 检查Ollama是否已安装 + if ! command -v ollama &> /dev/null; then + log_info "本地Ollama未安装，无需停止" + return 0 + fi + + # 查找并终止Ollama进程 + if pgrep -x "ollama" > /dev/null; then + # 优先使用systemctl + if command -v systemctl &> /dev/null; then + sudo systemctl stop ollama + else + pkill -f "ollama serve" + fi + log_success "本地Ollama服务已停止" + else + log_info "本地Ollama服务未运行" + fi + + return 0 +} + +# 检查Docker是否已安装 +check_docker() { + log_info "检查Docker环境..." + + if ! command -v docker &> /dev/null; then + log_error "未安装Docker，请先安装Docker" + return 1 + fi + + # 检查并选择可用的 Docker Compose 命令 + if detect_compose_cmd; then + if [ "$DOCKER_COMPOSE_BIN" = "docker" ]; then + log_info "已检测到 Docker Compose 插件 (docker compose)" + else + log_info "已检测到 docker-compose (v1)" + fi + else + log_error "未检测到 Docker Compose（既没有 docker compose 也没有 docker-compose）。请安装其中之一。" + return 1 + fi + + # 检查Docker服务运行状态 + if ! docker info &> /dev/null; then + log_error "Docker服务未运行，请启动Docker服务" + return 1 + fi + + log_success "Docker环境检查通过" + return 0 +} + +check_platform() { + # 检测当前系统平台 + log_info "检测系统平台信息..." + if [ "$(uname -m)" = "x86_64" ]; then + export PLATFORM="linux/amd64" + elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then + export PLATFORM="linux/arm64" + else + log_warning "未识别的平台类型：$(uname -m)，将使用默认平台 linux/amd64" + export PLATFORM="linux/amd64" + fi + log_info "当前平台：$PLATFORM" +} + +# 启动Docker容器 +start_docker() { + log_info "正在启动Docker容器..." + + # 检查Docker环境 + check_docker + if [ $? -ne 0 ]; then + return 1 + fi + + # 检查.env文件 + check_env_file + + # 读取.env文件 + source "$PROJECT_ROOT/.env" + storage_type=${STORAGE_TYPE:-local} + + check_platform + + # 进入项目根目录再执行docker-compose命令 + cd "$PROJECT_ROOT" + + # 启动基本服务 + log_info "启动核心服务容器..." + # 统一通过已检测到的 Compose 命令启动 + if [ "$NO_PULL" = true ]; then + # 不拉取镜像，使用本地镜像 + log_info "跳过镜像拉取，使用本地镜像..." + PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up --build -d + else + # 拉取最新镜像 + log_info "拉取最新镜像..." + PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up --build --pull always -d + fi + if [ $? -ne 0 ]; then + log_error "Docker容器启动失败" + return 1 + fi + + log_success "所有Docker容器已成功启动" + + # 显示容器状态 + log_info "当前容器状态:" + "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps + + return 0 +} + +# 停止Docker容器 +stop_docker() { + log_info "正在停止Docker容器..." + + # 检查Docker环境 + check_docker + if [ $? -ne 0 ]; then + # 即使检查失败也尝试停止，以防万一 + log_warning "Docker环境检查失败，仍将尝试停止容器..." + fi + + # 进入项目根目录再执行docker-compose命令 + cd "$PROJECT_ROOT" + + # 停止所有容器 + "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD down --remove-orphans + if [ $? -ne 0 ]; then + log_error "Docker容器停止失败" + return 1 + fi + + log_success "所有Docker容器已停止" + return 0 +} + +# 列出所有正在运行的容器 +list_containers() { + log_info "列出所有正在运行的容器..." + + # 检查Docker环境 + check_docker + if [ $? -ne 0 ]; then + return 1 + fi + + # 进入项目根目录再执行docker-compose命令 + cd "$PROJECT_ROOT" + + # 列出所有容器 + printf "%b\n" "${BLUE}当前正在运行的容器:${NC}" + "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps --services | sort + + return 0 +} + +# 拉取最新的Docker镜像 +pull_images() { + log_info "正在拉取最新的Docker镜像..." + + # 检查Docker环境 + check_docker + if [ $? -ne 0 ]; then + return 1 + fi + + # 检查.env文件 + check_env_file + + # 读取.env文件 + source "$PROJECT_ROOT/.env" + storage_type=${STORAGE_TYPE:-local} + + check_platform + + # 进入项目根目录再执行docker-compose命令 + cd "$PROJECT_ROOT" + + # 拉取所有镜像 + log_info "拉取所有服务的最新镜像..." + PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD pull + if [ $? -ne 0 ]; then + log_error "镜像拉取失败" + return 1 + fi + + log_success "所有镜像已成功拉取到最新版本" + + # 显示拉取的镜像信息 + log_info "已拉取的镜像:" + docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.CreatedAt}}\t{{.Size}}" | head -10 + + return 0 +} + +# 重启指定容器 +restart_container() { + local container_name="$1" + + if [ -z "$container_name" ]; then + log_error "未指定容器名称" + echo "可用的容器有:" + list_containers + return 1 + fi + + log_info "正在重新构建并重启容器: $container_name" + + # 检查Docker环境 + check_docker + if [ $? -ne 0 ]; then + return 1 + fi + + check_platform + + # 进入项目根目录再执行docker-compose命令 + cd "$PROJECT_ROOT" + + # 检查容器是否存在 + if ! "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD ps --services | grep -q "^$container_name$"; then + log_error "容器 '$container_name' 不存在或未运行" + echo "可用的容器有:" + list_containers + return 1 + fi + + # 构建并重启容器 + log_info "正在重新构建容器 '$container_name'..." + PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD build "$container_name" + if [ $? -ne 0 ]; then + log_error "容器 '$container_name' 构建失败" + return 1 + fi + + log_info "正在重启容器 '$container_name'..." + PLATFORM=$PLATFORM "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD up -d --no-deps "$container_name" + if [ $? -ne 0 ]; then + log_error "容器 '$container_name' 重启失败" + return 1 + fi + + log_success "容器 '$container_name' 已成功重新构建并重启" + return 0 +} + +# 检查系统环境 +check_environment() { + log_info "开始环境检查..." + + # 检查操作系统 + OS=$(uname) + log_info "操作系统: $OS" + + # 检查Docker + check_docker + + # 检查.env文件 + check_env_file + + get_ollama_base_url + + if [ $IS_REMOTE -eq 1 ]; then + log_info "检测到远程Ollama服务配置" + if curl -s "$OLLAMA_URL/api/tags" &> /dev/null; then + version=$(curl -s "$OLLAMA_URL/api/tags" | grep -o '"version":"[^"]*"' | cut -d'"' -f4) + log_success "远程Ollama服务可访问，版本: $version" + else + log_warning "远程Ollama服务不可访问，请确认服务地址正确且已启动" + fi + else + if command -v ollama &> /dev/null; then + log_success "本地Ollama已安装" + if curl -s "http://localhost:$OLLAMA_PORT/api/tags" &> /dev/null; then + version=$(curl -s "http://localhost:$OLLAMA_PORT/api/tags" | grep -o '"version":"[^"]*"' | cut -d'"' -f4) + log_success "本地Ollama服务正在运行，版本: $version" + else + log_warning "本地Ollama已安装但服务未运行" + fi + else + log_warning "本地Ollama未安装" + fi + fi + + # 检查磁盘空间 + log_info "检查磁盘空间..." + df -h | grep -E "(Filesystem|/$)" + + # 检查内存 + log_info "检查内存使用情况..." + if [ "$OS" = "Darwin" ]; then + vm_stat | perl -ne '/page size of (\d+)/ and $size=$1; /Pages free: (\d+)/ and print "Free Memory: ", $1 * $size / 1048576, " MB\n"' + else + free -h | grep -E "(total|Mem:)" + fi + + # 检查CPU + log_info "CPU信息:" + if [ "$OS" = "Darwin" ]; then + sysctl -n machdep.cpu.brand_string + echo "CPU核心数: $(sysctl -n hw.ncpu)" + else + grep "model name" /proc/cpuinfo | head -1 + echo "CPU核心数: $(nproc)" + fi + + # 检查容器状态 + log_info "检查容器状态..." + if docker info &> /dev/null; then + docker ps -a + else + log_warning "无法获取容器状态，Docker可能未运行" + fi + + log_success "环境检查完成" + return 0 +} + +# 解析命令行参数 +START_OLLAMA=false +START_DOCKER=false +STOP_SERVICES=false +CHECK_ENVIRONMENT=false +LIST_CONTAINERS=false +RESTART_CONTAINER=false +PULL_IMAGES=false +NO_PULL=false +CONTAINER_NAME="" + +# 没有参数时默认启动所有服务 +if [ $# -eq 0 ]; then + START_OLLAMA=true + START_DOCKER=true +fi + +while [ "$1" != "" ]; do + case $1 in + -h | --help ) show_help + ;; + -o | --ollama ) START_OLLAMA=true + ;; + -d | --docker ) START_DOCKER=true + ;; + -a | --all ) START_OLLAMA=true + START_DOCKER=true + ;; + -s | --stop ) STOP_SERVICES=true + ;; + -c | --check ) CHECK_ENVIRONMENT=true + ;; + -l | --list ) LIST_CONTAINERS=true + ;; + -p | --pull ) PULL_IMAGES=true + ;; + --no-pull ) NO_PULL=true + START_OLLAMA=true + START_DOCKER=true + ;; + -r | --restart ) RESTART_CONTAINER=true + CONTAINER_NAME="$2" + shift + ;; + -v | --version ) show_version + ;; + * ) log_error "未知选项: $1" + show_help + ;; + esac + shift +done + +# 执行环境检查 +if [ "$CHECK_ENVIRONMENT" = true ]; then + check_environment + exit $? +fi + +# 列出所有容器 +if [ "$LIST_CONTAINERS" = true ]; then + list_containers + exit $? +fi + +# 拉取最新镜像 +if [ "$PULL_IMAGES" = true ]; then + pull_images + exit $? +fi + +# 重启指定容器 +if [ "$RESTART_CONTAINER" = true ]; then + restart_container "$CONTAINER_NAME" + exit $? +fi + +# 执行服务操作 +if [ "$STOP_SERVICES" = true ]; then + # 停止服务 + stop_ollama + OLLAMA_RESULT=$? + + stop_docker + DOCKER_RESULT=$? + + # 显示总结 + echo "" + log_info "=== 停止结果 ===" + if [ $OLLAMA_RESULT -eq 0 ]; then + log_success "✓ Ollama服务已停止" + else + log_error "✗ Ollama服务停止失败" + fi + + if [ $DOCKER_RESULT -eq 0 ]; then + log_success "✓ Docker容器已停止" + else + log_error "✗ Docker容器停止失败" + fi + + log_success "服务停止完成。" +else + # 启动服务 + OLLAMA_RESULT=1 + DOCKER_RESULT=1 + if [ "$START_OLLAMA" = true ]; then + start_ollama + OLLAMA_RESULT=$? + fi + + if [ "$START_DOCKER" = true ]; then + start_docker + DOCKER_RESULT=$? + fi + + # 显示总结 + echo "" + log_info "=== 启动结果 ===" + if [ "$START_OLLAMA" = true ]; then + if [ $OLLAMA_RESULT -eq 0 ]; then + log_success "✓ Ollama服务已启动" + else + log_error "✗ Ollama服务启动失败" + fi + fi + + if [ "$START_DOCKER" = true ]; then + if [ $DOCKER_RESULT -eq 0 ]; then + log_success "✓ Docker容器已启动" + else + log_error "✗ Docker容器启动失败" + fi + fi + + if [ "$START_OLLAMA" = true ] && [ "$START_DOCKER" = true ]; then + if [ $OLLAMA_RESULT -eq 0 ] && [ $DOCKER_RESULT -eq 0 ]; then + log_success "所有服务启动完成，可通过以下地址访问:" + printf "%b\n" "${GREEN} - 前端界面: http://localhost:${FRONTEND_PORT:-80}${NC}" + printf "%b\n" "${GREEN} - API接口: http://localhost:${APP_PORT:-8080}${NC}" + printf "%b\n" "${GREEN} - Jaeger链路追踪: http://localhost:16686${NC}" + echo "" + log_info "正在持续输出容器日志（按 Ctrl+C 退出日志，容器不会停止）..." + "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD logs app docreader postgres --since=10s -f + else + log_error "部分服务启动失败，请检查日志并修复问题" + fi + elif [ "$START_OLLAMA" = true ] && [ $OLLAMA_RESULT -eq 0 ]; then + log_success "Ollama服务启动完成，可通过以下地址访问:" + printf "%b\n" "${GREEN} - Ollama API: http://localhost:$OLLAMA_PORT${NC}" + elif [ "$START_DOCKER" = true ] && [ $DOCKER_RESULT -eq 0 ]; then + log_success "Docker容器启动完成，可通过以下地址访问:" + printf "%b\n" "${GREEN} - 前端界面: http://localhost:${FRONTEND_PORT:-80}${NC}" + printf "%b\n" "${GREEN} - API接口: http://localhost:${APP_PORT:-8080}${NC}" + printf "%b\n" "${GREEN} - Jaeger链路追踪: http://localhost:16686${NC}" + echo "" + log_info "正在持续输出容器日志（按 Ctrl+C 退出日志，容器不会停止）..." + "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD logs app docreader postgres --since=10s -f + fi +fi + +exit 0 \ No newline at end of file diff --git a/services/docreader/Makefile b/services/docreader/Makefile new file mode 100644 index 0000000..b1ffb69 --- /dev/null +++ b/services/docreader/Makefile @@ -0,0 +1,23 @@ +.PHONY: proto build run docker-build docker-run clean + +# 生成 protobuf 代码 +proto: + @echo "Generating protobuf code..." + @sh ./scripts/generate_proto.sh + +# 构建 Go 客户端 +build: + @echo "Building Go client..." + @go build -o bin/client ./src/client + +# 运行 Python 服务 +run: + @echo "Running Python server..." + @python src/server/server.py + +# 清理 +clean: + @echo "Cleaning up..." + @rm -rf bin/ + @find . -name "*.pyc" -delete + @find . -name "__pycache__" -delete \ No newline at end of file diff --git a/services/docreader/poetry.lock b/services/docreader/poetry.lock new file mode 100644 index 0000000..28e0f11 --- /dev/null +++ b/services/docreader/poetry.lock @@ -0,0 +1,2228 @@ +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. + +[[package]] +name = "albucore" +version = "0.0.23" +description = "High-performance image processing functions for deep learning and computer vision." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "albucore-0.0.23-py3-none-any.whl", hash = "sha256:99274ac0c15a1a7d9a726df9d54d5ab70d9d0c189e2a935399dba3d4bafad415"}, + {file = "albucore-0.0.23.tar.gz", hash = "sha256:57823982b954913b84a9e2cf71058c4577b02397a62c41885be2d9b295efa8ab"}, +] + +[package.dependencies] +numpy = ">=1.24.4" +opencv-python-headless = ">=4.9.0.80" +simsimd = ">=5.9.2" +stringzilla = ">=3.10.4" + +[[package]] +name = "albumentations" +version = "2.0.5" +description = "Fast, flexible, and advanced augmentation library for deep learning, computer vision, and medical imaging. Albumentations offers a wide range of transformations for both 2D (images, masks, bboxes, keypoints) and 3D (volumes, volumetric masks, keypoints) data, with optimized performance and seamless integration into ML workflows." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "albumentations-2.0.5-py3-none-any.whl", hash = "sha256:1fc253942d34dd7c07652bf6511049c8bb7d522baec7f1fe355df16293c3c7b6"}, + {file = "albumentations-2.0.5.tar.gz", hash = "sha256:e19e1c0f14c903c3c230f3d83f14814b84f1180393189bf96779f653031f3278"}, +] + +[package.dependencies] +albucore = "0.0.23" +numpy = ">=1.24.4" +opencv-python-headless = ">=4.9.0.80" +pydantic = ">=2.9.2" +PyYAML = "*" +scipy = ">=1.10.0" + +[package.extras] +hub = ["huggingface-hub"] +pytorch = ["torch"] +text = ["pillow"] + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anyio" +version = "4.9.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c"}, + {file = "anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +doc = ["Sphinx (>=8.2,<9.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "blockbuster (>=1.5.23)", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""] +trio = ["trio (>=0.26.1)"] + +[[package]] +name = "astor" +version = "0.8.1" +description = "Read/rewrite/write Python ASTs" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] +files = [ + {file = "astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5"}, + {file = "astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e"}, +] + +[[package]] +name = "beautifulsoup4" +version = "4.13.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.7.0" +groups = ["main"] +files = [ + {file = "beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16"}, + {file = "beautifulsoup4-4.13.3.tar.gz", hash = "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b"}, +] + +[package.dependencies] +soupsieve = ">1.2" +typing-extensions = ">=4.0.0" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "certifi" +version = "2025.1.31" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, + {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win32.whl", hash = "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765"}, + {file = "charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85"}, + {file = "charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "platform_system == \"Windows\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cython" +version = "3.0.12" +description = "The Cython compiler for writing C extensions in the Python language." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] +files = [ + {file = "Cython-3.0.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ba67eee9413b66dd9fbacd33f0bc2e028a2a120991d77b5fd4b19d0b1e4039b9"}, + {file = "Cython-3.0.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bee2717e5b5f7d966d0c6e27d2efe3698c357aa4d61bb3201997c7a4f9fe485a"}, + {file = "Cython-3.0.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7cffc3464f641c8d0dda942c7c53015291beea11ec4d32421bed2f13b386b819"}, + {file = "Cython-3.0.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d3a8f81980ffbd74e52f9186d8f1654e347d0c44bfea6b5997028977f481a179"}, + {file = "Cython-3.0.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8d32856716c369d01f2385ad9177cdd1a11079ac89ea0932dc4882de1aa19174"}, + {file = "Cython-3.0.12-cp310-cp310-win32.whl", hash = "sha256:712c3f31adec140dc60d064a7f84741f50e2c25a8edd7ae746d5eb4d3ef7072a"}, + {file = "Cython-3.0.12-cp310-cp310-win_amd64.whl", hash = "sha256:d6945694c5b9170cfbd5f2c0d00ef7487a2de7aba83713a64ee4ebce7fad9e05"}, + {file = "Cython-3.0.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:feb86122a823937cc06e4c029d80ff69f082ebb0b959ab52a5af6cdd271c5dc3"}, + {file = "Cython-3.0.12-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfdbea486e702c328338314adb8e80f5f9741f06a0ae83aaec7463bc166d12e8"}, + {file = "Cython-3.0.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:563de1728c8e48869d2380a1b76bbc1b1b1d01aba948480d68c1d05e52d20c92"}, + {file = "Cython-3.0.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398d4576c1e1f6316282aa0b4a55139254fbed965cba7813e6d9900d3092b128"}, + {file = "Cython-3.0.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1e5eadef80143026944ea8f9904715a008f5108d1d644a89f63094cc37351e73"}, + {file = "Cython-3.0.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5a93cbda00a5451175b97dea5a9440a3fcee9e54b4cba7a7dbcba9a764b22aec"}, + {file = "Cython-3.0.12-cp311-cp311-win32.whl", hash = "sha256:3109e1d44425a2639e9a677b66cd7711721a5b606b65867cb2d8ef7a97e2237b"}, + {file = "Cython-3.0.12-cp311-cp311-win_amd64.whl", hash = "sha256:d4b70fc339adba1e2111b074ee6119fe9fd6072c957d8597bce9a0dd1c3c6784"}, + {file = "Cython-3.0.12-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fe030d4a00afb2844f5f70896b7f2a1a0d7da09bf3aa3d884cbe5f73fff5d310"}, + {file = "Cython-3.0.12-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7fec4f052b8fe173fe70eae75091389955b9a23d5cec3d576d21c5913b49d47"}, + {file = "Cython-3.0.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0faa5e39e5c8cdf6f9c3b1c3f24972826e45911e7f5b99cf99453fca5432f45e"}, + {file = "Cython-3.0.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d53de996ed340e9ab0fc85a88aaa8932f2591a2746e1ab1c06e262bd4ec4be7"}, + {file = "Cython-3.0.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ea3a0e19ab77266c738aa110684a753a04da4e709472cadeff487133354d6ab8"}, + {file = "Cython-3.0.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c151082884be468f2f405645858a857298ac7f7592729e5b54788b5c572717ba"}, + {file = "Cython-3.0.12-cp312-cp312-win32.whl", hash = "sha256:3083465749911ac3b2ce001b6bf17f404ac9dd35d8b08469d19dc7e717f5877a"}, + {file = "Cython-3.0.12-cp312-cp312-win_amd64.whl", hash = "sha256:c0b91c7ebace030dd558ea28730de8c580680b50768e5af66db2904a3716c3e3"}, + {file = "Cython-3.0.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4ee6f1ea1bead8e6cbc4e64571505b5d8dbdb3b58e679d31f3a84160cebf1a1a"}, + {file = "Cython-3.0.12-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57aefa6d3341109e46ec1a13e3a763aaa2cbeb14e82af2485b318194be1d9170"}, + {file = "Cython-3.0.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:879ae9023958d63c0675015369384642d0afb9c9d1f3473df9186c42f7a9d265"}, + {file = "Cython-3.0.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:36fcd584dae547de6f095500a380f4a0cce72b7a7e409e9ff03cb9beed6ac7a1"}, + {file = "Cython-3.0.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62b79dcc0de49efe9e84b9d0e2ae0a6fc9b14691a65565da727aa2e2e63c6a28"}, + {file = "Cython-3.0.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4aa255781b093a8401109d8f2104bbb2e52de7639d5896aefafddc85c30e0894"}, + {file = "Cython-3.0.12-cp313-cp313-win32.whl", hash = "sha256:77d48f2d4bab9fe1236eb753d18f03e8b2619af5b6f05d51df0532a92dfb38ab"}, + {file = "Cython-3.0.12-cp313-cp313-win_amd64.whl", hash = "sha256:86c304b20bd57c727c7357e90d5ba1a2b6f1c45492de2373814d7745ef2e63b4"}, + {file = "Cython-3.0.12-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ff5c0b6a65b08117d0534941d404833d516dac422eee88c6b4fd55feb409a5ed"}, + {file = "Cython-3.0.12-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:680f1d6ed4436ae94805db264d6155ed076d2835d84f20dcb31a7a3ad7f8668c"}, + {file = "Cython-3.0.12-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc24609613fa06d0d896309f7164ba168f7e8d71c1e490ed2a08d23351c3f41"}, + {file = "Cython-3.0.12-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1879c073e2b34924ce9b7ca64c212705dcc416af4337c45f371242b2e5f6d32"}, + {file = "Cython-3.0.12-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:bfb75123dd4ff767baa37d7036da0de2dfb6781ff256eef69b11b88b9a0691d1"}, + {file = "Cython-3.0.12-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:f39640f8df0400cde6882e23c734f15bb8196de0a008ae5dc6c8d1ec5957d7c8"}, + {file = "Cython-3.0.12-cp36-cp36m-win32.whl", hash = "sha256:8c9efe9a0895abee3cadfdad4130b30f7b5e57f6e6a51ef2a44f9fc66a913880"}, + {file = "Cython-3.0.12-cp36-cp36m-win_amd64.whl", hash = "sha256:63d840f2975e44d74512f8f34f1f7cb8121c9428e26a3f6116ff273deb5e60a2"}, + {file = "Cython-3.0.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:75c5acd40b97cff16fadcf6901a91586cbca5dcdba81f738efaf1f4c6bc8dccb"}, + {file = "Cython-3.0.12-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e62564457851db1c40399bd95a5346b9bb99e17a819bf583b362f418d8f3457a"}, + {file = "Cython-3.0.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ccd1228cc203b1f1b8a3d403f5a20ad1c40e5879b3fbf5851ce09d948982f2c"}, + {file = "Cython-3.0.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25529ee948f44d9a165ff960c49d4903267c20b5edf2df79b45924802e4cca6e"}, + {file = "Cython-3.0.12-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:90cf599372c5a22120609f7d3a963f17814799335d56dd0dcf8fe615980a8ae1"}, + {file = "Cython-3.0.12-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9f8c48748a9c94ea5d59c26ab49ad0fad514d36f894985879cf3c3ca0e600bf4"}, + {file = "Cython-3.0.12-cp37-cp37m-win32.whl", hash = "sha256:3e4fa855d98bc7bd6a2049e0c7dc0dcf595e2e7f571a26e808f3efd84d2db374"}, + {file = "Cython-3.0.12-cp37-cp37m-win_amd64.whl", hash = "sha256:120681093772bf3600caddb296a65b352a0d3556e962b9b147efcfb8e8c9801b"}, + {file = "Cython-3.0.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:731d719423e041242c9303c80cae4327467299b90ffe62d4cc407e11e9ea3160"}, + {file = "Cython-3.0.12-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3238a29f37999e27494d120983eca90d14896b2887a0bd858a381204549137a"}, + {file = "Cython-3.0.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b588c0a089a9f4dd316d2f9275230bad4a7271e5af04e1dc41d2707c816be44b"}, + {file = "Cython-3.0.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ab9f5198af74eb16502cc143cdde9ca1cbbf66ea2912e67440dd18a36e3b5fa"}, + {file = "Cython-3.0.12-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8ee841c0e114efa1e849c281ac9b8df8aa189af10b4a103b1c5fd71cbb799679"}, + {file = "Cython-3.0.12-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:43c48b5789398b228ea97499f5b864843ba9b1ab837562a9227c6f58d16ede8b"}, + {file = "Cython-3.0.12-cp38-cp38-win32.whl", hash = "sha256:5e5f17c48a4f41557fbcc7ee660ccfebe4536a34c557f553b6893c1b3c83df2d"}, + {file = "Cython-3.0.12-cp38-cp38-win_amd64.whl", hash = "sha256:309c081057930bb79dc9ea3061a1af5086c679c968206e9c9c2ec90ab7cb471a"}, + {file = "Cython-3.0.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54115fcc126840926ff3b53cfd2152eae17b3522ae7f74888f8a41413bd32f25"}, + {file = "Cython-3.0.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:629db614b9c364596d7c975fa3fb3978e8c5349524353dbe11429896a783fc1e"}, + {file = "Cython-3.0.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:af081838b0f9e12a83ec4c3809a00a64c817f489f7c512b0e3ecaf5f90a2a816"}, + {file = "Cython-3.0.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:34ce459808f7d8d5d4007bc5486fe50532529096b43957af6cbffcb4d9cc5c8d"}, + {file = "Cython-3.0.12-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d6c6cd6a75c8393e6805d17f7126b96a894f310a1a9ea91c47d141fb9341bfa8"}, + {file = "Cython-3.0.12-cp39-cp39-win32.whl", hash = "sha256:a4032e48d4734d2df68235d21920c715c451ac9de15fa14c71b378e8986b83be"}, + {file = "Cython-3.0.12-cp39-cp39-win_amd64.whl", hash = "sha256:dcdc3e5d4ce0e7a4af6903ed580833015641e968d18d528d8371e2435a34132c"}, + {file = "Cython-3.0.12-py2.py3-none-any.whl", hash = "sha256:0038c9bae46c459669390e53a1ec115f8096b2e4647ae007ff1bf4e6dee92806"}, + {file = "cython-3.0.12.tar.gz", hash = "sha256:b988bb297ce76c671e28c97d017b95411010f7c77fa6623dd0bb47eed1aee1bc"}, +] + +[[package]] +name = "decorator" +version = "5.2.1" +description = "Decorators for Humans" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a"}, + {file = "decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360"}, +] + +[[package]] +name = "fire" +version = "0.7.0" +description = "A library for automatically generating command line interfaces." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "fire-0.7.0.tar.gz", hash = "sha256:961550f07936eaf65ad1dc8360f2b2bf8408fad46abbfa4d2a3794f8d2a95cdf"}, +] + +[package.dependencies] +termcolor = "*" + +[[package]] +name = "fonttools" +version = "4.57.0" +description = "Tools to manipulate font files" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "fonttools-4.57.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:babe8d1eb059a53e560e7bf29f8e8f4accc8b6cfb9b5fd10e485bde77e71ef41"}, + {file = "fonttools-4.57.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:81aa97669cd726349eb7bd43ca540cf418b279ee3caba5e2e295fb4e8f841c02"}, + {file = "fonttools-4.57.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0e9618630edd1910ad4f07f60d77c184b2f572c8ee43305ea3265675cbbfe7e"}, + {file = "fonttools-4.57.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34687a5d21f1d688d7d8d416cb4c5b9c87fca8a1797ec0d74b9fdebfa55c09ab"}, + {file = "fonttools-4.57.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:69ab81b66ebaa8d430ba56c7a5f9abe0183afefd3a2d6e483060343398b13fb1"}, + {file = "fonttools-4.57.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d639397de852f2ccfb3134b152c741406752640a266d9c1365b0f23d7b88077f"}, + {file = "fonttools-4.57.0-cp310-cp310-win32.whl", hash = "sha256:cc066cb98b912f525ae901a24cd381a656f024f76203bc85f78fcc9e66ae5aec"}, + {file = "fonttools-4.57.0-cp310-cp310-win_amd64.whl", hash = "sha256:7a64edd3ff6a7f711a15bd70b4458611fb240176ec11ad8845ccbab4fe6745db"}, + {file = "fonttools-4.57.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3871349303bdec958360eedb619169a779956503ffb4543bb3e6211e09b647c4"}, + {file = "fonttools-4.57.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c59375e85126b15a90fcba3443eaac58f3073ba091f02410eaa286da9ad80ed8"}, + {file = "fonttools-4.57.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:967b65232e104f4b0f6370a62eb33089e00024f2ce143aecbf9755649421c683"}, + {file = "fonttools-4.57.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39acf68abdfc74e19de7485f8f7396fa4d2418efea239b7061d6ed6a2510c746"}, + {file = "fonttools-4.57.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9d077f909f2343daf4495ba22bb0e23b62886e8ec7c109ee8234bdbd678cf344"}, + {file = "fonttools-4.57.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:46370ac47a1e91895d40e9ad48effbe8e9d9db1a4b80888095bc00e7beaa042f"}, + {file = "fonttools-4.57.0-cp311-cp311-win32.whl", hash = "sha256:ca2aed95855506b7ae94e8f1f6217b7673c929e4f4f1217bcaa236253055cb36"}, + {file = "fonttools-4.57.0-cp311-cp311-win_amd64.whl", hash = "sha256:17168a4670bbe3775f3f3f72d23ee786bd965395381dfbb70111e25e81505b9d"}, + {file = "fonttools-4.57.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:889e45e976c74abc7256d3064aa7c1295aa283c6bb19810b9f8b604dfe5c7f31"}, + {file = "fonttools-4.57.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0425c2e052a5f1516c94e5855dbda706ae5a768631e9fcc34e57d074d1b65b92"}, + {file = "fonttools-4.57.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44c26a311be2ac130f40a96769264809d3b0cb297518669db437d1cc82974888"}, + {file = "fonttools-4.57.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84c41ba992df5b8d680b89fd84c6a1f2aca2b9f1ae8a67400c8930cd4ea115f6"}, + {file = "fonttools-4.57.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ea1e9e43ca56b0c12440a7c689b1350066595bebcaa83baad05b8b2675129d98"}, + {file = "fonttools-4.57.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:84fd56c78d431606332a0627c16e2a63d243d0d8b05521257d77c6529abe14d8"}, + {file = "fonttools-4.57.0-cp312-cp312-win32.whl", hash = "sha256:f4376819c1c778d59e0a31db5dc6ede854e9edf28bbfa5b756604727f7f800ac"}, + {file = "fonttools-4.57.0-cp312-cp312-win_amd64.whl", hash = "sha256:57e30241524879ea10cdf79c737037221f77cc126a8cdc8ff2c94d4a522504b9"}, + {file = "fonttools-4.57.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:408ce299696012d503b714778d89aa476f032414ae57e57b42e4b92363e0b8ef"}, + {file = "fonttools-4.57.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:bbceffc80aa02d9e8b99f2a7491ed8c4a783b2fc4020119dc405ca14fb5c758c"}, + {file = "fonttools-4.57.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f022601f3ee9e1f6658ed6d184ce27fa5216cee5b82d279e0f0bde5deebece72"}, + {file = "fonttools-4.57.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dea5893b58d4637ffa925536462ba626f8a1b9ffbe2f5c272cdf2c6ebadb817"}, + {file = "fonttools-4.57.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dff02c5c8423a657c550b48231d0a48d7e2b2e131088e55983cfe74ccc2c7cc9"}, + {file = "fonttools-4.57.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:767604f244dc17c68d3e2dbf98e038d11a18abc078f2d0f84b6c24571d9c0b13"}, + {file = "fonttools-4.57.0-cp313-cp313-win32.whl", hash = "sha256:8e2e12d0d862f43d51e5afb8b9751c77e6bec7d2dc00aad80641364e9df5b199"}, + {file = "fonttools-4.57.0-cp313-cp313-win_amd64.whl", hash = "sha256:f1d6bc9c23356908db712d282acb3eebd4ae5ec6d8b696aa40342b1d84f8e9e3"}, + {file = "fonttools-4.57.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9d57b4e23ebbe985125d3f0cabbf286efa191ab60bbadb9326091050d88e8213"}, + {file = "fonttools-4.57.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:579ba873d7f2a96f78b2e11028f7472146ae181cae0e4d814a37a09e93d5c5cc"}, + {file = "fonttools-4.57.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e3e1ec10c29bae0ea826b61f265ec5c858c5ba2ce2e69a71a62f285cf8e4595"}, + {file = "fonttools-4.57.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1968f2a2003c97c4ce6308dc2498d5fd4364ad309900930aa5a503c9851aec8"}, + {file = "fonttools-4.57.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:aff40f8ac6763d05c2c8f6d240c6dac4bb92640a86d9b0c3f3fff4404f34095c"}, + {file = "fonttools-4.57.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:d07f1b64008e39fceae7aa99e38df8385d7d24a474a8c9872645c4397b674481"}, + {file = "fonttools-4.57.0-cp38-cp38-win32.whl", hash = "sha256:51d8482e96b28fb28aa8e50b5706f3cee06de85cbe2dce80dbd1917ae22ec5a6"}, + {file = "fonttools-4.57.0-cp38-cp38-win_amd64.whl", hash = "sha256:03290e818782e7edb159474144fca11e36a8ed6663d1fcbd5268eb550594fd8e"}, + {file = "fonttools-4.57.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7339e6a3283e4b0ade99cade51e97cde3d54cd6d1c3744459e886b66d630c8b3"}, + {file = "fonttools-4.57.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:05efceb2cb5f6ec92a4180fcb7a64aa8d3385fd49cfbbe459350229d1974f0b1"}, + {file = "fonttools-4.57.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a97bb05eb24637714a04dee85bdf0ad1941df64fe3b802ee4ac1c284a5f97b7c"}, + {file = "fonttools-4.57.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:541cb48191a19ceb1a2a4b90c1fcebd22a1ff7491010d3cf840dd3a68aebd654"}, + {file = "fonttools-4.57.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:cdef9a056c222d0479a1fdb721430f9efd68268014c54e8166133d2643cb05d9"}, + {file = "fonttools-4.57.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3cf97236b192a50a4bf200dc5ba405aa78d4f537a2c6e4c624bb60466d5b03bd"}, + {file = "fonttools-4.57.0-cp39-cp39-win32.whl", hash = "sha256:e952c684274a7714b3160f57ec1d78309f955c6335c04433f07d36c5eb27b1f9"}, + {file = "fonttools-4.57.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2a722c0e4bfd9966a11ff55c895c817158fcce1b2b6700205a376403b546ad9"}, + {file = "fonttools-4.57.0-py3-none-any.whl", hash = "sha256:3122c604a675513c68bd24c6a8f9091f1c2376d18e8f5fe5a101746c81b3e98f"}, + {file = "fonttools-4.57.0.tar.gz", hash = "sha256:727ece10e065be2f9dd239d15dd5d60a66e17eac11aea47d447f9f03fdbc42de"}, +] + +[package.extras] +all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "fs (>=2.2.0,<3)", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.23.0)", "unicodedata2 (>=15.1.0) ; python_version <= \"3.12\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""] +lxml = ["lxml (>=4.0)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.23.0)"] +symfont = ["sympy"] +type1 = ["xattr ; sys_platform == \"darwin\""] +ufo = ["fs (>=2.2.0,<3)"] +unicode = ["unicodedata2 (>=15.1.0) ; python_version <= \"3.12\""] +woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"] + +[[package]] +name = "greenlet" +version = "3.1.1" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"}, + {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"}, + {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"}, + {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"}, + {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"}, + {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"}, + {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"}, + {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"}, + {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"}, + {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"}, + {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"}, + {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"}, + {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"}, + {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"}, + {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"}, + {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"}, + {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"}, + {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc"}, + {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa"}, + {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af"}, + {file = "greenlet-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798"}, + {file = "greenlet-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef"}, + {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"}, + {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"}, + {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"}, + {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"}, + {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"}, + {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"}, + {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"}, + {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, + {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, + {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, + {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.7" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpcore-1.0.7-py3-none-any.whl", hash = "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd"}, + {file = "httpcore-1.0.7.tar.gz", hash = "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "imageio" +version = "2.37.0" +description = "Library for reading and writing a wide range of image, video, scientific, and volumetric data formats." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "imageio-2.37.0-py3-none-any.whl", hash = "sha256:11efa15b87bc7871b61590326b2d635439acc321cf7f8ce996f812543ce10eed"}, + {file = "imageio-2.37.0.tar.gz", hash = "sha256:71b57b3669666272c818497aebba2b4c5f20d5b37c81720e5e1a56d59c492996"}, +] + +[package.dependencies] +numpy = "*" +pillow = ">=8.3.2" + +[package.extras] +all-plugins = ["astropy", "av", "imageio-ffmpeg", "numpy (>2)", "pillow-heif", "psutil", "rawpy", "tifffile"] +all-plugins-pypy = ["av", "imageio-ffmpeg", "pillow-heif", "psutil", "tifffile"] +build = ["wheel"] +dev = ["black", "flake8", "fsspec[github]", "pytest", "pytest-cov"] +docs = ["numpydoc", "pydata-sphinx-theme", "sphinx (<6)"] +ffmpeg = ["imageio-ffmpeg", "psutil"] +fits = ["astropy"] +full = ["astropy", "av", "black", "flake8", "fsspec[github]", "gdal", "imageio-ffmpeg", "itk", "numpy (>2)", "numpydoc", "pillow-heif", "psutil", "pydata-sphinx-theme", "pytest", "pytest-cov", "rawpy", "sphinx (<6)", "tifffile", "wheel"] +gdal = ["gdal"] +itk = ["itk"] +linting = ["black", "flake8"] +pillow-heif = ["pillow-heif"] +pyav = ["av"] +rawpy = ["numpy (>2)", "rawpy"] +test = ["fsspec[github]", "pytest", "pytest-cov"] +tifffile = ["tifffile"] + +[[package]] +name = "lazy-loader" +version = "0.4" +description = "Makes it easy to load subpackages and functions on demand." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "lazy_loader-0.4-py3-none-any.whl", hash = "sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc"}, + {file = "lazy_loader-0.4.tar.gz", hash = "sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1"}, +] + +[package.dependencies] +packaging = "*" + +[package.extras] +dev = ["changelist (==0.5)"] +lint = ["pre-commit (==3.7.0)"] +test = ["pytest (>=7.4)", "pytest-cov (>=4.1)"] + +[[package]] +name = "lmdb" +version = "1.6.2" +description = "Universal Python binding for the LMDB 'Lightning' Database" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "lmdb-1.6.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:18422de2b4e70fc91b6d845a83f5b7355f85ea1c02f8180be0ab42c15191bead"}, + {file = "lmdb-1.6.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:167328892282950cbb5a358fa6c05b60ffa00b2f40fb240e2e676359720256ac"}, + {file = "lmdb-1.6.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d3efbbded74b9d213059a5a4048fb5bb47b5b7c4c2b366e43685cb6dd9d3100"}, + {file = "lmdb-1.6.2-cp310-cp310-win_amd64.whl", hash = "sha256:13d4fdab8ac9ea90c317d2a5711feb9977e675b4ce50cf4706860d3617ade996"}, + {file = "lmdb-1.6.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c42b3eae0437edb95dda96e582816766a9964436f844cc9c8ddb5d4219a63ff8"}, + {file = "lmdb-1.6.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e365a09be1926dbc3ed2d4eacd9ab6125cb1f211c37e417917c26494dea24b64"}, + {file = "lmdb-1.6.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eae318f72cac81363f5218f529016c2fde6fcf7acc0d1cde55c18e2952cbf34a"}, + {file = "lmdb-1.6.2-cp311-cp311-win_amd64.whl", hash = "sha256:40f7824888a8821adc4028cf70bd031395463aaf832d4c0460f5dab8ca0e0c7a"}, + {file = "lmdb-1.6.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5dc6885b764aaccd2ecb6b244dc3ecbcce8b65d6f7e5b36fb39a603359909650"}, + {file = "lmdb-1.6.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de252e4a7895a0f1f8422b30c7f5b432612145538cf09e073b57f68fc5a88977"}, + {file = "lmdb-1.6.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c45d63a49a17ab2caa5ec0d1da35bc81c5c85fa38dfb6ab7eacf817242334a7"}, + {file = "lmdb-1.6.2-cp312-cp312-win_amd64.whl", hash = "sha256:129379009c65e67187208eff75cdad74f4ae6f840521498f3e2da407ce5aaf66"}, + {file = "lmdb-1.6.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3be24aa18f160958b90605fe5882b66998f9fa3d97221478d49782bd8f88392a"}, + {file = "lmdb-1.6.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32154f561d5cdbe833798401a752868bb1a9c056d8de1e8faf92e62ca8f22999"}, + {file = "lmdb-1.6.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45c772f24089a7ff1ebd8c59c6dc7f2da0c7b8b704b5e83ee9687c44c764554b"}, + {file = "lmdb-1.6.2-cp313-cp313-win_amd64.whl", hash = "sha256:d9676fcba2cd388f24a5d15055e4a6de351e1df9febf5b176c93cd39cc80585e"}, + {file = "lmdb-1.6.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b530f1bb092e85d344989bbf9d6555a5057f49ad97a95436af2e710b3ca824ed"}, + {file = "lmdb-1.6.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de6918cd36e1c10868b6a50e6a799238cb049e3db876c2d18a4dc7d32f516f4e"}, + {file = "lmdb-1.6.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e79c2dfe94f02e29424e9acb51de654ca992064b8379b0c1432d6639af5c40c2"}, + {file = "lmdb-1.6.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f07b91f2b2e5d120fca11b6b1e1aeeea41d25f9785df48308bc6daeedc75974"}, + {file = "lmdb-1.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a65f78ebfd8b1d0d444e6ca904a2041edc2154d795ef0285dfbd2398d92d3564"}, + {file = "lmdb-1.6.2-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:fe6ffea8c003b5766115429b8d2da17d6b3e05f644f6c535148a66c68ee3defb"}, + {file = "lmdb-1.6.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0651d4287fbb022556361842073049193767662c0896d9976cb46977471fc247"}, + {file = "lmdb-1.6.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf7cc99172dfa5acb766bdb1802f6c3f8467e85d0f7f9055211a604c884d758c"}, + {file = "lmdb-1.6.2-cp38-cp38-win_amd64.whl", hash = "sha256:27d659daf93f555f518e6c3aa9c39028035ed5f387a197e54f87986130861695"}, + {file = "lmdb-1.6.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9ae8e082cf587d06a8aef2753c959003eca3963be778e0d29762d3520e61ecd2"}, + {file = "lmdb-1.6.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ab9f270988fbca983b8381882cd271951cc5b2e4eb8bfac87caea828339c2ec"}, + {file = "lmdb-1.6.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b13d0028ac8993c967b158b78b66cc170353635cdcc823f42adf93d0114bce48"}, + {file = "lmdb-1.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:3c2cdfa34a2d0d5a465eeeabd6f4db219118fe1e71b50965b172e0803dafd236"}, + {file = "lmdb-1.6.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b2730b7be6667beb77b26dcc5e5668bfb16673715ca173af82c49be1783ae218"}, + {file = "lmdb-1.6.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d1d1d2eef0f898b7f6b37b8c72d57c6ce67b56dc4a854ee64ddbbdd3cc07a7fb"}, + {file = "lmdb-1.6.2.tar.gz", hash = "sha256:d28e3fa59935ff688858760ec52f202ecb8c1089a3f68d1f162ea3078d151e73"}, +] + +[[package]] +name = "lxml" +version = "5.3.2" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "lxml-5.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c4b84d6b580a9625dfa47269bf1fd7fbba7ad69e08b16366a46acb005959c395"}, + {file = "lxml-5.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b4c08ecb26e4270a62f81f81899dfff91623d349e433b126931c9c4577169666"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef926e9f11e307b5a7c97b17c5c609a93fb59ffa8337afac8f89e6fe54eb0b37"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:017ceeabe739100379fe6ed38b033cd244ce2da4e7f6f07903421f57da3a19a2"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dae97d9435dc90590f119d056d233c33006b2fd235dd990d5564992261ee7ae8"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:910f39425c6798ce63c93976ae5af5fff6949e2cb446acbd44d6d892103eaea8"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9780de781a0d62a7c3680d07963db3048b919fc9e3726d9cfd97296a65ffce1"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:1a06b0c6ba2e3ca45a009a78a4eb4d6b63831830c0a83dcdc495c13b9ca97d3e"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:4c62d0a34d1110769a1bbaf77871a4b711a6f59c4846064ccb78bc9735978644"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:8f961a4e82f411b14538fe5efc3e6b953e17f5e809c463f0756a0d0e8039b700"}, + {file = "lxml-5.3.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:3dfc78f5f9251b6b8ad37c47d4d0bfe63ceb073a916e5b50a3bf5fd67a703335"}, + {file = "lxml-5.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:10e690bc03214d3537270c88e492b8612d5e41b884f232df2b069b25b09e6711"}, + {file = "lxml-5.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aa837e6ee9534de8d63bc4c1249e83882a7ac22bd24523f83fad68e6ffdf41ae"}, + {file = "lxml-5.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:da4c9223319400b97a2acdfb10926b807e51b69eb7eb80aad4942c0516934858"}, + {file = "lxml-5.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dc0e9bdb3aa4d1de703a437576007d366b54f52c9897cae1a3716bb44fc1fc85"}, + {file = "lxml-5.3.2-cp310-cp310-win32.whl", hash = "sha256:5f94909a1022c8ea12711db7e08752ca7cf83e5b57a87b59e8a583c5f35016ad"}, + {file = "lxml-5.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:d64ea1686474074b38da13ae218d9fde0d1dc6525266976808f41ac98d9d7980"}, + {file = "lxml-5.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9d61a7d0d208ace43986a92b111e035881c4ed45b1f5b7a270070acae8b0bfb4"}, + {file = "lxml-5.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856dfd7eda0b75c29ac80a31a6411ca12209183e866c33faf46e77ace3ce8a79"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a01679e4aad0727bedd4c9407d4d65978e920f0200107ceeffd4b019bd48529"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6b37b4c3acb8472d191816d4582379f64d81cecbdce1a668601745c963ca5cc"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3df5a54e7b7c31755383f126d3a84e12a4e0333db4679462ef1165d702517477"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c09a40f28dcded933dc16217d6a092be0cc49ae25811d3b8e937c8060647c353"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1ef20f1851ccfbe6c5a04c67ec1ce49da16ba993fdbabdce87a92926e505412"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f79a63289dbaba964eb29ed3c103b7911f2dce28c36fe87c36a114e6bd21d7ad"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:75a72697d95f27ae00e75086aed629f117e816387b74a2f2da6ef382b460b710"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:b9b00c9ee1cc3a76f1f16e94a23c344e0b6e5c10bec7f94cf2d820ce303b8c01"}, + {file = "lxml-5.3.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:77cbcab50cbe8c857c6ba5f37f9a3976499c60eada1bf6d38f88311373d7b4bc"}, + {file = "lxml-5.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:29424058f072a24622a0a15357bca63d796954758248a72da6d512f9bd9a4493"}, + {file = "lxml-5.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7d82737a8afe69a7c80ef31d7626075cc7d6e2267f16bf68af2c764b45ed68ab"}, + {file = "lxml-5.3.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:95473d1d50a5d9fcdb9321fdc0ca6e1edc164dce4c7da13616247d27f3d21e31"}, + {file = "lxml-5.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2162068f6da83613f8b2a32ca105e37a564afd0d7009b0b25834d47693ce3538"}, + {file = "lxml-5.3.2-cp311-cp311-win32.whl", hash = "sha256:f8695752cf5d639b4e981afe6c99e060621362c416058effd5c704bede9cb5d1"}, + {file = "lxml-5.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:d1a94cbb4ee64af3ab386c2d63d6d9e9cf2e256ac0fd30f33ef0a3c88f575174"}, + {file = "lxml-5.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:16b3897691ec0316a1aa3c6585f61c8b7978475587c5b16fc1d2c28d283dc1b0"}, + {file = "lxml-5.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a8d4b34a0eeaf6e73169dcfd653c8d47f25f09d806c010daf074fba2db5e2d3f"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9cd7a959396da425022e1e4214895b5cfe7de7035a043bcc2d11303792b67554"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cac5eaeec3549c5df7f8f97a5a6db6963b91639389cdd735d5a806370847732b"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29b5f7d77334877c2146e7bb8b94e4df980325fab0a8af4d524e5d43cd6f789d"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13f3495cfec24e3d63fffd342cc8141355d1d26ee766ad388775f5c8c5ec3932"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e70ad4c9658beeff99856926fd3ee5fde8b519b92c693f856007177c36eb2e30"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:507085365783abd7879fa0a6fa55eddf4bdd06591b17a2418403bb3aff8a267d"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:5bb304f67cbf5dfa07edad904732782cbf693286b9cd85af27059c5779131050"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:3d84f5c093645c21c29a4e972b84cb7cf682f707f8706484a5a0c7ff13d7a988"}, + {file = "lxml-5.3.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:bdc13911db524bd63f37b0103af014b7161427ada41f1b0b3c9b5b5a9c1ca927"}, + {file = "lxml-5.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1ec944539543f66ebc060ae180d47e86aca0188bda9cbfadff47d86b0dc057dc"}, + {file = "lxml-5.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:59d437cc8a7f838282df5a199cf26f97ef08f1c0fbec6e84bd6f5cc2b7913f6e"}, + {file = "lxml-5.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e275961adbd32e15672e14e0cc976a982075208224ce06d149c92cb43db5b93"}, + {file = "lxml-5.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:038aeb6937aa404480c2966b7f26f1440a14005cb0702078c173c028eca72c31"}, + {file = "lxml-5.3.2-cp312-cp312-win32.whl", hash = "sha256:3c2c8d0fa3277147bff180e3590be67597e17d365ce94beb2efa3138a2131f71"}, + {file = "lxml-5.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:77809fcd97dfda3f399102db1794f7280737b69830cd5c961ac87b3c5c05662d"}, + {file = "lxml-5.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:77626571fb5270ceb36134765f25b665b896243529eefe840974269b083e090d"}, + {file = "lxml-5.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:78a533375dc7aa16d0da44af3cf6e96035e484c8c6b2b2445541a5d4d3d289ee"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a6f62b2404b3f3f0744bbcabb0381c5fe186fa2a9a67ecca3603480f4846c585"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ea918da00091194526d40c30c4996971f09dacab032607581f8d8872db34fbf"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c35326f94702a7264aa0eea826a79547d3396a41ae87a70511b9f6e9667ad31c"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3bef90af21d31c4544bc917f51e04f94ae11b43156356aff243cdd84802cbf2"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52fa7ba11a495b7cbce51573c73f638f1dcff7b3ee23697467dc063f75352a69"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ad131e2c4d2c3803e736bb69063382334e03648de2a6b8f56a878d700d4b557d"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:00a4463ca409ceacd20490a893a7e08deec7870840eff33dc3093067b559ce3e"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:87e8d78205331cace2b73ac8249294c24ae3cba98220687b5b8ec5971a2267f1"}, + {file = "lxml-5.3.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:bf6389133bb255e530a4f2f553f41c4dd795b1fbb6f797aea1eff308f1e11606"}, + {file = "lxml-5.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b3709fc752b42fb6b6ffa2ba0a5b9871646d97d011d8f08f4d5b3ee61c7f3b2b"}, + {file = "lxml-5.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:abc795703d0de5d83943a4badd770fbe3d1ca16ee4ff3783d7caffc252f309ae"}, + {file = "lxml-5.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:98050830bb6510159f65d9ad1b8aca27f07c01bb3884ba95f17319ccedc4bcf9"}, + {file = "lxml-5.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6ba465a91acc419c5682f8b06bcc84a424a7aa5c91c220241c6fd31de2a72bc6"}, + {file = "lxml-5.3.2-cp313-cp313-win32.whl", hash = "sha256:56a1d56d60ea1ec940f949d7a309e0bff05243f9bd337f585721605670abb1c1"}, + {file = "lxml-5.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:1a580dc232c33d2ad87d02c8a3069d47abbcdce974b9c9cc82a79ff603065dbe"}, + {file = "lxml-5.3.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1a59f7fe888d0ec1916d0ad69364c5400cfa2f885ae0576d909f342e94d26bc9"}, + {file = "lxml-5.3.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d67b50abc2df68502a26ed2ccea60c1a7054c289fb7fc31c12e5e55e4eec66bd"}, + {file = "lxml-5.3.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cb08d2cb047c98d6fbbb2e77d6edd132ad6e3fa5aa826ffa9ea0c9b1bc74a84"}, + {file = "lxml-5.3.2-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:495ddb7e10911fb4d673d8aa8edd98d1eadafb3b56e8c1b5f427fd33cadc455b"}, + {file = "lxml-5.3.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:884d9308ac7d581b705a3371185282e1b8eebefd68ccf288e00a2d47f077cc51"}, + {file = "lxml-5.3.2-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:37f3d7cf7f2dd2520df6cc8a13df4c3e3f913c8e0a1f9a875e44f9e5f98d7fee"}, + {file = "lxml-5.3.2-cp36-cp36m-win32.whl", hash = "sha256:e885a1bf98a76dff0a0648850c3083b99d9358ef91ba8fa307c681e8e0732503"}, + {file = "lxml-5.3.2-cp36-cp36m-win_amd64.whl", hash = "sha256:b45f505d0d85f4cdd440cd7500689b8e95110371eaa09da0c0b1103e9a05030f"}, + {file = "lxml-5.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b53cd668facd60b4f0dfcf092e01bbfefd88271b5b4e7b08eca3184dd006cb30"}, + {file = "lxml-5.3.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5dea998c891f082fe204dec6565dbc2f9304478f2fc97bd4d7a940fec16c873"}, + {file = "lxml-5.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d46bc3e58b01e4f38d75e0d7f745a46875b7a282df145aca9d1479c65ff11561"}, + {file = "lxml-5.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:661feadde89159fd5f7d7639a81ccae36eec46974c4a4d5ccce533e2488949c8"}, + {file = "lxml-5.3.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:43af2a69af2cacc2039024da08a90174e85f3af53483e6b2e3485ced1bf37151"}, + {file = "lxml-5.3.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:1539f962d82436f3d386eb9f29b2a29bb42b80199c74a695dff51b367a61ec0a"}, + {file = "lxml-5.3.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:6673920bf976421b5fac4f29b937702eef4555ee42329546a5fc68bae6178a48"}, + {file = "lxml-5.3.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9fa722a9cd8845594593cce399a49aa6bfc13b6c83a7ee05e2ab346d9253d52f"}, + {file = "lxml-5.3.2-cp37-cp37m-win32.whl", hash = "sha256:2eadd4efa487f4710755415aed3d6ae9ac8b4327ea45226ffccb239766c8c610"}, + {file = "lxml-5.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:83d8707b1b08cd02c04d3056230ec3b771b18c566ec35e723e60cdf037064e08"}, + {file = "lxml-5.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bc6e8678bfa5ccba370103976ccfcf776c85c83da9220ead41ea6fd15d2277b4"}, + {file = "lxml-5.3.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bed509662f67f719119ad56006cd4a38efa68cfa74383060612044915e5f7ad"}, + {file = "lxml-5.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e3925975fadd6fd72a6d80541a6ec75dfbad54044a03aa37282dafcb80fbdfa"}, + {file = "lxml-5.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83c0462dedc5213ac586164c6d7227da9d4d578cf45dd7fbab2ac49b63a008eb"}, + {file = "lxml-5.3.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:53e3f9ca72858834688afa17278649d62aa768a4b2018344be00c399c4d29e95"}, + {file = "lxml-5.3.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:32ba634ef3f1b20f781019a91d78599224dc45745dd572f951adbf1c0c9b0d75"}, + {file = "lxml-5.3.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:1b16504c53f41da5fcf04868a80ac40a39d3eec5329caf761114caec6e844ad1"}, + {file = "lxml-5.3.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:1f9682786138549da44ca4c49b20e7144d063b75f2b2ba611f4cff9b83db1062"}, + {file = "lxml-5.3.2-cp38-cp38-win32.whl", hash = "sha256:d8f74ef8aacdf6ee5c07566a597634bb8535f6b53dc89790db43412498cf6026"}, + {file = "lxml-5.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:49f1cee0fa27e1ee02589c696a9bdf4027e7427f184fa98e6bef0c6613f6f0fa"}, + {file = "lxml-5.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:741c126bcf9aa939e950e64e5e0a89c8e01eda7a5f5ffdfc67073f2ed849caea"}, + {file = "lxml-5.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ab6e9e6aca1fd7d725ffa132286e70dee5b9a4561c5ed291e836440b82888f89"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58e8c9b9ed3c15c2d96943c14efc324b69be6352fe5585733a7db2bf94d97841"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7811828ddfb8c23f4f1fbf35e7a7b2edec2f2e4c793dee7c52014f28c4b35238"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:72968623efb1e12e950cbdcd1d0f28eb14c8535bf4be153f1bfffa818b1cf189"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebfceaa2ea588b54efb6160e3520983663d45aed8a3895bb2031ada080fb5f04"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d685d458505b2bfd2e28c812749fe9194a2b0ce285a83537e4309a187ffa270b"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:334e0e414dab1f5366ead8ca34ec3148415f236d5660e175f1d640b11d645847"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:02e56f7de72fa82561eae69628a7d6febd7891d72248c7ff7d3e7814d4031017"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:638d06b4e1d34d1a074fa87deed5fb55c18485fa0dab97abc5604aad84c12031"}, + {file = "lxml-5.3.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:354dab7206d22d7a796fa27c4c5bffddd2393da2ad61835355a4759d435beb47"}, + {file = "lxml-5.3.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d9d9f82ff2c3bf9bb777cb355149f7f3a98ec58f16b7428369dc27ea89556a4c"}, + {file = "lxml-5.3.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:95ad58340e3b7d2b828efc370d1791856613c5cb62ae267158d96e47b3c978c9"}, + {file = "lxml-5.3.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:30fe05f4b7f6e9eb32862745512e7cbd021070ad0f289a7f48d14a0d3fc1d8a9"}, + {file = "lxml-5.3.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:34c688fef86f73dbca0798e0a61bada114677006afa524a8ce97d9e5fabf42e6"}, + {file = "lxml-5.3.2-cp39-cp39-win32.whl", hash = "sha256:4d6d3d1436d57f41984920667ec5ef04bcb158f80df89ac4d0d3f775a2ac0c87"}, + {file = "lxml-5.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:2996e1116bbb3ae2a1fbb2ba4da8f92742290b4011e7e5bce2bd33bbc9d9485a"}, + {file = "lxml-5.3.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:521ab9c80b98c30b2d987001c3ede2e647e92eeb2ca02e8cb66ef5122d792b24"}, + {file = "lxml-5.3.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f1231b0f9810289d41df1eacc4ebb859c63e4ceee29908a0217403cddce38d0"}, + {file = "lxml-5.3.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271f1a4d5d2b383c36ad8b9b489da5ea9c04eca795a215bae61ed6a57cf083cd"}, + {file = "lxml-5.3.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:6fca8a5a13906ba2677a5252752832beb0f483a22f6c86c71a2bb320fba04f61"}, + {file = "lxml-5.3.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ea0c3b7922209160faef194a5b6995bfe7fa05ff7dda6c423ba17646b7b9de10"}, + {file = "lxml-5.3.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0a006390834603e5952a2ff74b9a31a6007c7cc74282a087aa6467afb4eea987"}, + {file = "lxml-5.3.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eae4136a3b8c4cf76f69461fc8f9410d55d34ea48e1185338848a888d71b9675"}, + {file = "lxml-5.3.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d48e06be8d8c58e7feaedd8a37897a6122637efb1637d7ce00ddf5f11f9a92ad"}, + {file = "lxml-5.3.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4b83aed409134093d90e114007034d2c1ebcd92e501b71fd9ec70e612c8b2eb"}, + {file = "lxml-5.3.2-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7a0e77edfe26d3703f954d46bed52c3ec55f58586f18f4b7f581fc56954f1d84"}, + {file = "lxml-5.3.2-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:19f6fcfd15b82036b4d235749d78785eb9c991c7812012dc084e0d8853b4c1c0"}, + {file = "lxml-5.3.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d49919c95d31ee06eefd43d8c6f69a3cc9bdf0a9b979cc234c4071f0eb5cb173"}, + {file = "lxml-5.3.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2d0a60841410123c533990f392819804a8448853f06daf412c0f383443925e89"}, + {file = "lxml-5.3.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b7f729e03090eb4e3981f10efaee35e6004b548636b1a062b8b9a525e752abc"}, + {file = "lxml-5.3.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:579df6e20d8acce3bcbc9fb8389e6ae00c19562e929753f534ba4c29cfe0be4b"}, + {file = "lxml-5.3.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:2abcf3f3b8367d6400b908d00d4cd279fc0b8efa287e9043820525762d383699"}, + {file = "lxml-5.3.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:348c06cb2e3176ce98bee8c397ecc89181681afd13d85870df46167f140a305f"}, + {file = "lxml-5.3.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:617ecaccd565cbf1ac82ffcaa410e7da5bd3a4b892bb3543fb2fe19bd1c4467d"}, + {file = "lxml-5.3.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c3eb4278dcdb9d86265ed2c20b9ecac45f2d6072e3904542e591e382c87a9c00"}, + {file = "lxml-5.3.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258b6b53458c5cbd2a88795557ff7e0db99f73a96601b70bc039114cd4ee9e02"}, + {file = "lxml-5.3.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a9d8d25ed2f2183e8471c97d512a31153e123ac5807f61396158ef2793cb6e"}, + {file = "lxml-5.3.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:73bcb635a848c18a3e422ea0ab0092f2e4ef3b02d8ebe87ab49748ebc8ec03d8"}, + {file = "lxml-5.3.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1545de0a69a16ced5767bae8cca1801b842e6e49e96f5e4a8a5acbef023d970b"}, + {file = "lxml-5.3.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:165fcdc2f40fc0fe88a3c3c06c9c2a097388a90bda6a16e6f7c9199c903c9b8e"}, + {file = "lxml-5.3.2.tar.gz", hash = "sha256:773947d0ed809ddad824b7b14467e1a481b8976e87278ac4a730c2f7c7fcddc1"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml_html_clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=3.0.11,<3.1.0)"] + +[[package]] +name = "networkx" +version = "3.4.2" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, + {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, +] + +[package.extras] +default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] +example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] +extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] + +[[package]] +name = "numpy" +version = "2.2.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "numpy-2.2.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8146f3550d627252269ac42ae660281d673eb6f8b32f113538e0cc2a9aed42b9"}, + {file = "numpy-2.2.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e642d86b8f956098b564a45e6f6ce68a22c2c97a04f5acd3f221f57b8cb850ae"}, + {file = "numpy-2.2.4-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:a84eda42bd12edc36eb5b53bbcc9b406820d3353f1994b6cfe453a33ff101775"}, + {file = "numpy-2.2.4-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:4ba5054787e89c59c593a4169830ab362ac2bee8a969249dc56e5d7d20ff8df9"}, + {file = "numpy-2.2.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7716e4a9b7af82c06a2543c53ca476fa0b57e4d760481273e09da04b74ee6ee2"}, + {file = "numpy-2.2.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adf8c1d66f432ce577d0197dceaac2ac00c0759f573f28516246351c58a85020"}, + {file = "numpy-2.2.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:218f061d2faa73621fa23d6359442b0fc658d5b9a70801373625d958259eaca3"}, + {file = "numpy-2.2.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:df2f57871a96bbc1b69733cd4c51dc33bea66146b8c63cacbfed73eec0883017"}, + {file = "numpy-2.2.4-cp310-cp310-win32.whl", hash = "sha256:a0258ad1f44f138b791327961caedffbf9612bfa504ab9597157806faa95194a"}, + {file = "numpy-2.2.4-cp310-cp310-win_amd64.whl", hash = "sha256:0d54974f9cf14acf49c60f0f7f4084b6579d24d439453d5fc5805d46a165b542"}, + {file = "numpy-2.2.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e9e0a277bb2eb5d8a7407e14688b85fd8ad628ee4e0c7930415687b6564207a4"}, + {file = "numpy-2.2.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9eeea959168ea555e556b8188da5fa7831e21d91ce031e95ce23747b7609f8a4"}, + {file = "numpy-2.2.4-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bd3ad3b0a40e713fc68f99ecfd07124195333f1e689387c180813f0e94309d6f"}, + {file = "numpy-2.2.4-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:cf28633d64294969c019c6df4ff37f5698e8326db68cc2b66576a51fad634880"}, + {file = "numpy-2.2.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fa8fa7697ad1646b5c93de1719965844e004fcad23c91228aca1cf0800044a1"}, + {file = "numpy-2.2.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4162988a360a29af158aeb4a2f4f09ffed6a969c9776f8f3bdee9b06a8ab7e5"}, + {file = "numpy-2.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:892c10d6a73e0f14935c31229e03325a7b3093fafd6ce0af704be7f894d95687"}, + {file = "numpy-2.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db1f1c22173ac1c58db249ae48aa7ead29f534b9a948bc56828337aa84a32ed6"}, + {file = "numpy-2.2.4-cp311-cp311-win32.whl", hash = "sha256:ea2bb7e2ae9e37d96835b3576a4fa4b3a97592fbea8ef7c3587078b0068b8f09"}, + {file = "numpy-2.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:f7de08cbe5551911886d1ab60de58448c6df0f67d9feb7d1fb21e9875ef95e91"}, + {file = "numpy-2.2.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a7b9084668aa0f64e64bd00d27ba5146ef1c3a8835f3bd912e7a9e01326804c4"}, + {file = "numpy-2.2.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dbe512c511956b893d2dacd007d955a3f03d555ae05cfa3ff1c1ff6df8851854"}, + {file = "numpy-2.2.4-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:bb649f8b207ab07caebba230d851b579a3c8711a851d29efe15008e31bb4de24"}, + {file = "numpy-2.2.4-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:f34dc300df798742b3d06515aa2a0aee20941c13579d7a2f2e10af01ae4901ee"}, + {file = "numpy-2.2.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3f7ac96b16955634e223b579a3e5798df59007ca43e8d451a0e6a50f6bfdfba"}, + {file = "numpy-2.2.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f92084defa704deadd4e0a5ab1dc52d8ac9e8a8ef617f3fbb853e79b0ea3592"}, + {file = "numpy-2.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4e84a6283b36632e2a5b56e121961f6542ab886bc9e12f8f9818b3c266bfbb"}, + {file = "numpy-2.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:11c43995255eb4127115956495f43e9343736edb7fcdb0d973defd9de14cd84f"}, + {file = "numpy-2.2.4-cp312-cp312-win32.whl", hash = "sha256:65ef3468b53269eb5fdb3a5c09508c032b793da03251d5f8722b1194f1790c00"}, + {file = "numpy-2.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:2aad3c17ed2ff455b8eaafe06bcdae0062a1db77cb99f4b9cbb5f4ecb13c5146"}, + {file = "numpy-2.2.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cf4e5c6a278d620dee9ddeb487dc6a860f9b199eadeecc567f777daace1e9e7"}, + {file = "numpy-2.2.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1974afec0b479e50438fc3648974268f972e2d908ddb6d7fb634598cdb8260a0"}, + {file = "numpy-2.2.4-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:79bd5f0a02aa16808fcbc79a9a376a147cc1045f7dfe44c6e7d53fa8b8a79392"}, + {file = "numpy-2.2.4-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:3387dd7232804b341165cedcb90694565a6015433ee076c6754775e85d86f1fc"}, + {file = "numpy-2.2.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f527d8fdb0286fd2fd97a2a96c6be17ba4232da346931d967a0630050dfd298"}, + {file = "numpy-2.2.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bce43e386c16898b91e162e5baaad90c4b06f9dcbe36282490032cec98dc8ae7"}, + {file = "numpy-2.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:31504f970f563d99f71a3512d0c01a645b692b12a63630d6aafa0939e52361e6"}, + {file = "numpy-2.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:81413336ef121a6ba746892fad881a83351ee3e1e4011f52e97fba79233611fd"}, + {file = "numpy-2.2.4-cp313-cp313-win32.whl", hash = "sha256:f486038e44caa08dbd97275a9a35a283a8f1d2f0ee60ac260a1790e76660833c"}, + {file = "numpy-2.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:207a2b8441cc8b6a2a78c9ddc64d00d20c303d79fba08c577752f080c4007ee3"}, + {file = "numpy-2.2.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8120575cb4882318c791f839a4fd66161a6fa46f3f0a5e613071aae35b5dd8f8"}, + {file = "numpy-2.2.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a761ba0fa886a7bb33c6c8f6f20213735cb19642c580a931c625ee377ee8bd39"}, + {file = "numpy-2.2.4-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:ac0280f1ba4a4bfff363a99a6aceed4f8e123f8a9b234c89140f5e894e452ecd"}, + {file = "numpy-2.2.4-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:879cf3a9a2b53a4672a168c21375166171bc3932b7e21f622201811c43cdd3b0"}, + {file = "numpy-2.2.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f05d4198c1bacc9124018109c5fba2f3201dbe7ab6e92ff100494f236209c960"}, + {file = "numpy-2.2.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2f085ce2e813a50dfd0e01fbfc0c12bbe5d2063d99f8b29da30e544fb6483b8"}, + {file = "numpy-2.2.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:92bda934a791c01d6d9d8e038363c50918ef7c40601552a58ac84c9613a665bc"}, + {file = "numpy-2.2.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ee4d528022f4c5ff67332469e10efe06a267e32f4067dc76bb7e2cddf3cd25ff"}, + {file = "numpy-2.2.4-cp313-cp313t-win32.whl", hash = "sha256:05c076d531e9998e7e694c36e8b349969c56eadd2cdcd07242958489d79a7286"}, + {file = "numpy-2.2.4-cp313-cp313t-win_amd64.whl", hash = "sha256:188dcbca89834cc2e14eb2f106c96d6d46f200fe0200310fc29089657379c58d"}, + {file = "numpy-2.2.4-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7051ee569db5fbac144335e0f3b9c2337e0c8d5c9fee015f259a5bd70772b7e8"}, + {file = "numpy-2.2.4-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:ab2939cd5bec30a7430cbdb2287b63151b77cf9624de0532d629c9a1c59b1d5c"}, + {file = "numpy-2.2.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0f35b19894a9e08639fd60a1ec1978cb7f5f7f1eace62f38dd36be8aecdef4d"}, + {file = "numpy-2.2.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b4adfbbc64014976d2f91084915ca4e626fbf2057fb81af209c1a6d776d23e3d"}, + {file = "numpy-2.2.4.tar.gz", hash = "sha256:9ba03692a45d3eef66559efe1d1096c4b9b75c0986b5dff5530c378fb8331d4f"}, +] + +[[package]] +name = "opencv-contrib-python" +version = "4.11.0.86" +description = "Wrapper package for OpenCV python bindings." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "opencv-contrib-python-4.11.0.86.tar.gz", hash = "sha256:4ff773dab44911da366b906621c9592d4eb96f6ad3777098933a23f064aab38e"}, + {file = "opencv_contrib_python-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:d911cedc511d98f79994580b245d59fc97f57f0f9923a99945d8b92c7ac671f6"}, + {file = "opencv_contrib_python-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:e10a293af18aa5f842d012fa14e87345b3ee06db4c29bd592ff94b51f7ffca2b"}, + {file = "opencv_contrib_python-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f21034bc8b00eb286a0a0a92b99767bf596bfe426cf4bc2e79647d64ad0dd6da"}, + {file = "opencv_contrib_python-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c47c0ef1098461cdc6fa1cdce4c942b8ec974c87423f4b5951443d26bb9ae407"}, + {file = "opencv_contrib_python-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:194841c664ceaa0692410b4ed0af557425608e33db3a181ded28b87acb66748d"}, + {file = "opencv_contrib_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:654758a9ae8ca9a75fca7b64b19163636534f0eedffe1e14c3d7218988625c8d"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} + +[[package]] +name = "opencv-python" +version = "4.11.0.86" +description = "Wrapper package for OpenCV python bindings." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "opencv-python-4.11.0.86.tar.gz", hash = "sha256:03d60ccae62304860d232272e4a4fda93c39d595780cb40b161b310244b736a4"}, + {file = "opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:432f67c223f1dc2824f5e73cdfcd9db0efc8710647d4e813012195dc9122a52a"}, + {file = "opencv_python-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:9d05ef13d23fe97f575153558653e2d6e87103995d54e6a35db3f282fe1f9c66"}, + {file = "opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b92ae2c8852208817e6776ba1ea0d6b1e0a1b5431e971a2a0ddd2a8cc398202"}, + {file = "opencv_python-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b02611523803495003bd87362db3e1d2a0454a6a63025dc6658a9830570aa0d"}, + {file = "opencv_python-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:810549cb2a4aedaa84ad9a1c92fbfdfc14090e2749cedf2c1589ad8359aa169b"}, + {file = "opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} + +[[package]] +name = "opencv-python-headless" +version = "4.11.0.86" +description = "Wrapper package for OpenCV python bindings." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "opencv-python-headless-4.11.0.86.tar.gz", hash = "sha256:996eb282ca4b43ec6a3972414de0e2331f5d9cda2b41091a49739c19fb843798"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:48128188ade4a7e517237c8e1e11a9cdf5c282761473383e77beb875bb1e61ca"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-macosx_13_0_x86_64.whl", hash = "sha256:a66c1b286a9de872c343ee7c3553b084244299714ebb50fbdcd76f07ebbe6c81"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6efabcaa9df731f29e5ea9051776715b1bdd1845d7c9530065c7951d2a2899eb"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e0a27c19dd1f40ddff94976cfe43066fbbe9dfbb2ec1907d66c19caef42a57b"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-win32.whl", hash = "sha256:f447d8acbb0b6f2808da71fddd29c1cdd448d2bc98f72d9bb78a7a898fc9621b"}, + {file = "opencv_python_headless-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:6c304df9caa7a6a5710b91709dd4786bf20a74d57672b3c31f7033cc638174ca"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} + +[[package]] +name = "opt-einsum" +version = "3.3.0" +description = "Optimizing numpys einsum function" +optional = false +python-versions = ">=3.5" +groups = ["main"] +files = [ + {file = "opt_einsum-3.3.0-py3-none-any.whl", hash = "sha256:2455e59e3947d3c275477df7f5205b30635e266fe6dc300e3d9f9646bfcea147"}, + {file = "opt_einsum-3.3.0.tar.gz", hash = "sha256:59f6475f77bbc37dcf7cd748519c0ec60722e91e63ca114e68821c0c54a46549"}, +] + +[package.dependencies] +numpy = ">=1.7" + +[package.extras] +docs = ["numpydoc", "sphinx (==1.2.3)", "sphinx-rtd-theme", "sphinxcontrib-napoleon"] +tests = ["pytest", "pytest-cov", "pytest-pep8"] + +[[package]] +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] + +[[package]] +name = "paddleocr" +version = "2.10.0" +description = "Awesome OCR toolkits based on PaddlePaddle(8.6M ultra-lightweight pre-trained model, support training and deployment among server, mobile, embedded and IoT devices)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "paddleocr-2.10.0-py3-none-any.whl", hash = "sha256:38df818d87a00af854cbfd14e33615edc3c4fa2caeb662149d51e6f2e212013f"}, + {file = "paddleocr-2.10.0.tar.gz", hash = "sha256:c0646aae769c957df698558e9264c94113ee60244a4ac0c46dfcd2324d215df4"}, +] + +[package.dependencies] +albucore = "*" +albumentations = "*" +beautifulsoup4 = "*" +cython = "*" +fire = ">=0.3.0" +fonttools = ">=4.24.0" +lmdb = "*" +numpy = "*" +opencv-contrib-python = "*" +opencv-python = "*" +Pillow = "*" +pyclipper = "*" +python-docx = "*" +pyyaml = "*" +rapidfuzz = "*" +requests = "*" +scikit-image = "*" +shapely = "*" +tqdm = "*" + +[[package]] +name = "paddlepaddle" +version = "3.0.0" +description = "Parallel Distributed Deep Learning" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "paddlepaddle-3.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:69321d2fed61f9c4b502843d213af0919e058f8399c3fb7da4b90ff9f9e8544d"}, + {file = "paddlepaddle-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7a8a3de6238d7588224353d7412d1869772fa88d0cc0119cb7c921bee9b1869c"}, + {file = "paddlepaddle-3.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:73b1349d91a85dae81e23aee0faf1d04f244e26c688c23e387a504437066829a"}, + {file = "paddlepaddle-3.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:ea8841cdbea7f26dbe548d3b129dd9764f36a4338656c9b5257cac43b382a674"}, + {file = "paddlepaddle-3.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:9fe8544bce8d034da5cdab1417782c7d53e1bef97533bcef5710dfdb59e87ca5"}, + {file = "paddlepaddle-3.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f69b4ec5ec881e632bf80d8d2a0ca74f87925335dc57b14bb8971e72e2d87951"}, + {file = "paddlepaddle-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb4a98192079cde5b5dac1109d334c12b376b97edc8682490b5f5065c8f708d9"}, + {file = "paddlepaddle-3.0.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:cfd73da79e2eb9325bac4a3a1a7995063c7662dde724bf452659dda88e55b76f"}, + {file = "paddlepaddle-3.0.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:3ef99688f4d5113dcf0e026a0b225efa77975da04e8ca1d0666ebdb729d4a409"}, + {file = "paddlepaddle-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:215daf5c154855e9f4139844076853cabc86425fd127c390405895ae2b820d85"}, + {file = "paddlepaddle-3.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dcfebfb10deb15aa33d2a2d66d5a12d9de65461d8a40d2687e2fb3221150d0ba"}, + {file = "paddlepaddle-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8677e8d42ff66f74fbca32ff32d8b64bbe3d56ed29a92d43dc418f0f168fcb69"}, + {file = "paddlepaddle-3.0.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:81a7c5b10150b07521d1cd1491dca1fb76ff621b14db10e899d959f3a77935ea"}, + {file = "paddlepaddle-3.0.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:ddaa996a8b0a00dbe4269da21d65052877ec3693a22ecc07b9cada3ecd116886"}, + {file = "paddlepaddle-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:68efccd9b64233243e48d89a20d34904e2827abc3458c7dddb9e97ef8e2209f6"}, + {file = "paddlepaddle-3.0.0-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:52c401ffe4aae56316660a33015a6fd755cda23d3d70a28b13b73d70cbdbb8cb"}, + {file = "paddlepaddle-3.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9d1051f2e208da1d439daee7c861bcb50240f09712496eb61e9ce8c618faf2f9"}, + {file = "paddlepaddle-3.0.0-cp313-cp313-manylinux1_x86_64.whl", hash = "sha256:8fc4865efd3656ee99424124792cb88a642151b4d591a7d6d4a5c6afdae95959"}, + {file = "paddlepaddle-3.0.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:bbc4d573b35d24a262e02d72696e80f5b4aee41f698671af9574deeb1a83e6f7"}, + {file = "paddlepaddle-3.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:2e1292d50f80a8577407008f7321f85c8309938c0c575927950aec3c45502c2a"}, + {file = "paddlepaddle-3.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a98145ff8e554c6a85fe233ff5a6480def4b6b6390ee12436f028c819b4c36f"}, + {file = "paddlepaddle-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:92d5398b7c6fd92d31b5cacb81dd1196c87a3bc7ee5021325df914c8187d03c2"}, + {file = "paddlepaddle-3.0.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:414ae8b44259bc363f3e120ef917390f98a53bc57cc38de27d04f5f1e043aae1"}, + {file = "paddlepaddle-3.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:157a3b11f0599175ea07759357d331be0031df3d462ba81b9c3c31fea9a57faf"}, + {file = "paddlepaddle-3.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:93a21b03a960515e1431bfe5f404060e2ce8c82c63d885600e0b45b775f2d081"}, + {file = "paddlepaddle-3.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:394a942b7281a8eb87083f80a9592f31b8aec1a360464e536c1d955d3e6e0bb4"}, + {file = "paddlepaddle-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:00bd606cdce5dbae2b9752afdc07479c5fab9df73efdf1bf577d187fd492f289"}, + {file = "paddlepaddle-3.0.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:220f8010c30cfd7712d0b0364e4732a344779ac202888ae1045e3567e24a6213"}, + {file = "paddlepaddle-3.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f68630572edc4eb76ab6bbbe57689cb08b961f0d08bc0fc8c969d79b4bc213e9"}, + {file = "paddlepaddle-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:4b078cee4f6da995e4d2b1c46d1d94545f31a487b02ec1c61cfbb9e7d6d082f2"}, +] + +[package.dependencies] +astor = "*" +decorator = "*" +httpx = "*" +networkx = "*" +numpy = ">=1.21" +opt-einsum = "3.3.0" +Pillow = "*" +protobuf = ">=3.20.2" +typing-extensions = "*" + +[[package]] +name = "pillow" +version = "11.1.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pillow-11.1.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e1abe69aca89514737465752b4bcaf8016de61b3be1397a8fc260ba33321b3a8"}, + {file = "pillow-11.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c640e5a06869c75994624551f45e5506e4256562ead981cce820d5ab39ae2192"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a07dba04c5e22824816b2615ad7a7484432d7f540e6fa86af60d2de57b0fcee2"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e267b0ed063341f3e60acd25c05200df4193e15a4a5807075cd71225a2386e26"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bd165131fd51697e22421d0e467997ad31621b74bfc0b75956608cb2906dda07"}, + {file = "pillow-11.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:abc56501c3fd148d60659aae0af6ddc149660469082859fa7b066a298bde9482"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:54ce1c9a16a9561b6d6d8cb30089ab1e5eb66918cb47d457bd996ef34182922e"}, + {file = "pillow-11.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73ddde795ee9b06257dac5ad42fcb07f3b9b813f8c1f7f870f402f4dc54b5269"}, + {file = "pillow-11.1.0-cp310-cp310-win32.whl", hash = "sha256:3a5fe20a7b66e8135d7fd617b13272626a28278d0e578c98720d9ba4b2439d49"}, + {file = "pillow-11.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:b6123aa4a59d75f06e9dd3dac5bf8bc9aa383121bb3dd9a7a612e05eabc9961a"}, + {file = "pillow-11.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:a76da0a31da6fcae4210aa94fd779c65c75786bc9af06289cd1c184451ef7a65"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e06695e0326d05b06833b40b7ef477e475d0b1ba3a6d27da1bb48c23209bf457"}, + {file = "pillow-11.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96f82000e12f23e4f29346e42702b6ed9a2f2fea34a740dd5ffffcc8c539eb35"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3cd561ded2cf2bbae44d4605837221b987c216cff94f49dfeed63488bb228d2"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f189805c8be5ca5add39e6f899e6ce2ed824e65fb45f3c28cb2841911da19070"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dd0052e9db3474df30433f83a71b9b23bd9e4ef1de13d92df21a52c0303b8ab6"}, + {file = "pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:837060a8599b8f5d402e97197d4924f05a2e0d68756998345c829c33186217b1"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa8dd43daa836b9a8128dbe7d923423e5ad86f50a7a14dc688194b7be5c0dea2"}, + {file = "pillow-11.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0a2f91f8a8b367e7a57c6e91cd25af510168091fb89ec5146003e424e1558a96"}, + {file = "pillow-11.1.0-cp311-cp311-win32.whl", hash = "sha256:c12fc111ef090845de2bb15009372175d76ac99969bdf31e2ce9b42e4b8cd88f"}, + {file = "pillow-11.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fbd43429d0d7ed6533b25fc993861b8fd512c42d04514a0dd6337fb3ccf22761"}, + {file = "pillow-11.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:f7955ecf5609dee9442cbface754f2c6e541d9e6eda87fad7f7a989b0bdb9d71"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2062ffb1d36544d42fcaa277b069c88b01bb7298f4efa06731a7fd6cc290b81a"}, + {file = "pillow-11.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a85b653980faad27e88b141348707ceeef8a1186f75ecc600c395dcac19f385b"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9409c080586d1f683df3f184f20e36fb647f2e0bc3988094d4fd8c9f4eb1b3b3"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fdadc077553621911f27ce206ffcbec7d3f8d7b50e0da39f10997e8e2bb7f6a"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:93a18841d09bcdd774dcdc308e4537e1f867b3dec059c131fde0327899734aa1"}, + {file = "pillow-11.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:9aa9aeddeed452b2f616ff5507459e7bab436916ccb10961c4a382cd3e03f47f"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cdcdb0b896e981678eee140d882b70092dac83ac1cdf6b3a60e2216a73f2b91"}, + {file = "pillow-11.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36ba10b9cb413e7c7dfa3e189aba252deee0602c86c309799da5a74009ac7a1c"}, + {file = "pillow-11.1.0-cp312-cp312-win32.whl", hash = "sha256:cfd5cd998c2e36a862d0e27b2df63237e67273f2fc78f47445b14e73a810e7e6"}, + {file = "pillow-11.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a697cd8ba0383bba3d2d3ada02b34ed268cb548b369943cd349007730c92bddf"}, + {file = "pillow-11.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:4dd43a78897793f60766563969442020e90eb7847463eca901e41ba186a7d4a5"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae98e14432d458fc3de11a77ccb3ae65ddce70f730e7c76140653048c71bfcbc"}, + {file = "pillow-11.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cc1331b6d5a6e144aeb5e626f4375f5b7ae9934ba620c0ac6b3e43d5e683a0f0"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:758e9d4ef15d3560214cddbc97b8ef3ef86ce04d62ddac17ad39ba87e89bd3b1"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b523466b1a31d0dcef7c5be1f20b942919b62fd6e9a9be199d035509cbefc0ec"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9044b5e4f7083f209c4e35aa5dd54b1dd5b112b108648f5c902ad586d4f945c5"}, + {file = "pillow-11.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:3764d53e09cdedd91bee65c2527815d315c6b90d7b8b79759cc48d7bf5d4f114"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:31eba6bbdd27dde97b0174ddf0297d7a9c3a507a8a1480e1e60ef914fe23d352"}, + {file = "pillow-11.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5d658fbd9f0d6eea113aea286b21d3cd4d3fd978157cbf2447a6035916506d3"}, + {file = "pillow-11.1.0-cp313-cp313-win32.whl", hash = "sha256:f86d3a7a9af5d826744fabf4afd15b9dfef44fe69a98541f666f66fbb8d3fef9"}, + {file = "pillow-11.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:593c5fd6be85da83656b93ffcccc2312d2d149d251e98588b14fbc288fd8909c"}, + {file = "pillow-11.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:11633d58b6ee5733bde153a8dafd25e505ea3d32e261accd388827ee987baf65"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:70ca5ef3b3b1c4a0812b5c63c57c23b63e53bc38e758b37a951e5bc466449861"}, + {file = "pillow-11.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8000376f139d4d38d6851eb149b321a52bb8893a88dae8ee7d95840431977081"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ee85f0696a17dd28fbcfceb59f9510aa71934b483d1f5601d1030c3c8304f3c"}, + {file = "pillow-11.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:dd0e081319328928531df7a0e63621caf67652c8464303fd102141b785ef9547"}, + {file = "pillow-11.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e63e4e5081de46517099dc30abe418122f54531a6ae2ebc8680bcd7096860eab"}, + {file = "pillow-11.1.0-cp313-cp313t-win32.whl", hash = "sha256:dda60aa465b861324e65a78c9f5cf0f4bc713e4309f83bc387be158b077963d9"}, + {file = "pillow-11.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ad5db5781c774ab9a9b2c4302bbf0c1014960a0a7be63278d13ae6fdf88126fe"}, + {file = "pillow-11.1.0-cp313-cp313t-win_arm64.whl", hash = "sha256:67cd427c68926108778a9005f2a04adbd5e67c442ed21d95389fe1d595458756"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:bf902d7413c82a1bfa08b06a070876132a5ae6b2388e2712aab3a7cbc02205c6"}, + {file = "pillow-11.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c1eec9d950b6fe688edee07138993e54ee4ae634c51443cfb7c1e7613322718e"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e275ee4cb11c262bd108ab2081f750db2a1c0b8c12c1897f27b160c8bd57bbc"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4db853948ce4e718f2fc775b75c37ba2efb6aaea41a1a5fc57f0af59eee774b2"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:ab8a209b8485d3db694fa97a896d96dd6533d63c22829043fd9de627060beade"}, + {file = "pillow-11.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:54251ef02a2309b5eec99d151ebf5c9904b77976c8abdcbce7891ed22df53884"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5bb94705aea800051a743aa4874bb1397d4695fb0583ba5e425ee0328757f196"}, + {file = "pillow-11.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89dbdb3e6e9594d512780a5a1c42801879628b38e3efc7038094430844e271d8"}, + {file = "pillow-11.1.0-cp39-cp39-win32.whl", hash = "sha256:e5449ca63da169a2e6068dd0e2fcc8d91f9558aba89ff6d02121ca8ab11e79e5"}, + {file = "pillow-11.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3362c6ca227e65c54bf71a5f88b3d4565ff1bcbc63ae72c34b07bbb1cc59a43f"}, + {file = "pillow-11.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:b20be51b37a75cc54c2c55def3fa2c65bb94ba859dde241cd0a4fd302de5ae0a"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8c730dc3a83e5ac137fbc92dfcfe1511ce3b2b5d7578315b63dbbb76f7f51d90"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7d33d2fae0e8b170b6a6c57400e077412240f6f5bb2a342cf1ee512a787942bb"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8d65b38173085f24bc07f8b6c505cbb7418009fa1a1fcb111b1f4961814a442"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:015c6e863faa4779251436db398ae75051469f7c903b043a48f078e437656f83"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d44ff19eea13ae4acdaaab0179fa68c0c6f2f45d66a4d8ec1eda7d6cecbcc15f"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3d8da4a631471dfaf94c10c85f5277b1f8e42ac42bade1ac67da4b4a7359b73"}, + {file = "pillow-11.1.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4637b88343166249fe8aa94e7c4a62a180c4b3898283bb5d3d2fd5fe10d8e4e0"}, + {file = "pillow-11.1.0.tar.gz", hash = "sha256:368da70808b36d73b4b390a8ffac11069f8a5c85f29eff1f1b01bcf3ef5b2a20"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"] +typing = ["typing-extensions ; python_version < \"3.10\""] +xmp = ["defusedxml"] + +[[package]] +name = "playwright" +version = "1.51.0" +description = "A high-level API to automate web browsers" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "playwright-1.51.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:bcaaa3d5d73bda659bfb9ff2a288b51e85a91bd89eda86eaf8186550973e416a"}, + {file = "playwright-1.51.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2e0ae6eb44297b24738e1a6d9c580ca4243b4e21b7e65cf936a71492c08dd0d4"}, + {file = "playwright-1.51.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:ab4c0ff00bded52c946be60734868febc964c8a08a9b448d7c20cb3811c6521c"}, + {file = "playwright-1.51.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:d5c9f67bc6ef49094618991c78a1466c5bac5ed09157660d78b8510b77f92746"}, + {file = "playwright-1.51.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:814e4ec2a1a0d6f6221f075622c06b31ceb2bdc6d622258cfefed900c01569ae"}, + {file = "playwright-1.51.0-py3-none-win32.whl", hash = "sha256:4cef804991867ea27f608b70fa288ee52a57651e22d02ab287f98f8620b9408c"}, + {file = "playwright-1.51.0-py3-none-win_amd64.whl", hash = "sha256:9ece9316c5d383aed1a207f079fc2d552fff92184f0ecf37cc596e912d00a8c3"}, +] + +[package.dependencies] +greenlet = ">=3.1.1,<4.0.0" +pyee = ">=12,<13" + +[[package]] +name = "protobuf" +version = "6.30.2" +description = "" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "protobuf-6.30.2-cp310-abi3-win32.whl", hash = "sha256:b12ef7df7b9329886e66404bef5e9ce6a26b54069d7f7436a0853ccdeb91c103"}, + {file = "protobuf-6.30.2-cp310-abi3-win_amd64.whl", hash = "sha256:7653c99774f73fe6b9301b87da52af0e69783a2e371e8b599b3e9cb4da4b12b9"}, + {file = "protobuf-6.30.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:0eb523c550a66a09a0c20f86dd554afbf4d32b02af34ae53d93268c1f73bc65b"}, + {file = "protobuf-6.30.2-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:50f32cc9fd9cb09c783ebc275611b4f19dfdfb68d1ee55d2f0c7fa040df96815"}, + {file = "protobuf-6.30.2-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:4f6c687ae8efae6cf6093389a596548214467778146b7245e886f35e1485315d"}, + {file = "protobuf-6.30.2-cp39-cp39-win32.whl", hash = "sha256:524afedc03b31b15586ca7f64d877a98b184f007180ce25183d1a5cb230ee72b"}, + {file = "protobuf-6.30.2-cp39-cp39-win_amd64.whl", hash = "sha256:acec579c39c88bd8fbbacab1b8052c793efe83a0a5bd99db4a31423a25c0a0e2"}, + {file = "protobuf-6.30.2-py3-none-any.whl", hash = "sha256:ae86b030e69a98e08c77beab574cbcb9fff6d031d57209f574a5aea1445f4b51"}, + {file = "protobuf-6.30.2.tar.gz", hash = "sha256:35c859ae076d8c56054c25b59e5e59638d86545ed6e2b6efac6be0b6ea3ba048"}, +] + +[[package]] +name = "pyclipper" +version = "1.3.0.post6" +description = "Cython wrapper for the C++ translation of the Angus Johnson's Clipper library (ver. 6.4.2)" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pyclipper-1.3.0.post6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fa0f5e78cfa8262277bb3d0225537b3c2a90ef68fd90a229d5d24cf49955dcf4"}, + {file = "pyclipper-1.3.0.post6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a01f182d8938c1dc515e8508ed2442f7eebd2c25c7d5cb29281f583c1a8008a4"}, + {file = "pyclipper-1.3.0.post6-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:640f20975727994d4abacd07396f564e9e5665ba5cb66ceb36b300c281f84fa4"}, + {file = "pyclipper-1.3.0.post6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a63002f6bb0f1efa87c0b81634cbb571066f237067e23707dabf746306c92ba5"}, + {file = "pyclipper-1.3.0.post6-cp310-cp310-win32.whl", hash = "sha256:106b8622cd9fb07d80cbf9b1d752334c55839203bae962376a8c59087788af26"}, + {file = "pyclipper-1.3.0.post6-cp310-cp310-win_amd64.whl", hash = "sha256:9699e98862dadefd0bea2360c31fa61ca553c660cbf6fb44993acde1b959f58f"}, + {file = "pyclipper-1.3.0.post6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4247e7c44b34c87acbf38f99d48fb1acaf5da4a2cf4dcd601a9b24d431be4ef"}, + {file = "pyclipper-1.3.0.post6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:851b3e58106c62a5534a1201295fe20c21714dee2eda68081b37ddb0367e6caa"}, + {file = "pyclipper-1.3.0.post6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16cc1705a915896d2aff52131c427df02265631279eac849ebda766432714cc0"}, + {file = "pyclipper-1.3.0.post6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ace1f0753cf71c5c5f6488b8feef5dd0fa8b976ad86b24bb51f708f513df4aac"}, + {file = "pyclipper-1.3.0.post6-cp311-cp311-win32.whl", hash = "sha256:dbc828641667142751b1127fd5c4291663490cf05689c85be4c5bcc89aaa236a"}, + {file = "pyclipper-1.3.0.post6-cp311-cp311-win_amd64.whl", hash = "sha256:1c03f1ae43b18ee07730c3c774cc3cf88a10c12a4b097239b33365ec24a0a14a"}, + {file = "pyclipper-1.3.0.post6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6363b9d79ba1b5d8f32d1623e797c1e9f994600943402e68d5266067bdde173e"}, + {file = "pyclipper-1.3.0.post6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32cd7fb9c1c893eb87f82a072dbb5e26224ea7cebbad9dc306d67e1ac62dd229"}, + {file = "pyclipper-1.3.0.post6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3aab10e3c10ed8fa60c608fb87c040089b83325c937f98f06450cf9fcfdaf1d"}, + {file = "pyclipper-1.3.0.post6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58eae2ff92a8cae1331568df076c4c5775bf946afab0068b217f0cf8e188eb3c"}, + {file = "pyclipper-1.3.0.post6-cp312-cp312-win32.whl", hash = "sha256:793b0aa54b914257aa7dc76b793dd4dcfb3c84011d48df7e41ba02b571616eaf"}, + {file = "pyclipper-1.3.0.post6-cp312-cp312-win_amd64.whl", hash = "sha256:d3f9da96f83b8892504923beb21a481cd4516c19be1d39eb57a92ef1c9a29548"}, + {file = "pyclipper-1.3.0.post6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f129284d2c7bcd213d11c0f35e1ae506a1144ce4954e9d1734d63b120b0a1b58"}, + {file = "pyclipper-1.3.0.post6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:188fbfd1d30d02247f92c25ce856f5f3c75d841251f43367dbcf10935bc48f38"}, + {file = "pyclipper-1.3.0.post6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6d129d0c2587f2f5904d201a4021f859afbb45fada4261c9fdedb2205b09d23"}, + {file = "pyclipper-1.3.0.post6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c9c80b5c46eef38ba3f12dd818dc87f5f2a0853ba914b6f91b133232315f526"}, + {file = "pyclipper-1.3.0.post6-cp313-cp313-win32.whl", hash = "sha256:b15113ec4fc423b58e9ae80aa95cf5a0802f02d8f02a98a46af3d7d66ff0cc0e"}, + {file = "pyclipper-1.3.0.post6-cp313-cp313-win_amd64.whl", hash = "sha256:e5ff68fa770ac654c7974fc78792978796f068bd274e95930c0691c31e192889"}, + {file = "pyclipper-1.3.0.post6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c92e41301a8f25f9adcd90954512038ed5f774a2b8c04a4a9db261b78ff75e3a"}, + {file = "pyclipper-1.3.0.post6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04214d23cf79f4ddcde36e299dea9f23f07abb88fa47ef399bf0e819438bbefd"}, + {file = "pyclipper-1.3.0.post6-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:aa604f8665ade434f9eafcd23f89435057d5d09427dfb4554c5e6d19f6d8aa1a"}, + {file = "pyclipper-1.3.0.post6-cp36-cp36m-win32.whl", hash = "sha256:1fd56855ca92fa7eb0d8a71cf3a24b80b9724c8adcc89b385bbaa8924e620156"}, + {file = "pyclipper-1.3.0.post6-cp36-cp36m-win_amd64.whl", hash = "sha256:6893f9b701f3132d86018594d99b724200b937a3a3ddfe1be0432c4ff0284e6e"}, + {file = "pyclipper-1.3.0.post6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2737df106b8487103916147fe30f887aff439d9f2bd2f67c9d9b5c13eac88ccf"}, + {file = "pyclipper-1.3.0.post6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33ab72260f144693e1f7735e93276c3031e1ed243a207eff1f8b98c7162ba22c"}, + {file = "pyclipper-1.3.0.post6-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:491ec1bfd2ee3013269c2b652dde14a85539480e0fb82f89bb12198fa59fff82"}, + {file = "pyclipper-1.3.0.post6-cp37-cp37m-win32.whl", hash = "sha256:2e257009030815853528ba4b2ef7fb7e172683a3f4255a63f00bde34cfab8b58"}, + {file = "pyclipper-1.3.0.post6-cp37-cp37m-win_amd64.whl", hash = "sha256:ed6e50c6e87ed190141573615d54118869bd63e9cd91ca5660d2ca926bf25110"}, + {file = "pyclipper-1.3.0.post6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:cf0a535cfa02b207435928e991c60389671fe1ea1dfae79170973f82f52335b2"}, + {file = "pyclipper-1.3.0.post6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:48dd55fbd55f63902cad511432ec332368cbbbc1dd2110c0c6c1e9edd735713a"}, + {file = "pyclipper-1.3.0.post6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05ae2ea878fdfa31dd375326f6191b03de98a9602cc9c2b6d4ff960b20a974c"}, + {file = "pyclipper-1.3.0.post6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:903176952a159c4195b8be55e597978e24804c838c7a9b12024c39704d341f72"}, + {file = "pyclipper-1.3.0.post6-cp38-cp38-win32.whl", hash = "sha256:fb1e52cf4ee0a9fa8b2254ed589cc51b0c989efc58fa8804289aca94a21253f7"}, + {file = "pyclipper-1.3.0.post6-cp38-cp38-win_amd64.whl", hash = "sha256:9cbdc517e75e647aa9bf6e356b3a3d2e3af344f82af38e36031eb46ba0ab5425"}, + {file = "pyclipper-1.3.0.post6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:383f3433b968f2e4b0843f338c1f63b85392b6e1d936de722e8c5d4f577dbff5"}, + {file = "pyclipper-1.3.0.post6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cf5ca2b9358d30a395ac6e14b3154a9fd1f9b557ad7153ea15cf697e88d07ce1"}, + {file = "pyclipper-1.3.0.post6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3404dfcb3415eee863564b5f49be28a8c7fb99ad5e31c986bcc33c8d47d97df7"}, + {file = "pyclipper-1.3.0.post6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:aa0e7268f8ceba218964bc3a482a5e9d32e352e8c3538b03f69a6b3db979078d"}, + {file = "pyclipper-1.3.0.post6-cp39-cp39-win32.whl", hash = "sha256:47a214f201ff930595a30649c2a063f78baa3a8f52e1f38da19f7930c90ed80c"}, + {file = "pyclipper-1.3.0.post6-cp39-cp39-win_amd64.whl", hash = "sha256:28bb590ae79e6beb15794eaee12b6f1d769589572d33e494faf5aa3b1f31b9fa"}, + {file = "pyclipper-1.3.0.post6-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3e5e65176506da6335f6cbab497ae1a29772064467fa69f66de6bab4b6304d34"}, + {file = "pyclipper-1.3.0.post6-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3d58202de8b8da4d1559afbda4e90a8c260a5373672b6d7bc5448c4614385144"}, + {file = "pyclipper-1.3.0.post6-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2cd8600bd16d209d5d45a33b45c278e1cc8bedc169af1a1f2187b581c521395"}, + {file = "pyclipper-1.3.0.post6.tar.gz", hash = "sha256:42bff0102fa7a7f2abdd795a2594654d62b786d0c6cd67b72d469114fdeb608c"}, +] + +[[package]] +name = "pydantic" +version = "2.11.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic-2.11.3-py3-none-any.whl", hash = "sha256:a082753436a07f9ba1289c6ffa01cd93db3548776088aa917cc43b63f68fa60f"}, + {file = "pydantic-2.11.3.tar.gz", hash = "sha256:7471657138c16adad9322fe3070c0116dd6c3ad8d649300e3cbdfe91f4db4ec3"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.33.1" +typing-extensions = ">=4.12.2" +typing-inspection = ">=0.4.0" + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] + +[[package]] +name = "pydantic-core" +version = "2.33.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic_core-2.33.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3077cfdb6125cc8dab61b155fdd714663e401f0e6883f9632118ec12cf42df26"}, + {file = "pydantic_core-2.33.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ffab8b2908d152e74862d276cf5017c81a2f3719f14e8e3e8d6b83fda863927"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5183e4f6a2d468787243ebcd70cf4098c247e60d73fb7d68d5bc1e1beaa0c4db"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:398a38d323f37714023be1e0285765f0a27243a8b1506b7b7de87b647b517e48"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87d3776f0001b43acebfa86f8c64019c043b55cc5a6a2e313d728b5c95b46969"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c566dd9c5f63d22226409553531f89de0cac55397f2ab8d97d6f06cfce6d947e"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d5f3acc81452c56895e90643a625302bd6be351e7010664151cc55b7b97f89"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d3a07fadec2a13274a8d861d3d37c61e97a816beae717efccaa4b36dfcaadcde"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f99aeda58dce827f76963ee87a0ebe75e648c72ff9ba1174a253f6744f518f65"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:902dbc832141aa0ec374f4310f1e4e7febeebc3256f00dc359a9ac3f264a45dc"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fe44d56aa0b00d66640aa84a3cbe80b7a3ccdc6f0b1ca71090696a6d4777c091"}, + {file = "pydantic_core-2.33.1-cp310-cp310-win32.whl", hash = "sha256:ed3eb16d51257c763539bde21e011092f127a2202692afaeaccb50db55a31383"}, + {file = "pydantic_core-2.33.1-cp310-cp310-win_amd64.whl", hash = "sha256:694ad99a7f6718c1a498dc170ca430687a39894a60327f548e02a9c7ee4b6504"}, + {file = "pydantic_core-2.33.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6e966fc3caaf9f1d96b349b0341c70c8d6573bf1bac7261f7b0ba88f96c56c24"}, + {file = "pydantic_core-2.33.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bfd0adeee563d59c598ceabddf2c92eec77abcb3f4a391b19aa7366170bd9e30"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91815221101ad3c6b507804178a7bb5cb7b2ead9ecd600041669c8d805ebd595"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fea9c1869bb4742d174a57b4700c6dadea951df8b06de40c2fedb4f02931c2e"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d20eb4861329bb2484c021b9d9a977566ab16d84000a57e28061151c62b349a"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb935c5591573ae3201640579f30128ccc10739b45663f93c06796854405505"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c964fd24e6166420d18fb53996d8c9fd6eac9bf5ae3ec3d03015be4414ce497f"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:681d65e9011f7392db5aa002b7423cc442d6a673c635668c227c6c8d0e5a4f77"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e100c52f7355a48413e2999bfb4e139d2977a904495441b374f3d4fb4a170961"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:048831bd363490be79acdd3232f74a0e9951b11b2b4cc058aeb72b22fdc3abe1"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bdc84017d28459c00db6f918a7272a5190bec3090058334e43a76afb279eac7c"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win32.whl", hash = "sha256:32cd11c5914d1179df70406427097c7dcde19fddf1418c787540f4b730289896"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win_amd64.whl", hash = "sha256:2ea62419ba8c397e7da28a9170a16219d310d2cf4970dbc65c32faf20d828c83"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win_arm64.whl", hash = "sha256:fc903512177361e868bc1f5b80ac8c8a6e05fcdd574a5fb5ffeac5a9982b9e89"}, + {file = "pydantic_core-2.33.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1293d7febb995e9d3ec3ea09caf1a26214eec45b0f29f6074abb004723fc1de8"}, + {file = "pydantic_core-2.33.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:99b56acd433386c8f20be5c4000786d1e7ca0523c8eefc995d14d79c7a081498"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35a5ec3fa8c2fe6c53e1b2ccc2454398f95d5393ab398478f53e1afbbeb4d939"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b172f7b9d2f3abc0efd12e3386f7e48b576ef309544ac3a63e5e9cdd2e24585d"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9097b9f17f91eea659b9ec58148c0747ec354a42f7389b9d50701610d86f812e"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc77ec5b7e2118b152b0d886c7514a4653bcb58c6b1d760134a9fab915f777b3"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3d15245b08fa4a84cefc6c9222e6f37c98111c8679fbd94aa145f9a0ae23d"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ef99779001d7ac2e2461d8ab55d3373fe7315caefdbecd8ced75304ae5a6fc6b"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fc6bf8869e193855e8d91d91f6bf59699a5cdfaa47a404e278e776dd7f168b39"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:b1caa0bc2741b043db7823843e1bde8aaa58a55a58fda06083b0569f8b45693a"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ec259f62538e8bf364903a7d0d0239447059f9434b284f5536e8402b7dd198db"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win32.whl", hash = "sha256:e14f369c98a7c15772b9da98987f58e2b509a93235582838bd0d1d8c08b68fda"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win_amd64.whl", hash = "sha256:1c607801d85e2e123357b3893f82c97a42856192997b95b4d8325deb1cd0c5f4"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d13f0276806ee722e70a1c93da19748594f19ac4299c7e41237fc791d1861ea"}, + {file = "pydantic_core-2.33.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:70af6a21237b53d1fe7b9325b20e65cbf2f0a848cf77bed492b029139701e66a"}, + {file = "pydantic_core-2.33.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:282b3fe1bbbe5ae35224a0dbd05aed9ccabccd241e8e6b60370484234b456266"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b315e596282bbb5822d0c7ee9d255595bd7506d1cb20c2911a4da0b970187d3"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1dfae24cf9921875ca0ca6a8ecb4bb2f13c855794ed0d468d6abbec6e6dcd44a"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6dd8ecfde08d8bfadaea669e83c63939af76f4cf5538a72597016edfa3fad516"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f593494876eae852dc98c43c6f260f45abdbfeec9e4324e31a481d948214764"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:948b73114f47fd7016088e5186d13faf5e1b2fe83f5e320e371f035557fd264d"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e11f3864eb516af21b01e25fac915a82e9ddad3bb0fb9e95a246067398b435a4"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:549150be302428b56fdad0c23c2741dcdb5572413776826c965619a25d9c6bde"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:495bc156026efafd9ef2d82372bd38afce78ddd82bf28ef5276c469e57c0c83e"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ec79de2a8680b1a67a07490bddf9636d5c2fab609ba8c57597e855fa5fa4dacd"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win32.whl", hash = "sha256:ee12a7be1742f81b8a65b36c6921022301d466b82d80315d215c4c691724986f"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win_amd64.whl", hash = "sha256:ede9b407e39949d2afc46385ce6bd6e11588660c26f80576c11c958e6647bc40"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win_arm64.whl", hash = "sha256:aa687a23d4b7871a00e03ca96a09cad0f28f443690d300500603bd0adba4b523"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:401d7b76e1000d0dd5538e6381d28febdcacb097c8d340dde7d7fc6e13e9f95d"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aeb055a42d734c0255c9e489ac67e75397d59c6fbe60d155851e9782f276a9c"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-win_amd64.whl", hash = "sha256:338ea9b73e6e109f15ab439e62cb3b78aa752c7fd9536794112e14bee02c8d18"}, + {file = "pydantic_core-2.33.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:5ab77f45d33d264de66e1884fca158bc920cb5e27fd0764a72f72f5756ae8bdb"}, + {file = "pydantic_core-2.33.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7aaba1b4b03aaea7bb59e1b5856d734be011d3e6d98f5bcaa98cb30f375f2ad"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fb66263e9ba8fea2aa85e1e5578980d127fb37d7f2e292773e7bc3a38fb0c7b"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3f2648b9262607a7fb41d782cc263b48032ff7a03a835581abbf7a3bec62bcf5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:723c5630c4259400818b4ad096735a829074601805d07f8cafc366d95786d331"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d100e3ae783d2167782391e0c1c7a20a31f55f8015f3293647544df3f9c67824"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177d50460bc976a0369920b6c744d927b0ecb8606fb56858ff542560251b19e5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3edde68d1a1f9af1273b2fe798997b33f90308fb6d44d8550c89fc6a3647cf6"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a62c3c3ef6a7e2c45f7853b10b5bc4ddefd6ee3cd31024754a1a5842da7d598d"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:c91dbb0ab683fa0cd64a6e81907c8ff41d6497c346890e26b23de7ee55353f96"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f466e8bf0a62dc43e068c12166281c2eca72121dd2adc1040f3aa1e21ef8599"}, + {file = "pydantic_core-2.33.1-cp39-cp39-win32.whl", hash = "sha256:ab0277cedb698749caada82e5d099dc9fed3f906a30d4c382d1a21725777a1e5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-win_amd64.whl", hash = "sha256:5773da0ee2d17136b1f1c6fbde543398d452a6ad2a7b54ea1033e2daa739b8d2"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c834f54f8f4640fd7e4b193f80eb25a0602bba9e19b3cd2fc7ffe8199f5ae02"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:049e0de24cf23766f12cc5cc71d8abc07d4a9deb9061b334b62093dedc7cb068"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a28239037b3d6f16916a4c831a5a0eadf856bdd6d2e92c10a0da3a59eadcf3e"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d3da303ab5f378a268fa7d45f37d7d85c3ec19769f28d2cc0c61826a8de21fe"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:25626fb37b3c543818c14821afe0fd3830bc327a43953bc88db924b68c5723f1"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3ab2d36e20fbfcce8f02d73c33a8a7362980cff717926bbae030b93ae46b56c7"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:2f9284e11c751b003fd4215ad92d325d92c9cb19ee6729ebd87e3250072cdcde"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:048c01eee07d37cbd066fc512b9d8b5ea88ceeb4e629ab94b3e56965ad655add"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5ccd429694cf26af7997595d627dd2637e7932214486f55b8a357edaac9dae8c"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3a371dc00282c4b84246509a5ddc808e61b9864aa1eae9ecc92bb1268b82db4a"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f59295ecc75a1788af8ba92f2e8c6eeaa5a94c22fc4d151e8d9638814f85c8fc"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08530b8ac922003033f399128505f513e30ca770527cc8bbacf75a84fcc2c74b"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae370459da6a5466978c0eacf90690cb57ec9d533f8e63e564ef3822bfa04fe"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e3de2777e3b9f4d603112f78006f4ae0acb936e95f06da6cb1a45fbad6bdb4b5"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3a64e81e8cba118e108d7126362ea30e021291b7805d47e4896e52c791be2761"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:52928d8c1b6bda03cc6d811e8923dffc87a2d3c8b3bfd2ce16471c7147a24850"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1b30d92c9412beb5ac6b10a3eb7ef92ccb14e3f2a8d7732e2d739f58b3aa7544"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f995719707e0e29f0f41a8aa3bcea6e761a36c9136104d3189eafb83f5cec5e5"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7edbc454a29fc6aeae1e1eecba4f07b63b8d76e76a748532233c4c167b4cb9ea"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ad05b683963f69a1d5d2c2bdab1274a31221ca737dbbceaa32bcb67359453cdd"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df6a94bf9452c6da9b5d76ed229a5683d0306ccb91cca8e1eea883189780d568"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7965c13b3967909a09ecc91f21d09cfc4576bf78140b988904e94f130f188396"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3f1fdb790440a34f6ecf7679e1863b825cb5ffde858a9197f851168ed08371e5"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5277aec8d879f8d05168fdd17ae811dd313b8ff894aeeaf7cd34ad28b4d77e33"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8ab581d3530611897d863d1a649fb0644b860286b4718db919bfd51ece41f10b"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0483847fa9ad5e3412265c1bd72aad35235512d9ce9d27d81a56d935ef489672"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:de9e06abe3cc5ec6a2d5f75bc99b0bdca4f5c719a5b34026f8c57efbdecd2ee3"}, + {file = "pydantic_core-2.33.1.tar.gz", hash = "sha256:bcc9c6fdb0ced789245b02b7d6603e17d1563064ddcfc36f046b61c0c05dd9df"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pyee" +version = "12.1.1" +description = "A rough port of Node.js's EventEmitter to Python with a few tricks of its own" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyee-12.1.1-py3-none-any.whl", hash = "sha256:18a19c650556bb6b32b406d7f017c8f513aceed1ef7ca618fb65de7bd2d347ef"}, + {file = "pyee-12.1.1.tar.gz", hash = "sha256:bbc33c09e2ff827f74191e3e5bbc6be7da02f627b7ec30d86f5ce1a6fb2424a3"}, +] + +[package.dependencies] +typing-extensions = "*" + +[package.extras] +dev = ["black", "build", "flake8", "flake8-black", "isort", "jupyter-console", "mkdocs", "mkdocs-include-markdown-plugin", "mkdocstrings[python]", "pytest", "pytest-asyncio ; python_version >= \"3.4\"", "pytest-trio ; python_version >= \"3.7\"", "sphinx", "toml", "tox", "trio", "trio ; python_version > \"3.6\"", "trio-typing ; python_version > \"3.6\"", "twine", "twisted", "validate-pyproject[all]"] + +[[package]] +name = "python-docx" +version = "1.1.2" +description = "Create, read, and update Microsoft Word .docx files." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "python_docx-1.1.2-py3-none-any.whl", hash = "sha256:08c20d6058916fb19853fcf080f7f42b6270d89eac9fa5f8c15f691c0017fabe"}, + {file = "python_docx-1.1.2.tar.gz", hash = "sha256:0cf1f22e95b9002addca7948e16f2cd7acdfd498047f1941ca5d293db7762efd"}, +] + +[package.dependencies] +lxml = ">=3.1.0" +typing-extensions = ">=4.9.0" + +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "rapidfuzz" +version = "3.13.0" +description = "rapid fuzzy string matching" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "rapidfuzz-3.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:aafc42a1dc5e1beeba52cd83baa41372228d6d8266f6d803c16dbabbcc156255"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:85c9a131a44a95f9cac2eb6e65531db014e09d89c4f18c7b1fa54979cb9ff1f3"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d7cec4242d30dd521ef91c0df872e14449d1dffc2a6990ede33943b0dae56c3"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e297c09972698c95649e89121e3550cee761ca3640cd005e24aaa2619175464e"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef0f5f03f61b0e5a57b1df7beafd83df993fd5811a09871bad6038d08e526d0d"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d8cf5f7cd6e4d5eb272baf6a54e182b2c237548d048e2882258336533f3f02b7"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9256218ac8f1a957806ec2fb9a6ddfc6c32ea937c0429e88cf16362a20ed8602"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e1bdd2e6d0c5f9706ef7595773a81ca2b40f3b33fd7f9840b726fb00c6c4eb2e"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5280be8fd7e2bee5822e254fe0a5763aa0ad57054b85a32a3d9970e9b09bbcbf"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fd742c03885db1fce798a1cd87a20f47f144ccf26d75d52feb6f2bae3d57af05"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5435fcac94c9ecf0504bf88a8a60c55482c32e18e108d6079a0089c47f3f8cf6"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:93a755266856599be4ab6346273f192acde3102d7aa0735e2f48b456397a041f"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-win32.whl", hash = "sha256:3abe6a4e8eb4cfc4cda04dd650a2dc6d2934cbdeda5def7e6fd1c20f6e7d2a0b"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:e8ddb58961401da7d6f55f185512c0d6bd24f529a637078d41dd8ffa5a49c107"}, + {file = "rapidfuzz-3.13.0-cp310-cp310-win_arm64.whl", hash = "sha256:c523620d14ebd03a8d473c89e05fa1ae152821920c3ff78b839218ff69e19ca3"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d395a5cad0c09c7f096433e5fd4224d83b53298d53499945a9b0e5a971a84f3a"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b7b3eda607a019169f7187328a8d1648fb9a90265087f6903d7ee3a8eee01805"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98e0bfa602e1942d542de077baf15d658bd9d5dcfe9b762aff791724c1c38b70"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bef86df6d59667d9655905b02770a0c776d2853971c0773767d5ef8077acd624"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fedd316c165beed6307bf754dee54d3faca2c47e1f3bcbd67595001dfa11e969"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5158da7f2ec02a930be13bac53bb5903527c073c90ee37804090614cab83c29e"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b6f913ee4618ddb6d6f3e387b76e8ec2fc5efee313a128809fbd44e65c2bbb2"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d25fdbce6459ccbbbf23b4b044f56fbd1158b97ac50994eaae2a1c0baae78301"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25343ccc589a4579fbde832e6a1e27258bfdd7f2eb0f28cb836d6694ab8591fc"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a9ad1f37894e3ffb76bbab76256e8a8b789657183870be11aa64e306bb5228fd"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5dc71ef23845bb6b62d194c39a97bb30ff171389c9812d83030c1199f319098c"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b7f4c65facdb94f44be759bbd9b6dda1fa54d0d6169cdf1a209a5ab97d311a75"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-win32.whl", hash = "sha256:b5104b62711565e0ff6deab2a8f5dbf1fbe333c5155abe26d2cfd6f1849b6c87"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:9093cdeb926deb32a4887ebe6910f57fbcdbc9fbfa52252c10b56ef2efb0289f"}, + {file = "rapidfuzz-3.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:f70f646751b6aa9d05be1fb40372f006cc89d6aad54e9d79ae97bd1f5fce5203"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a1a6a906ba62f2556372282b1ef37b26bca67e3d2ea957277cfcefc6275cca7"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fd0975e015b05c79a97f38883a11236f5a24cca83aa992bd2558ceaa5652b26"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d4e13593d298c50c4f94ce453f757b4b398af3fa0fd2fde693c3e51195b7f69"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed6f416bda1c9133000009d84d9409823eb2358df0950231cc936e4bf784eb97"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dc82b6ed01acb536b94a43996a94471a218f4d89f3fdd9185ab496de4b2a981"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9d824de871daa6e443b39ff495a884931970d567eb0dfa213d234337343835f"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d18228a2390375cf45726ce1af9d36ff3dc1f11dce9775eae1f1b13ac6ec50f"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f5fe634c9482ec5d4a6692afb8c45d370ae86755e5f57aa6c50bfe4ca2bdd87"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:694eb531889f71022b2be86f625a4209c4049e74be9ca836919b9e395d5e33b3"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11b47b40650e06147dee5e51a9c9ad73bb7b86968b6f7d30e503b9f8dd1292db"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:98b8107ff14f5af0243f27d236bcc6e1ef8e7e3b3c25df114e91e3a99572da73"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b836f486dba0aceb2551e838ff3f514a38ee72b015364f739e526d720fdb823a"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-win32.whl", hash = "sha256:4671ee300d1818d7bdfd8fa0608580d7778ba701817216f0c17fb29e6b972514"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e2065f68fb1d0bf65adc289c1bdc45ba7e464e406b319d67bb54441a1b9da9e"}, + {file = "rapidfuzz-3.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:65cc97c2fc2c2fe23586599686f3b1ceeedeca8e598cfcc1b7e56dc8ca7e2aa7"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:09e908064d3684c541d312bd4c7b05acb99a2c764f6231bd507d4b4b65226c23"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:57c390336cb50d5d3bfb0cfe1467478a15733703af61f6dffb14b1cd312a6fae"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0da54aa8547b3c2c188db3d1c7eb4d1bb6dd80baa8cdaeaec3d1da3346ec9caa"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df8e8c21e67afb9d7fbe18f42c6111fe155e801ab103c81109a61312927cc611"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:461fd13250a2adf8e90ca9a0e1e166515cbcaa5e9c3b1f37545cbbeff9e77f6b"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2b3dd5d206a12deca16870acc0d6e5036abeb70e3cad6549c294eff15591527"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1343d745fbf4688e412d8f398c6e6d6f269db99a54456873f232ba2e7aeb4939"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b1b065f370d54551dcc785c6f9eeb5bd517ae14c983d2784c064b3aa525896df"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:11b125d8edd67e767b2295eac6eb9afe0b1cdc82ea3d4b9257da4b8e06077798"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c33f9c841630b2bb7e69a3fb5c84a854075bb812c47620978bddc591f764da3d"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ae4574cb66cf1e85d32bb7e9ec45af5409c5b3970b7ceb8dea90168024127566"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e05752418b24bbd411841b256344c26f57da1148c5509e34ea39c7eb5099ab72"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-win32.whl", hash = "sha256:0e1d08cb884805a543f2de1f6744069495ef527e279e05370dd7c83416af83f8"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9a7c6232be5f809cd39da30ee5d24e6cadd919831e6020ec6c2391f4c3bc9264"}, + {file = "rapidfuzz-3.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:3f32f15bacd1838c929b35c84b43618481e1b3d7a61b5ed2db0291b70ae88b53"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc64da907114d7a18b5e589057e3acaf2fec723d31c49e13fedf043592a3f6a7"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4d9d7f84c8e992a8dbe5a3fdbea73d733da39bf464e62c912ac3ceba9c0cff93"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a79a2f07786a2070669b4b8e45bd96a01c788e7a3c218f531f3947878e0f956"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f338e71c45b69a482de8b11bf4a029993230760120c8c6e7c9b71760b6825a1"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adb40ca8ddfcd4edd07b0713a860be32bdf632687f656963bcbce84cea04b8d8"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48719f7dcf62dfb181063b60ee2d0a39d327fa8ad81b05e3e510680c44e1c078"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9327a4577f65fc3fb712e79f78233815b8a1c94433d0c2c9f6bc5953018b3565"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:200030dfc0a1d5d6ac18e993c5097c870c97c41574e67f227300a1fb74457b1d"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:cc269e74cad6043cb8a46d0ce580031ab642b5930562c2bb79aa7fbf9c858d26"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:e62779c6371bd2b21dbd1fdce89eaec2d93fd98179d36f61130b489f62294a92"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:f4797f821dc5d7c2b6fc818b89f8a3f37bcc900dd9e4369e6ebf1e525efce5db"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d21f188f6fe4fbf422e647ae9d5a68671d00218e187f91859c963d0738ccd88c"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-win32.whl", hash = "sha256:45dd4628dd9c21acc5c97627dad0bb791764feea81436fb6e0a06eef4c6dceaa"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:624a108122039af89ddda1a2b7ab2a11abe60c1521956f142f5d11bcd42ef138"}, + {file = "rapidfuzz-3.13.0-cp39-cp39-win_arm64.whl", hash = "sha256:435071fd07a085ecbf4d28702a66fd2e676a03369ee497cc38bcb69a46bc77e2"}, + {file = "rapidfuzz-3.13.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:fe5790a36d33a5d0a6a1f802aa42ecae282bf29ac6f7506d8e12510847b82a45"}, + {file = "rapidfuzz-3.13.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:cdb33ee9f8a8e4742c6b268fa6bd739024f34651a06b26913381b1413ebe7590"}, + {file = "rapidfuzz-3.13.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c99b76b93f7b495eee7dcb0d6a38fb3ce91e72e99d9f78faa5664a881cb2b7d"}, + {file = "rapidfuzz-3.13.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6af42f2ede8b596a6aaf6d49fdee3066ca578f4856b85ab5c1e2145de367a12d"}, + {file = "rapidfuzz-3.13.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c0efa73afbc5b265aca0d8a467ae2a3f40d6854cbe1481cb442a62b7bf23c99"}, + {file = "rapidfuzz-3.13.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7ac21489de962a4e2fc1e8f0b0da4aa1adc6ab9512fd845563fecb4b4c52093a"}, + {file = "rapidfuzz-3.13.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1ba007f4d35a45ee68656b2eb83b8715e11d0f90e5b9f02d615a8a321ff00c27"}, + {file = "rapidfuzz-3.13.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d7a217310429b43be95b3b8ad7f8fc41aba341109dc91e978cd7c703f928c58f"}, + {file = "rapidfuzz-3.13.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:558bf526bcd777de32b7885790a95a9548ffdcce68f704a81207be4a286c1095"}, + {file = "rapidfuzz-3.13.0-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:202a87760f5145140d56153b193a797ae9338f7939eb16652dd7ff96f8faf64c"}, + {file = "rapidfuzz-3.13.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfcccc08f671646ccb1e413c773bb92e7bba789e3a1796fd49d23c12539fe2e4"}, + {file = "rapidfuzz-3.13.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1f219f1e3c3194d7a7de222f54450ce12bc907862ff9a8962d83061c1f923c86"}, + {file = "rapidfuzz-3.13.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ccbd0e7ea1a216315f63ffdc7cd09c55f57851afc8fe59a74184cb7316c0598b"}, + {file = "rapidfuzz-3.13.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a50856f49a4016ef56edd10caabdaf3608993f9faf1e05c3c7f4beeac46bd12a"}, + {file = "rapidfuzz-3.13.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fd05336db4d0b8348d7eaaf6fa3c517b11a56abaa5e89470ce1714e73e4aca7"}, + {file = "rapidfuzz-3.13.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:573ad267eb9b3f6e9b04febce5de55d8538a87c56c64bf8fd2599a48dc9d8b77"}, + {file = "rapidfuzz-3.13.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30fd1451f87ccb6c2f9d18f6caa483116bbb57b5a55d04d3ddbd7b86f5b14998"}, + {file = "rapidfuzz-3.13.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6dd36d4916cf57ddb05286ed40b09d034ca5d4bca85c17be0cb6a21290597d9"}, + {file = "rapidfuzz-3.13.0.tar.gz", hash = "sha256:d2eaf3839e52cbcc0accbe9817a67b4b0fcf70aaeb229cfddc1c28061f9ce5d8"}, +] + +[package.extras] +all = ["numpy"] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "scikit-image" +version = "0.25.2" +description = "Image processing in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "scikit_image-0.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d3278f586793176599df6a4cf48cb6beadae35c31e58dc01a98023af3dc31c78"}, + {file = "scikit_image-0.25.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:5c311069899ce757d7dbf1d03e32acb38bb06153236ae77fcd820fd62044c063"}, + {file = "scikit_image-0.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be455aa7039a6afa54e84f9e38293733a2622b8c2fb3362b822d459cc5605e99"}, + {file = "scikit_image-0.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4c464b90e978d137330be433df4e76d92ad3c5f46a22f159520ce0fdbea8a09"}, + {file = "scikit_image-0.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:60516257c5a2d2f74387c502aa2f15a0ef3498fbeaa749f730ab18f0a40fd054"}, + {file = "scikit_image-0.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f4bac9196fb80d37567316581c6060763b0f4893d3aca34a9ede3825bc035b17"}, + {file = "scikit_image-0.25.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:d989d64ff92e0c6c0f2018c7495a5b20e2451839299a018e0e5108b2680f71e0"}, + {file = "scikit_image-0.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2cfc96b27afe9a05bc92f8c6235321d3a66499995675b27415e0d0c76625173"}, + {file = "scikit_image-0.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24cc986e1f4187a12aa319f777b36008764e856e5013666a4a83f8df083c2641"}, + {file = "scikit_image-0.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:b4f6b61fc2db6340696afe3db6b26e0356911529f5f6aee8c322aa5157490c9b"}, + {file = "scikit_image-0.25.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8db8dd03663112783221bf01ccfc9512d1cc50ac9b5b0fe8f4023967564719fb"}, + {file = "scikit_image-0.25.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:483bd8cc10c3d8a7a37fae36dfa5b21e239bd4ee121d91cad1f81bba10cfb0ed"}, + {file = "scikit_image-0.25.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d1e80107bcf2bf1291acfc0bf0425dceb8890abe9f38d8e94e23497cbf7ee0d"}, + {file = "scikit_image-0.25.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a17e17eb8562660cc0d31bb55643a4da996a81944b82c54805c91b3fe66f4824"}, + {file = "scikit_image-0.25.2-cp312-cp312-win_amd64.whl", hash = "sha256:bdd2b8c1de0849964dbc54037f36b4e9420157e67e45a8709a80d727f52c7da2"}, + {file = "scikit_image-0.25.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7efa888130f6c548ec0439b1a7ed7295bc10105458a421e9bf739b457730b6da"}, + {file = "scikit_image-0.25.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:dd8011efe69c3641920614d550f5505f83658fe33581e49bed86feab43a180fc"}, + {file = "scikit_image-0.25.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28182a9d3e2ce3c2e251383bdda68f8d88d9fff1a3ebe1eb61206595c9773341"}, + {file = "scikit_image-0.25.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8abd3c805ce6944b941cfed0406d88faeb19bab3ed3d4b50187af55cf24d147"}, + {file = "scikit_image-0.25.2-cp313-cp313-win_amd64.whl", hash = "sha256:64785a8acefee460ec49a354706db0b09d1f325674107d7fa3eadb663fb56d6f"}, + {file = "scikit_image-0.25.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:330d061bd107d12f8d68f1d611ae27b3b813b8cdb0300a71d07b1379178dd4cd"}, + {file = "scikit_image-0.25.2.tar.gz", hash = "sha256:e5a37e6cd4d0c018a7a55b9d601357e3382826d3888c10d0213fc63bff977dde"}, +] + +[package.dependencies] +imageio = ">=2.33,<2.35.0 || >2.35.0" +lazy-loader = ">=0.4" +networkx = ">=3.0" +numpy = ">=1.24" +packaging = ">=21" +pillow = ">=10.1" +scipy = ">=1.11.4" +tifffile = ">=2022.8.12" + +[package.extras] +build = ["Cython (>=3.0.8)", "build (>=1.2.1)", "meson-python (>=0.16)", "ninja (>=1.11.1.1)", "numpy (>=2.0)", "pythran (>=0.16)", "spin (==0.13)"] +data = ["pooch (>=1.6.0)"] +developer = ["ipython", "pre-commit", "tomli ; python_version < \"3.11\""] +docs = ["PyWavelets (>=1.6)", "dask[array] (>=2023.2.0)", "intersphinx-registry (>=0.2411.14)", "ipykernel", "ipywidgets", "kaleido (==0.2.1)", "matplotlib (>=3.7)", "myst-parser", "numpydoc (>=1.7)", "pandas (>=2.0)", "plotly (>=5.20)", "pooch (>=1.6)", "pydata-sphinx-theme (>=0.16)", "pytest-doctestplus", "scikit-learn (>=1.2)", "seaborn (>=0.11)", "sphinx (>=8.0)", "sphinx-copybutton", "sphinx-gallery[parallel] (>=0.18)", "sphinx_design (>=0.5)", "tifffile (>=2022.8.12)"] +optional = ["PyWavelets (>=1.6)", "SimpleITK", "astropy (>=5.0)", "cloudpickle (>=1.1.1)", "dask[array] (>=2023.2.0)", "matplotlib (>=3.7)", "pooch (>=1.6.0)", "pyamg (>=5.2)", "scikit-learn (>=1.2)"] +test = ["asv", "numpydoc (>=1.7)", "pooch (>=1.6.0)", "pytest (>=8)", "pytest-cov (>=2.11.0)", "pytest-doctestplus", "pytest-faulthandler", "pytest-localserver"] + +[[package]] +name = "scipy" +version = "1.15.2" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "scipy-1.15.2-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a2ec871edaa863e8213ea5df811cd600734f6400b4af272e1c011e69401218e9"}, + {file = "scipy-1.15.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:6f223753c6ea76983af380787611ae1291e3ceb23917393079dcc746ba60cfb5"}, + {file = "scipy-1.15.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:ecf797d2d798cf7c838c6d98321061eb3e72a74710e6c40540f0e8087e3b499e"}, + {file = "scipy-1.15.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:9b18aa747da280664642997e65aab1dd19d0c3d17068a04b3fe34e2559196cb9"}, + {file = "scipy-1.15.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:87994da02e73549dfecaed9e09a4f9d58a045a053865679aeb8d6d43747d4df3"}, + {file = "scipy-1.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69ea6e56d00977f355c0f84eba69877b6df084516c602d93a33812aa04d90a3d"}, + {file = "scipy-1.15.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:888307125ea0c4466287191e5606a2c910963405ce9671448ff9c81c53f85f58"}, + {file = "scipy-1.15.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9412f5e408b397ff5641080ed1e798623dbe1ec0d78e72c9eca8992976fa65aa"}, + {file = "scipy-1.15.2-cp310-cp310-win_amd64.whl", hash = "sha256:b5e025e903b4f166ea03b109bb241355b9c42c279ea694d8864d033727205e65"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:92233b2df6938147be6fa8824b8136f29a18f016ecde986666be5f4d686a91a4"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:62ca1ff3eb513e09ed17a5736929429189adf16d2d740f44e53270cc800ecff1"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4c6676490ad76d1c2894d77f976144b41bd1a4052107902238047fb6a473e971"}, + {file = "scipy-1.15.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a8bf5cb4a25046ac61d38f8d3c3426ec11ebc350246a4642f2f315fe95bda655"}, + {file = "scipy-1.15.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a8e34cf4c188b6dd004654f88586d78f95639e48a25dfae9c5e34a6dc34547e"}, + {file = "scipy-1.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28a0d2c2075946346e4408b211240764759e0fabaeb08d871639b5f3b1aca8a0"}, + {file = "scipy-1.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:42dabaaa798e987c425ed76062794e93a243be8f0f20fff6e7a89f4d61cb3d40"}, + {file = "scipy-1.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6f5e296ec63c5da6ba6fa0343ea73fd51b8b3e1a300b0a8cae3ed4b1122c7462"}, + {file = "scipy-1.15.2-cp311-cp311-win_amd64.whl", hash = "sha256:597a0c7008b21c035831c39927406c6181bcf8f60a73f36219b69d010aa04737"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c4697a10da8f8765bb7c83e24a470da5797e37041edfd77fd95ba3811a47c4fd"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:869269b767d5ee7ea6991ed7e22b3ca1f22de73ab9a49c44bad338b725603301"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:bad78d580270a4d32470563ea86c6590b465cb98f83d760ff5b0990cb5518a93"}, + {file = "scipy-1.15.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:b09ae80010f52efddb15551025f9016c910296cf70adbf03ce2a8704f3a5ad20"}, + {file = "scipy-1.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a6fd6eac1ce74a9f77a7fc724080d507c5812d61e72bd5e4c489b042455865e"}, + {file = "scipy-1.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b871df1fe1a3ba85d90e22742b93584f8d2b8e6124f8372ab15c71b73e428b8"}, + {file = "scipy-1.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:03205d57a28e18dfd39f0377d5002725bf1f19a46f444108c29bdb246b6c8a11"}, + {file = "scipy-1.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:601881dfb761311045b03114c5fe718a12634e5608c3b403737ae463c9885d53"}, + {file = "scipy-1.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:e7c68b6a43259ba0aab737237876e5c2c549a031ddb7abc28c7b47f22e202ded"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:01edfac9f0798ad6b46d9c4c9ca0e0ad23dbf0b1eb70e96adb9fa7f525eff0bf"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:08b57a9336b8e79b305a143c3655cc5bdbe6d5ece3378578888d2afbb51c4e37"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:54c462098484e7466362a9f1672d20888f724911a74c22ae35b61f9c5919183d"}, + {file = "scipy-1.15.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:cf72ff559a53a6a6d77bd8eefd12a17995ffa44ad86c77a5df96f533d4e6c6bb"}, + {file = "scipy-1.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9de9d1416b3d9e7df9923ab23cd2fe714244af10b763975bea9e4f2e81cebd27"}, + {file = "scipy-1.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb530e4794fc8ea76a4a21ccb67dea33e5e0e60f07fc38a49e821e1eae3b71a0"}, + {file = "scipy-1.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5ea7ed46d437fc52350b028b1d44e002646e28f3e8ddc714011aaf87330f2f32"}, + {file = "scipy-1.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:11e7ad32cf184b74380f43d3c0a706f49358b904fa7d5345f16ddf993609184d"}, + {file = "scipy-1.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:a5080a79dfb9b78b768cebf3c9dcbc7b665c5875793569f48bf0e2b1d7f68f6f"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:447ce30cee6a9d5d1379087c9e474628dab3db4a67484be1b7dc3196bfb2fac9"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c90ebe8aaa4397eaefa8455a8182b164a6cc1d59ad53f79943f266d99f68687f"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:def751dd08243934c884a3221156d63e15234a3155cf25978b0a668409d45eb6"}, + {file = "scipy-1.15.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:302093e7dfb120e55515936cb55618ee0b895f8bcaf18ff81eca086c17bd80af"}, + {file = "scipy-1.15.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd5b77413e1855351cdde594eca99c1f4a588c2d63711388b6a1f1c01f62274"}, + {file = "scipy-1.15.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d0194c37037707b2afa7a2f2a924cf7bac3dc292d51b6a925e5fcb89bc5c776"}, + {file = "scipy-1.15.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:bae43364d600fdc3ac327db99659dcb79e6e7ecd279a75fe1266669d9a652828"}, + {file = "scipy-1.15.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f031846580d9acccd0044efd1a90e6f4df3a6e12b4b6bd694a7bc03a89892b28"}, + {file = "scipy-1.15.2-cp313-cp313t-win_amd64.whl", hash = "sha256:fe8a9eb875d430d81755472c5ba75e84acc980e4a8f6204d402849234d3017db"}, + {file = "scipy-1.15.2.tar.gz", hash = "sha256:cd58a314d92838f7e6f755c8a2167ead4f27e1fd5c1251fd54289569ef3495ec"}, +] + +[package.dependencies] +numpy = ">=1.23.5,<2.5" + +[package.extras] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] +doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.16.5)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + +[[package]] +name = "setuptools" +version = "79.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "setuptools-79.0.0-py3-none-any.whl", hash = "sha256:b9ab3a104bedb292323f53797b00864e10e434a3ab3906813a7169e4745b912a"}, + {file = "setuptools-79.0.0.tar.gz", hash = "sha256:9828422e7541213b0aacb6e10bbf9dd8febeaa45a48570e09b6d100e063fc9f9"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] + +[[package]] +name = "shapely" +version = "2.1.0" +description = "Manipulation and analysis of geometric objects" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "shapely-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d3e5c5e3864d4dc431dd85a8e5137ebd39c8ac287b009d3fa80a07017b29c940"}, + {file = "shapely-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6eea89b16f5f3a064659126455d23fa3066bc3d6cd385c35214f06bf5871aa6"}, + {file = "shapely-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:183174ad0b21a81ee661f05e7c47aa92ebfae01814cd3cbe54adea7a4213f5f4"}, + {file = "shapely-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f239c1484af66bc14b81a76f2a8e0fada29d59010423253ff857d0ccefdaa93f"}, + {file = "shapely-2.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6220a466d1475141dad0cd8065d2549a5c2ed3fa4e2e02fb8ea65d494cfd5b07"}, + {file = "shapely-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4822d3ed3efb06145c34d29d5b56792f72b7d713300f603bfd5d825892c6f79f"}, + {file = "shapely-2.1.0-cp310-cp310-win32.whl", hash = "sha256:ea51ddf3d3c60866dca746081b56c75f34ff1b01acbd4d44269071a673c735b9"}, + {file = "shapely-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:a6f5e02e2cded9f4ec5709900a296c7f2cce5f8e9e9d80ba7d89ae2f4ed89d7b"}, + {file = "shapely-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8323031ef7c1bdda7a92d5ddbc7b6b62702e73ba37e9a8ccc8da99ec2c0b87c"}, + {file = "shapely-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4da7c6cd748d86ec6aace99ad17129d30954ccf5e73e9911cdb5f0fa9658b4f8"}, + {file = "shapely-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f0cdf85ff80831137067e7a237085a3ee72c225dba1b30beef87f7d396cf02b"}, + {file = "shapely-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41f2be5d79aac39886f23000727cf02001aef3af8810176c29ee12cdc3ef3a50"}, + {file = "shapely-2.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:21a4515009f56d7a159cf5c2554264e82f56405b4721f9a422cb397237c5dca8"}, + {file = "shapely-2.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:15cebc323cec2cb6b2eaa310fdfc621f6dbbfaf6bde336d13838fcea76c885a9"}, + {file = "shapely-2.1.0-cp311-cp311-win32.whl", hash = "sha256:cad51b7a5c8f82f5640472944a74f0f239123dde9a63042b3c5ea311739b7d20"}, + {file = "shapely-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:d4005309dde8658e287ad9c435c81877f6a95a9419b932fa7a1f34b120f270ae"}, + {file = "shapely-2.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:53e7ee8bd8609cf12ee6dce01ea5affe676976cf7049315751d53d8db6d2b4b2"}, + {file = "shapely-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cab20b665d26dbec0b380e15749bea720885a481fa7b1eedc88195d4a98cfa4"}, + {file = "shapely-2.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4a38b39a09340273c3c92b3b9a374272a12cc7e468aeeea22c1c46217a03e5c"}, + {file = "shapely-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:edaec656bdd9b71278b98e6f77c464b1c3b2daa9eace78012ff0f0b4b5b15b04"}, + {file = "shapely-2.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c8a732ddd9b25e7a54aa748e7df8fd704e23e5d5d35b7d376d80bffbfc376d04"}, + {file = "shapely-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9c93693ad8adfdc9138a5a2d42da02da94f728dd2e82d2f0f442f10e25027f5f"}, + {file = "shapely-2.1.0-cp312-cp312-win32.whl", hash = "sha256:d8ac6604eefe807e71a908524de23a37920133a1729fe3a4dfe0ed82c044cbf4"}, + {file = "shapely-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:f4f47e631aa4f9ec5576eac546eb3f38802e2f82aeb0552f9612cb9a14ece1db"}, + {file = "shapely-2.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b64423295b563f43a043eb786e7a03200ebe68698e36d2b4b1c39f31dfb50dfb"}, + {file = "shapely-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1b5578f45adc25b235b22d1ccb9a0348c8dc36f31983e57ea129a88f96f7b870"}, + {file = "shapely-2.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a7e83d383b27f02b684e50ab7f34e511c92e33b6ca164a6a9065705dd64bcb"}, + {file = "shapely-2.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:942031eb4d8f7b3b22f43ba42c09c7aa3d843aa10d5cc1619fe816e923b66e55"}, + {file = "shapely-2.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d2843c456a2e5627ee6271800f07277c0d2652fb287bf66464571a057dbc00b3"}, + {file = "shapely-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8c4b17469b7f39a5e6a7cfea79f38ae08a275427f41fe8b48c372e1449147908"}, + {file = "shapely-2.1.0-cp313-cp313-win32.whl", hash = "sha256:30e967abd08fce49513d4187c01b19f139084019f33bec0673e8dbeb557c45e4"}, + {file = "shapely-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:1dc8d4364483a14aba4c844b7bd16a6fa3728887e2c33dfa1afa34a3cf4d08a5"}, + {file = "shapely-2.1.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:673e073fea099d1c82f666fb7ab0a00a77eff2999130a69357ce11941260d855"}, + {file = "shapely-2.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:6d1513f915a56de67659fe2047c1ad5ff0f8cbff3519d1e74fced69c9cb0e7da"}, + {file = "shapely-2.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d6a7043178890b9e028d80496ff4c79dc7629bff4d78a2f25323b661756bab8"}, + {file = "shapely-2.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb638378dc3d76f7e85b67d7e2bb1366811912430ac9247ac00c127c2b444cdc"}, + {file = "shapely-2.1.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:737124e87d91d616acf9a911f74ac55e05db02a43a6a7245b3d663817b876055"}, + {file = "shapely-2.1.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e6c229e7bb87aae5df82fa00b6718987a43ec168cc5affe095cca59d233f314"}, + {file = "shapely-2.1.0-cp313-cp313t-win32.whl", hash = "sha256:a9580bda119b1f42f955aa8e52382d5c73f7957e0203bc0c0c60084846f3db94"}, + {file = "shapely-2.1.0-cp313-cp313t-win_amd64.whl", hash = "sha256:e8ff4e5cfd799ba5b6f37b5d5527dbd85b4a47c65b6d459a03d0962d2a9d4d10"}, + {file = "shapely-2.1.0.tar.gz", hash = "sha256:2cbe90e86fa8fc3ca8af6ffb00a77b246b918c7cf28677b7c21489b678f6b02e"}, +] + +[package.dependencies] +numpy = ">=1.21" + +[package.extras] +docs = ["matplotlib", "numpydoc (==1.1.*)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"] +test = ["pytest", "pytest-cov", "scipy-doctest"] + +[[package]] +name = "simsimd" +version = "6.2.1" +description = "Portable mixed-precision BLAS-like vector math library for x86 and ARM" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "simsimd-6.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9c79486cf75eb06c5e1f623e8315f9fb73620ac63b846d5a6c843f14905de43f"}, + {file = "simsimd-6.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:104d53f2489dcbf569b8260d678e2183af605510115dc2b22ed0340aa47fe892"}, + {file = "simsimd-6.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fef886c8220d3566b9f43d441226ca267a11682dea5496bb6e007f655eee1fd1"}, + {file = "simsimd-6.2.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:522e56451481bff3468653c2818ad1240b4cb13cff0ec76bc88d8860bfc775c9"}, + {file = "simsimd-6.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5dfb02fa141a6e039803044930753aef1df5ed05cae8b14fe348cdc160cef1e"}, + {file = "simsimd-6.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39eb6abdd44adfddec181a713e9cfad8742d03abbc6247c4e5ca2caee38e4775"}, + {file = "simsimd-6.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:9ca68b9d2cc1c19af6afe6f01a764861fc8bb919d688a64cf0b0ac0abae7e0fa"}, + {file = "simsimd-6.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:2b56b1ca7b76c0d4515938a036e688b73a866b19e6f6eb743596144fdf498a0c"}, + {file = "simsimd-6.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:02d7b7c7afecc63ddf501460f09c1da90625bfd59b4da5fda126c1aa5c54bb95"}, + {file = "simsimd-6.2.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:8abc529daf0a61649ca4a237cd9e63723f3355394686898654c643bd63846cf5"}, + {file = "simsimd-6.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9ea60422d0f45d3a1899984c3fc3a14dbd248cfca8f67c24751029441464a806"}, + {file = "simsimd-6.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:98e38a0ca4805c1de2882d0641b54e249eabca4ed2980c82465822130d7f8c98"}, + {file = "simsimd-6.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cbbc2434286493b88f3b8211e922d37b46588b34d4cc28f3262f154c8ca1141c"}, + {file = "simsimd-6.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4f2ecd459f4917facdb287c42c5e68030b21cb98edac0fec9919a7215968e38a"}, + {file = "simsimd-6.2.1-cp310-cp310-win32.whl", hash = "sha256:4ec31c076dc839114bff5d83526ddf46551d4720cc8cd0f16516896809a4fca6"}, + {file = "simsimd-6.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:94282e040be985c993d415290371f6b22bec3eeadafe747a6d8dfbd2c317f35e"}, + {file = "simsimd-6.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:0784e98ca48a0075fb0cbd7782df11eaa17ce15c60f09a65e8477864208afb8a"}, + {file = "simsimd-6.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e9614309af75be4d08a051dc61ed5cf41b5239b8303b37dc2f9c8a7223534392"}, + {file = "simsimd-6.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ea4f0f68be5f85bbcf4322bfdd1b449176cf5fdd99960c546514457635632443"}, + {file = "simsimd-6.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:12a8d60ccc8991dfbbf056c221ce4f02135f5892492894972f421a6f155015d9"}, + {file = "simsimd-6.2.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a74142ea21a6fd3ec5c64e4d4acf1ec6f4d80c0bb1a5989d68af6e84f7ac612e"}, + {file = "simsimd-6.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:298f7c793fc2a1eeedcefa1278eb2ef6f52ce0b36aaa8780885f96a39ce1a4e8"}, + {file = "simsimd-6.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4025ebad36fb3fa5cffcd48d33375d5e5decc59c1129a259b74fed097eab1ab5"}, + {file = "simsimd-6.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:f486682aa7a8918d86df411d3c11c635db4b67d514cb6bb499c0edab7fb8ec58"}, + {file = "simsimd-6.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:173e66699597a4fcf6fa50b52cced40216fdcfba15f60b761a2bd9cb1d98a444"}, + {file = "simsimd-6.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b5c6f79f797cc020a2ff64950162dfb6d130c51a07cdac5ad97ec836e85ce50"}, + {file = "simsimd-6.2.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:25812637f43feaef1a33ae00b81a4d2b0116aadae3a08267486c1e57236fc368"}, + {file = "simsimd-6.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:592a578c788a9cb7877eff41487cc7f50474e00f774de74bea8590fa95c804ae"}, + {file = "simsimd-6.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:191c020f312350ac06eee829376b11d8c1282da8fefb4381fe0625edfb678d8d"}, + {file = "simsimd-6.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e9ad2c247ed58ba9bb170a01295cb315a45c817775cc7e51ad342f70978a1057"}, + {file = "simsimd-6.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0ff603134600da12175e66b842b7a7331c827fa070d1d8b63386a40bc8d09fcd"}, + {file = "simsimd-6.2.1-cp311-cp311-win32.whl", hash = "sha256:99dff4e04663c82284152ecc2e8bf76b2825f3f17e179abf7892e06196061056"}, + {file = "simsimd-6.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:0efc6343c440a26cf16463c4c667655af9597bcbd55ad66f33a80b2b84de7412"}, + {file = "simsimd-6.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:2d364f2c24dd38578bf0eec436c4b901c900ae1893680f46eb5632e01330d814"}, + {file = "simsimd-6.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9b3315e41bb759dc038ecd6f4fa7bcf278bf72ee7d982f752482cdc732aea271"}, + {file = "simsimd-6.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8d476c874bafa0d12d4c8c5c47faf17407f3c96140616384421c2aa980342b6f"}, + {file = "simsimd-6.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9d4f15c06cc221d29e181197c7bbf92c5e829220cbeb3cd1cf080de78b04f2a"}, + {file = "simsimd-6.2.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d286fd4538cb1a1c70e69da00a3acee301519d578931b41161f4f1379d1195c6"}, + {file = "simsimd-6.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:050f68cfa85f1fb2cfa156280928e42926e3977034b755023ce1315bf59e87ff"}, + {file = "simsimd-6.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67bb4b17e04919545f29c7b708faaccbe027f164f8b5c9f4328604fa8f5560ea"}, + {file = "simsimd-6.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3d6bffd999dbb36e606b065e0180365efac2606049c4f7818e4cba2d34c3678f"}, + {file = "simsimd-6.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:25adb244fb75dbf49af0d1bcac4ed4a3fef8e847d78449faa5595af0a3e20d61"}, + {file = "simsimd-6.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b4542cee77e801a9c27370fc36ae271514fc0fb2ce14a35f8b25f47989e3d267"}, + {file = "simsimd-6.2.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4f665228f8ff4911790b485e74b00fa9586a141dde6011970be71bb303b5a22f"}, + {file = "simsimd-6.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:783b4308f80ae00763b0eaa0dac26196958f9c2df60d35a0347ebd2f82ece46d"}, + {file = "simsimd-6.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:95055e72cfe313c1c8694783bf8a631cc15673b3b775abef367e396d931db0b8"}, + {file = "simsimd-6.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a98f2b383f51b4f4ee568a637fc7958a347fdae0bd184cff8faa8030b6454a39"}, + {file = "simsimd-6.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2e474fd10ceb38e2c9f826108a7762f8ff7912974846d86f08c4e7b19cd35ed4"}, + {file = "simsimd-6.2.1-cp312-cp312-win32.whl", hash = "sha256:b2530ea44fffeab25e5752bec6a5991f30fbc430b04647980db5b195c0971d48"}, + {file = "simsimd-6.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:dc23283235d5b8f0373b95a547e26da2d7785647a5d0fa15c282fc8c49c0dcb0"}, + {file = "simsimd-6.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:5692ce7e56253178eea9dbd58191734918409b83d54b07cfdcecf868d0150a73"}, + {file = "simsimd-6.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:76b32fdc7142c9714e94651ece8bc00dd5139c554813211552aa358e44af0e07"}, + {file = "simsimd-6.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f44e5e2319427f94db658c6f75caae78850da505902874a1664a83ef5713f333"}, + {file = "simsimd-6.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:05323cbad7200592c2e53fbcc759e615594e8ca444ef5eddf9f3fb196ad4de9c"}, + {file = "simsimd-6.2.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b1f3cbe5c39db2bb64f30999104de1215ba3805d6059af7bc5a9d662d50f4707"}, + {file = "simsimd-6.2.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaa94e0932ae2a48b7e4df8c29204dc9fe59f72b1faeb08e9d5015bf51fb9f21"}, + {file = "simsimd-6.2.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:508465f8d4e3e0fff07c939921aeedf55b0ade9f56f64e938c350c283dea42fb"}, + {file = "simsimd-6.2.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ca67f6273ef544c74c48b134af756de7c98a711ccf69cd0791225f26dd449281"}, + {file = "simsimd-6.2.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d470b43ce606f21f54a23fc19ad6928333e17d0956b02eb27b7b112edc156a10"}, + {file = "simsimd-6.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59518b9834c167a1dd8900600718e95cdadc9d74525452f426aa8455a38c55ef"}, + {file = "simsimd-6.2.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:59c2978c4e402097d8a4b38f076ff98cc43e6b059d53f89736404f26e9a9bd5a"}, + {file = "simsimd-6.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:edc68e727d53ed2866dcfb625f15e52be8f1e6809f4be2147bf8d2115a2542b7"}, + {file = "simsimd-6.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9e5e82551d75c0e2cd0d4b8af8db1cae7b5ac6dcc076c0c760870ff81f78135b"}, + {file = "simsimd-6.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2fa19f8c9786757d19afcbda9f8fb68de55e4f5562725ae8727f887d01bf0e4d"}, + {file = "simsimd-6.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b0748aa6bd4df4c5a3f5e979aec14b26588f1b2e0d44075dcc9eaf4d555e15b"}, + {file = "simsimd-6.2.1-cp313-cp313-win32.whl", hash = "sha256:7f43721e1a4ebe8d2245b0e85dd7de7153d1bf22839579d5f69a345909c68d9e"}, + {file = "simsimd-6.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:6af1565e0ef7060bc52a38e3273a8e6e92aff47835965dc5311298563475935e"}, + {file = "simsimd-6.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:e690b41377c8dd157d585713b0bc35c845aee7742334bf12d1f087fc8a65b6c3"}, + {file = "simsimd-6.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9264abf5dabe046d3951d162dbba21c7a3c3f491587c84038df1b94de0b6742a"}, + {file = "simsimd-6.2.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e93ffe6ea7417bffdee9a1b9ebb682f35f41e3e75b7e51f0f3a2fb5f7dd4c079"}, + {file = "simsimd-6.2.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d09ea4d3c0224bedf9f72881d1e5896a265fc89311abba078e615b0c06d989da"}, + {file = "simsimd-6.2.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dae5f7c37ffd0313ea59aa0a20203e7624bc5a39065fc5505991268689f2b6a2"}, + {file = "simsimd-6.2.1-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:2f573d706e44018cba63a6ff44f4a1a7733fb55ee504a12b345c012bc114f7d5"}, + {file = "simsimd-6.2.1-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:63a48c50c0ff44ac4d463f8c963f718de5aff54e1c4a6ce8363e291ac2f1fc14"}, + {file = "simsimd-6.2.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:77912f9b4c230eea2bca7ba35c33dfd5590b41a867abba9fe7e152a7ae976307"}, + {file = "simsimd-6.2.1-cp37-cp37m-musllinux_1_2_armv7l.whl", hash = "sha256:731635de9e771571fbf61edb81cfa466fed37845fbfb35d719afb7c6ea3d4bce"}, + {file = "simsimd-6.2.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:03c94c9dcf80c93c58c9435f295fd35399d88097464d1a0a5995372868d852e3"}, + {file = "simsimd-6.2.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:bbcfc905d90343c7b7e07f7b80385abc017405125246908181f6841c5f3cbde3"}, + {file = "simsimd-6.2.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:4cf0180f4b17ea3758523f644eddc38124ac98c4aac1c5572f44fd04c3bcb2f3"}, + {file = "simsimd-6.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:31163917ce2848f7896e633b8d1ae0db9004dc8eb6605cf959f6319e31cd569c"}, + {file = "simsimd-6.2.1-cp37-cp37m-win32.whl", hash = "sha256:c7af7da114f81af0bcfbf9563ea109479550e62dd5dde39ea2e93bc5f1e306ca"}, + {file = "simsimd-6.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:cad9b5503d35b7be3e704594bcdf3883bbcdb9987086d942a2a52e7b0927288e"}, + {file = "simsimd-6.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5b62fcf02e33a88e4c4a93da9d682e475bb08979d7d18f91a76bee2fe2f9d335"}, + {file = "simsimd-6.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0d7eeed41600bb229c34d822e0011c80019c16c689f16c82b875012e7116b2d5"}, + {file = "simsimd-6.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0da7f30f11cbe7c6ced372af3f5da24b7df1038bad82cfd0032667024622b794"}, + {file = "simsimd-6.2.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae496f16f2d759dc103ed8b8a5533c0a52e5c96c88e5d6a9e26eff24f174537b"}, + {file = "simsimd-6.2.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9046d108b3fc7cd1808df53083b3a2e26f70a1efb4f378971fefe76c27d64488"}, + {file = "simsimd-6.2.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1919957071b6d19e337ebba9c04f4b48604f927fc9118ce877b1fbcec1975f57"}, + {file = "simsimd-6.2.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:ef6d998496e5569ce9b5ce21a9ecbe3b59f9426ce27e6bf1db0eae67613d8d9e"}, + {file = "simsimd-6.2.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:3cb54ec20235d81dd9596c8fe8b2bd35fad027d3f5cd52e23a17a71b3ac44d3f"}, + {file = "simsimd-6.2.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:598330828b922700aac8a7939c562f80e4ee9000671081ff264c8daae4692d76"}, + {file = "simsimd-6.2.1-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:1b45987216a5d5b9b1441ea8acbf5d731e5ee60c0727999f10438827d201b40d"}, + {file = "simsimd-6.2.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:8c9b79c189ab40e1633c4cecba1a58133a8454662c40af8abdf18f0766a1cf94"}, + {file = "simsimd-6.2.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:1324d7433f0cefd29a55716197112d22b259c49d7c62425517dc37d0c6494b69"}, + {file = "simsimd-6.2.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:c5101d1204e42b15c1e3772ec8b357cec9bce5eea0ccb76ec8faff5104233241"}, + {file = "simsimd-6.2.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:d8c7b7b286d7be1756fb837b9f3330f7d03eb6a7329cd717c88d635e441a8eb0"}, + {file = "simsimd-6.2.1-cp38-cp38-win32.whl", hash = "sha256:2e07e5b4abbb5561a62acfc4d1f2c4fb9051cc0f6919b0456d0bb37dc6749f0a"}, + {file = "simsimd-6.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:87b963f862ba50a61527af281a66e1d6cee34c535b621718e45de1df8f277cba"}, + {file = "simsimd-6.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:45010111c39117af851a323e78bd43e6a344349b4ed7b1f5ca4c4ebb2284c7e5"}, + {file = "simsimd-6.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dd6ecae57a481f9fc0bceb331cba7b18a0b23a71f15af7d06cdf8aa8aac38645"}, + {file = "simsimd-6.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ffbb874d4c3ed53443468f9c20704845cc8736d5717817c35d5cb12ad5548c7a"}, + {file = "simsimd-6.2.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b6147ddc390c08a802af258ad204b1d775bb3d180ec6f6fcea82f4fd71fb447"}, + {file = "simsimd-6.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0048df2245d239ed016e5f4b5d75e96987149bf7245e90713e1fe3b53e321a74"}, + {file = "simsimd-6.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc087d9dacab1eb4abc2f3d9f33047fc601db501cb43165e658973fe5fd50c9b"}, + {file = "simsimd-6.2.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:d1d2e6c3d655a34b42c6e0d0c28ac7b86498858ffb68c58733893fc538bd26a9"}, + {file = "simsimd-6.2.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:d063beb7a53d8525af56c4247e1e43a7fa161b70bcbacf30daab639b32ad4a10"}, + {file = "simsimd-6.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4a517ae74d18a8b7d4d349cf4afed45f33cd660cb44d0ae34c95d00c1f7fa760"}, + {file = "simsimd-6.2.1-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:a79a2bd32ba0f90f70c22accf4b441846049b55aeae73556f4b5c6e9fe6e024f"}, + {file = "simsimd-6.2.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4c9487acdae92b4089a0142cd3691328bfdcaaebf2587a0c11df4039ff7005e8"}, + {file = "simsimd-6.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:1c4760dee8f65a890b82a6175d5294d30271637495a9e4195969fc1ad38ec056"}, + {file = "simsimd-6.2.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:abee753fbb8584373218bf78396ae3d2b2a1202c7284cd9c70695535c62cdc31"}, + {file = "simsimd-6.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:300042eeab379923d77bca328fdc2ac1df8adfdffa9a6939f28ba6b196f02002"}, + {file = "simsimd-6.2.1-cp39-cp39-win32.whl", hash = "sha256:2eed0ad770b18a3b74b19ad744ee3224dae9bf1a86bd9126eae0636ada53eebd"}, + {file = "simsimd-6.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:e99cc8aa19af5ca3574aa72e1d0e959c4859345fdf553a887ce22e469c1145a8"}, + {file = "simsimd-6.2.1-cp39-cp39-win_arm64.whl", hash = "sha256:37b0db92ca0fec835ec1256d6dd167d7c9f727d3372b98bf27b1fd59ad299768"}, + {file = "simsimd-6.2.1.tar.gz", hash = "sha256:5e202c5386a4141946b7aee05faac8ebc2e36bca0a360b24080e57b59bc4ef6a"}, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "soupsieve" +version = "2.6" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, +] + +[[package]] +name = "stringzilla" +version = "3.12.3" +description = "SIMD-accelerated string search, sort, hashes, fingerprints, & edit distances" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "stringzilla-3.12.3-cp310-cp310-macosx_10_11_universal2.whl", hash = "sha256:8ec8f95af09d62b4ca5dc8c6f557035acadfe794edd2291f515b54f3afb59260"}, + {file = "stringzilla-3.12.3-cp310-cp310-macosx_10_11_x86_64.whl", hash = "sha256:d305ed6f35132852844f59964910b202675f76ad48060fea8c6c959b67959c3d"}, + {file = "stringzilla-3.12.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8d1eb356ca15400b1410187bc7fde421f6e20460f05ae1dece4c60821bfffba6"}, + {file = "stringzilla-3.12.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d0f7c5bccff1125fb77c01d7166e525c8dbf40f700f4120a00ad2f4ccfb3f3d3"}, + {file = "stringzilla-3.12.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:db02c9baa70eceb6f1362237411ef4b6b59064e17008da339f378717050c2bdb"}, + {file = "stringzilla-3.12.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da5938ba65d0f7bff83479662b7f397d71cac3e84fc37b4504616712e37d0417"}, + {file = "stringzilla-3.12.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3ce27645cbe2593ede23c581a50428f1d0ba1d666af06d445ba89cef6613df0"}, + {file = "stringzilla-3.12.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:680db2ce49bee340904a8c3149b0bca527cd9dc322a97071e4500f646d87404e"}, + {file = "stringzilla-3.12.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32537a5165f473cd99440baa23e9202ab2185d72896960daa4f8bddac5c80544"}, + {file = "stringzilla-3.12.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c39d612f26ea7385b6a134292b097efe7444cae5b2dafbfccd3b65ce23d73ccc"}, + {file = "stringzilla-3.12.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:dea7c8ca195f8c79e85163fd5b76495b4a11effd2e17956e6825b4fffa3c915b"}, + {file = "stringzilla-3.12.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0d21de495a84b05196832e5aec0d5ee83b0667f9c165d0e326a753022625adfa"}, + {file = "stringzilla-3.12.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:7c836c28a75cd4cccb0737854ed780444c45cce93012ed100dd4a431b60ebd86"}, + {file = "stringzilla-3.12.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:745f144c039c2c2787ef0d63296f06d4d0d776b141288a74670aea38d86b3078"}, + {file = "stringzilla-3.12.3-cp310-cp310-win32.whl", hash = "sha256:2d4803357a07592a7fdbea599e93cfd4366c4e887290cfa5988bc7ec44da93b5"}, + {file = "stringzilla-3.12.3-cp310-cp310-win_amd64.whl", hash = "sha256:0f41b85c38445f7a1fed677984590c943b16cbc00727e2b093b2f0b2bdbfcac5"}, + {file = "stringzilla-3.12.3-cp310-cp310-win_arm64.whl", hash = "sha256:088ca8105ff027172277d2221ea0241d5ed21cc10ee91d5f45c7961ddab3d12a"}, + {file = "stringzilla-3.12.3-cp311-cp311-macosx_10_11_universal2.whl", hash = "sha256:d0e79931ae66cd4566f25d77ccf646e9d180ead603fab4278a6ecdae7570e85b"}, + {file = "stringzilla-3.12.3-cp311-cp311-macosx_10_11_x86_64.whl", hash = "sha256:b3a2f047dfe21468f90e8cab3f6a4b8e46e876b6563b78dc54ba154a56f1e383"}, + {file = "stringzilla-3.12.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b8404b55fa180d1e6da1fc10f739890af427d93afd02a408e229be8d7383a5e"}, + {file = "stringzilla-3.12.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdbbb9f0dd3f8d1ec3201a3fa7de3e0c92d056da9ca61ada2af8ca662cab4834"}, + {file = "stringzilla-3.12.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:162e68d44e87f3b7591a9c18f8a7794bc9bbf8ab9b16705bfe5c552c676b2d8c"}, + {file = "stringzilla-3.12.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee7284d1c62cc4d4cf7772178c02cae91933a38e8b11390da6e8a8b4f20e0663"}, + {file = "stringzilla-3.12.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4d08a9fda6b9489667dfba38dcc8ebe7a94e4ebbe8e741557cccd5b09f864ed7"}, + {file = "stringzilla-3.12.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c11a94e9123e411e8d74cd0ea860dca58bec6e48a95b5ff3707b595eaf71eecd"}, + {file = "stringzilla-3.12.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:639844f35f0be5ade5849686067076add2214ce93a01ffd2ab64b2a6c506f666"}, + {file = "stringzilla-3.12.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:98579b756f10aa515b97b58d69a6aff3daefb7267cdf84e225e5b7fda584a431"}, + {file = "stringzilla-3.12.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dd95a2f8782183340fd706f75aa8615c21a2eacc0c11684fd6b3ee17b1ba3542"}, + {file = "stringzilla-3.12.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:06d543e4949e43c43a8b4334fda8c162999249daeb787af5ea6b0e8d0682ce79"}, + {file = "stringzilla-3.12.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:783ff26460fc8028cf53ec08ddacf438d0efffed78f75f34131cdfc77d15c2cf"}, + {file = "stringzilla-3.12.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:39cfca4f26c70f262016ed16a035f42d1ed0917e28f78db6c61670690983b96b"}, + {file = "stringzilla-3.12.3-cp311-cp311-win32.whl", hash = "sha256:bb593a0e809451254a819e156a8180cb53a239f1f427a8bdb2a39f7c52e85a43"}, + {file = "stringzilla-3.12.3-cp311-cp311-win_amd64.whl", hash = "sha256:587f1934ef615b5e11ce1b1779cf391d40e0ead6f6be6083d313dc6b4cc7f4dd"}, + {file = "stringzilla-3.12.3-cp311-cp311-win_arm64.whl", hash = "sha256:329d734c4eb943d9746d8bb2fc2008063b8b33b8f9af27833abea876b6027aeb"}, + {file = "stringzilla-3.12.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28c644def937fd0baa887c1e4e758778d59773e996ac830013b3468671d96aa"}, + {file = "stringzilla-3.12.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:773da6ad2993c3bbbdbff87b583eb1cc4b655c33809bfd3f913f749be12bfdd0"}, + {file = "stringzilla-3.12.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e43e92b66d12c6d6487bccbb0b35e6dca670932a92ebb5e59b62e7212aaf739f"}, + {file = "stringzilla-3.12.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27d29eb61ced7a2dcf44195eea058e5aa44d4c8b73c2095a435ca9533418f6d7"}, + {file = "stringzilla-3.12.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07d51f4c20cbc7d68f5291ef43c2c6b6b772f4be27adb6c9a6f895ad06004fd8"}, + {file = "stringzilla-3.12.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e914d237b4d1829974cbda70e70b058373fef6a5fef90e36953d531f5bdc1063"}, + {file = "stringzilla-3.12.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32999c1c21eb799a97d0659b54cbdd01bfdd1b7ffc53a8c16183b5542e5052c9"}, + {file = "stringzilla-3.12.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6e18d9604b7a083c3da5df8e1029c692e9b69e4f17e08b773cfbf8c6a141106"}, + {file = "stringzilla-3.12.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd8f488f281d8e198c28c77ef8638e062622d1c0ce18e9bee5380c0d848b248d"}, + {file = "stringzilla-3.12.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4dd8cb6730f230a76ea623fc111372e06f07fdea3b27471ba1a8cf23e6751eda"}, + {file = "stringzilla-3.12.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5469ae13ffeb03fb30a99241af6f5584ee193c549d2635616ce7558415d13f22"}, + {file = "stringzilla-3.12.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:95f245d8e526fc25691329996a67b661acf0ea7baef9c3a402d555e32aa49863"}, + {file = "stringzilla-3.12.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2b1884630696cf51ac2ea53867b629aa80e01dead3a47c021b7d643eb0595f68"}, + {file = "stringzilla-3.12.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5e492370b5de18ce68c96213544124221dc52f833477e177a4052ec704506e58"}, + {file = "stringzilla-3.12.3-cp312-cp312-win32.whl", hash = "sha256:5f20f8679ce88dad1eaae8effff8262694c65b359618e8ed6476a389eaf172a7"}, + {file = "stringzilla-3.12.3-cp312-cp312-win_amd64.whl", hash = "sha256:291c024cc4365d6c0099d9ee7e61142392ab1315a6d4a8097e3b63af71d0d97c"}, + {file = "stringzilla-3.12.3-cp312-cp312-win_arm64.whl", hash = "sha256:3cf28d68273ba12ee970c682e67410516cdde087d207a2bb0cdd44ab2f533421"}, + {file = "stringzilla-3.12.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c29727cf895aef132ee7d1196bc04cc7900bbacf9ce08980c2557399cbb83222"}, + {file = "stringzilla-3.12.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:32631978e9fa79e9a579923a7c97a4603143c5fda5b63f9103507762604bd098"}, + {file = "stringzilla-3.12.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3df1a74419732a18a7885474c858bcb1270bdf5c35888d97d3b16c3646d539c5"}, + {file = "stringzilla-3.12.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9366d7a903f617472107767716502d4897a272512ead3a5e3c5e32d217f3a2e8"}, + {file = "stringzilla-3.12.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fcebbea3ea4fe58b2bb0dc85a6be705e7d2cc4746113a79940d8bc2755df87d"}, + {file = "stringzilla-3.12.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2a1219e4c79842893d6ace2e2875c879affdb14107877d3037e3373bedc8a56"}, + {file = "stringzilla-3.12.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b14251c50ec73eb75ce83aa07673f68c4f187b53b8b868ebc0311f3012ee71b"}, + {file = "stringzilla-3.12.3-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9cba1e18d1d167edf67fdfc5fa11ece06ec376ae55dd65401125d546e3d2150b"}, + {file = "stringzilla-3.12.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:43467714a52103f819ffbdef2d58aa39c938fcd42023ff6100a77bbb3f6cb398"}, + {file = "stringzilla-3.12.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3d5c7fe1f2d04d595dd3fedf7776f8ab9819e4f4c681ae1f0fb3e28bb29247b3"}, + {file = "stringzilla-3.12.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bbd85919eaf7eb01ca25304dfe085b21d6db693499cd25da0a915d348ec42c38"}, + {file = "stringzilla-3.12.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:57867d712049b8222311e32fd93ebfd60864b48b35aefa860788f6eafba61bc2"}, + {file = "stringzilla-3.12.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1db2e8092c72ca7750d7b5a0d367a80efb8d3831a7ea60deeb5885f301aac035"}, + {file = "stringzilla-3.12.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793b8acfa4f1dae7d070742b69f423d8d073643eaa56eb89078e781d031caada"}, + {file = "stringzilla-3.12.3-cp313-cp313-win32.whl", hash = "sha256:7af63431268f018af41b15adeab7a732585f397a0941adaf5d2fc624f1f3a790"}, + {file = "stringzilla-3.12.3-cp313-cp313-win_amd64.whl", hash = "sha256:0a3dcd565d833e7c5814eeba2ebfcbf9d06a4ae32467423d4071702c1084e74a"}, + {file = "stringzilla-3.12.3-cp313-cp313-win_arm64.whl", hash = "sha256:bcce4ed759cee812d5fcec3b87eafa9996641483cbc1b0f81006ca15bf6a16b6"}, + {file = "stringzilla-3.12.3-cp36-cp36m-macosx_10_11_x86_64.whl", hash = "sha256:f94fb88b8064e228345ca5ab9d817b0bc1e5cd17dfbad3f0166334dee704b0c6"}, + {file = "stringzilla-3.12.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a4063243b73170f4114d61b97da4c8ab1a94a0cd7d54b82593df6294a995ad3"}, + {file = "stringzilla-3.12.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:11587f5f298f6ae67f66e8257973e1a3aa6003194e18d3839f330bf95613ee9b"}, + {file = "stringzilla-3.12.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:81d900b1d3b2cd8a1c8a9db1643d7ba516213831f62b7142c4fd689b692bb293"}, + {file = "stringzilla-3.12.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56bb42ebf203d2c87e6a385cd31d097e593f9c2138eb4b0d9c847dcd1cc5f484"}, + {file = "stringzilla-3.12.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df20a8fc23133eb8578f85fb555c5d1e57d0c14c580a6a487a401f4ed1d6d957"}, + {file = "stringzilla-3.12.3-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:660f7b65a95d4265a42b15aee948e1b2977295c70c6cd298a1059a19c51cf692"}, + {file = "stringzilla-3.12.3-cp36-cp36m-musllinux_1_2_armv7l.whl", hash = "sha256:e38cfa8eb3a4a69c7a017957f376953e8d0150ef416eae46931d9a795358a856"}, + {file = "stringzilla-3.12.3-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:aac0ed4aa7ab8c00f8ffe0bfc8ecbf5ca1da8a57a7b72aa009deeab18b6c6b1e"}, + {file = "stringzilla-3.12.3-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:53ff36ecde47a3fd76f1cf4983dcc449c7a22fcaa090678b1b8fea43b97edefd"}, + {file = "stringzilla-3.12.3-cp36-cp36m-musllinux_1_2_s390x.whl", hash = "sha256:cc5548891aa00866bcd61d3d7c0db38fef3f36a097112e030ffff0b7cf711755"}, + {file = "stringzilla-3.12.3-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:f9ec5615c807097d375eb84f13bccd9c0f8ed55a3ea162802336653d60c914a1"}, + {file = "stringzilla-3.12.3-cp36-cp36m-win32.whl", hash = "sha256:e7f2f8da99e3150717fb886fc428f21b997865de58b00ee45b9caa67cc3418a1"}, + {file = "stringzilla-3.12.3-cp36-cp36m-win_amd64.whl", hash = "sha256:58cb7e88770f1ca1b83398fe6fba8ed6e5bb56f225184dd0bda138cf398d3ef9"}, + {file = "stringzilla-3.12.3-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:7eeb34d049d49b5f05b14d2878e6d660d3b81651749f63fd898d6f799a9f5f55"}, + {file = "stringzilla-3.12.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb74a67a1af51b4553e93daf681ec4b05b22bce0e582345e6305c31e4a790a30"}, + {file = "stringzilla-3.12.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fec66a18175edf8ce07ed9ca9e94287b0f6e5f4d8adac570f6c5f1df99f4ef21"}, + {file = "stringzilla-3.12.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:878cb7068fb21ccdc8529a34fb40a67a7dfce306245fe1d90c754acdadb3c9af"}, + {file = "stringzilla-3.12.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e65714dce2ee73e77e15dd705a9e6b8b61f539d2fbe2833800883083fd404cad"}, + {file = "stringzilla-3.12.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c49a55da8a1c7a9a1e15955bdb330ce2023bf04fa0a922ac776236ec655a8ad"}, + {file = "stringzilla-3.12.3-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:0aaa6fb26f55b965598d8115187b36a292d994c63292a0f5bdc0ae6b3fdd8086"}, + {file = "stringzilla-3.12.3-cp37-cp37m-musllinux_1_2_armv7l.whl", hash = "sha256:e25d99cdc20d3ba239250bbb195667bf9254ebb9b15bf2a59b81ba36525ee112"}, + {file = "stringzilla-3.12.3-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:2855d5ad326cf4eb05c3a8d090b82d97ed22023210774c50c3beed9f5e5d1f16"}, + {file = "stringzilla-3.12.3-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:05649732a1f1c0ed76876259d6adbceb8691c36226ae85a63d1c456277c3c463"}, + {file = "stringzilla-3.12.3-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:76772c2c42020fa34e0d761edf93ec07fd36df5c38ce4c406fc5dac1749fbb0d"}, + {file = "stringzilla-3.12.3-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:87abcfe64a5088cbe5d78cf83124248686062c651b3b20d85723918b805c9652"}, + {file = "stringzilla-3.12.3-cp37-cp37m-win32.whl", hash = "sha256:1c4564425e44b11b5c5a1d569e786855c68d25ae7b9abfc48060516cb756167d"}, + {file = "stringzilla-3.12.3-cp37-cp37m-win_amd64.whl", hash = "sha256:f2fd71fa4c554653e476ea97d09f4d1f15e8f497f829db46139cf0aa698abd4f"}, + {file = "stringzilla-3.12.3-cp38-cp38-macosx_10_11_universal2.whl", hash = "sha256:1dc00bb8f18f0dc18da671af985ce05e0c89de357e54361b79c535a7567cbac1"}, + {file = "stringzilla-3.12.3-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:e06982803400023a105faff4e9ad78326fc55e18200a7dea60e2814c1fbc588a"}, + {file = "stringzilla-3.12.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fff766b0bfe8b93602e3066483ba16bd5be6b3bcdd369130bb242e50d29ef3b6"}, + {file = "stringzilla-3.12.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:33a880aee9b227fbcecfe1104fdf814d69727d4de1002b0d1697e4dc13a0b10d"}, + {file = "stringzilla-3.12.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b951825f83f8464fc49684c1b54e2ba12e07a38b8d8fc04d630bcae7b77284a"}, + {file = "stringzilla-3.12.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1bcc12a7e05919383663fc6c4c6352447c9e0a2fb91865168c8352e5202722d6"}, + {file = "stringzilla-3.12.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:484931036dc85ab0c768e1f14e0d22397c3c7619b675c07b37825f5524d5d31f"}, + {file = "stringzilla-3.12.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c08d700cf7769beb2e6810629d60ea2036a2ed7cd66eb500af3f1d551f9f023"}, + {file = "stringzilla-3.12.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:cdd478f4ce671ab57a9ed7fb7c3673c91004fe65ad7220777683d4c3978936d0"}, + {file = "stringzilla-3.12.3-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:1915ac69956ce1fc7c793944259fc223e6bae6d0ad5693f53db42317d7728a9b"}, + {file = "stringzilla-3.12.3-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:436279413939e9ebed09bbfba7f5c6c4aadd05e0e639d2992b70a474524bc95b"}, + {file = "stringzilla-3.12.3-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cb4abcf117d126028e8a917fd27390b8e259823d67ce85f85c73a3aac561881a"}, + {file = "stringzilla-3.12.3-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:d9e74323816e1e7984f13ce2453b65ee8a70a39e08b22ae9b22741cad3f894d1"}, + {file = "stringzilla-3.12.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3fd0836e0298e9a3b219a76c8179643b29b84090b41d6123c03277672035c852"}, + {file = "stringzilla-3.12.3-cp38-cp38-win32.whl", hash = "sha256:caf6165ec9a1cb4baf24146ca2d123c31fedf0b51b1d3c4f913257f956f1551e"}, + {file = "stringzilla-3.12.3-cp38-cp38-win_amd64.whl", hash = "sha256:810ff7f3a187f33f9027df6453045b65aecdb2f94a51ee7461ff918ba174cfb6"}, + {file = "stringzilla-3.12.3-cp39-cp39-macosx_10_11_universal2.whl", hash = "sha256:1207e68c89e0002c75f3c8bdcc19365262a31088c5f0341b54903c98f4428971"}, + {file = "stringzilla-3.12.3-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:f52273bdcb4086a3f0391f9ea292e3f463814afee4643a2e27f2ff991088b580"}, + {file = "stringzilla-3.12.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ce7776eb035f1e1450694abfb9c4e60e3c6f31358966ccd440f2bf7a58f5b2d4"}, + {file = "stringzilla-3.12.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0c409967e23673dd500ca91d3889eba3944b75cb3ae70fc22b22e7298ab4c76"}, + {file = "stringzilla-3.12.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3f29c5aaaeaa1b6fd357726bdd1eb66ac6879f7be9136c585527e61d35e8250f"}, + {file = "stringzilla-3.12.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a73108cbe96f08f10de2c25e943e05c606b315ae8f02efb6600a2058485d76c2"}, + {file = "stringzilla-3.12.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9bbb9efaf50a08ea3cfeb910c768b530f5d4d99b9a5c89de0535f8c0d8c0247"}, + {file = "stringzilla-3.12.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f5fa3b63eef32ae403b85a53783d892a65d9303c99eaf0659c3792e430ecd01"}, + {file = "stringzilla-3.12.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8270d07a2e68a46141af1cb959ac8c3ac823d0d6f8107c287211add01e8223fd"}, + {file = "stringzilla-3.12.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:f8673b11b8bcea17a574b038b1349672ee1067462f598c031f7e59bb35339fd8"}, + {file = "stringzilla-3.12.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f1a20cf3760176652affe5f66f1f7f6c106e7bdbfc6d27e4ffb5f0ebfc03a656"}, + {file = "stringzilla-3.12.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:23212e9da476d551a792f6e9a7ee7dd739d7b6a568685c10752275ca5f35fd9e"}, + {file = "stringzilla-3.12.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ea06ee9a2b5a56f817e1742e8d502a60315aaeaad26631723c6a3405959be61e"}, + {file = "stringzilla-3.12.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4afee16ca8f89ac249b2d3581a24b9df63efb73e9d71d136411fbf088e036103"}, + {file = "stringzilla-3.12.3-cp39-cp39-win32.whl", hash = "sha256:28f05183f0a0c9461ba9f66155aa6fd21f26a8b4daeb07b7a8833e9b17ffb264"}, + {file = "stringzilla-3.12.3-cp39-cp39-win_amd64.whl", hash = "sha256:b6535c2768f4e365997b6483a6f38cd24ff1444a3e1da85fa411ba21a9c814dd"}, + {file = "stringzilla-3.12.3-cp39-cp39-win_arm64.whl", hash = "sha256:11e6e6ccede0eb59022d0634540adf29afa1a00c6612b0daa1c207b440ee8012"}, + {file = "stringzilla-3.12.3.tar.gz", hash = "sha256:33ed7cb71724373474d387a0e17751bd9ad21caa08a1b8b74b961dea4b890a66"}, +] + +[[package]] +name = "termcolor" +version = "3.0.1" +description = "ANSI color formatting for output in terminal" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "termcolor-3.0.1-py3-none-any.whl", hash = "sha256:da1ed4ec8a5dc5b2e17476d859febdb3cccb612be1c36e64511a6f2485c10c69"}, + {file = "termcolor-3.0.1.tar.gz", hash = "sha256:a6abd5c6e1284cea2934443ba806e70e5ec8fd2449021be55c280f8a3731b611"}, +] + +[package.extras] +tests = ["pytest", "pytest-cov"] + +[[package]] +name = "tifffile" +version = "2025.3.30" +description = "Read and write TIFF files" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "tifffile-2025.3.30-py3-none-any.whl", hash = "sha256:0ed6eee7b66771db2d1bfc42262a51b01887505d35539daef118f4ff8c0f629c"}, + {file = "tifffile-2025.3.30.tar.gz", hash = "sha256:3cdee47fe06cd75367c16bc3ff34523713156dae6cd498e3a392e5b39a51b789"}, +] + +[package.dependencies] +numpy = "*" + +[package.extras] +all = ["defusedxml", "fsspec", "imagecodecs (>=2024.12.30)", "lxml", "matplotlib", "zarr (<3)"] +codecs = ["imagecodecs (>=2024.12.30)"] +plot = ["matplotlib"] +test = ["cmapfile", "czifile", "dask", "defusedxml", "fsspec", "imagecodecs", "lfdfiles", "lxml", "ndtiff", "oiffile", "psdtags", "pytest", "roifile", "xarray", "zarr (<3)"] +xml = ["defusedxml", "lxml"] +zarr = ["fsspec", "zarr (<3)"] + +[[package]] +name = "tqdm" +version = "4.67.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] +discord = ["requests"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "typing-extensions" +version = "4.13.1" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "typing_extensions-4.13.1-py3-none-any.whl", hash = "sha256:4b6cf02909eb5495cfbc3f6e8fd49217e6cc7944e145cdda8caa3734777f9e69"}, + {file = "typing_extensions-4.13.1.tar.gz", hash = "sha256:98795af00fb9640edec5b8e31fc647597b4691f099ad75f469a2616be1a76dff"}, +] + +[[package]] +name = "typing-inspection" +version = "0.4.0" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f"}, + {file = "typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + +[[package]] +name = "urllib3" +version = "2.3.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, + {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.12,<4.0" +content-hash = "e9e804116cb06501db7bc04112ecc9abadb632372dc424232bbfbf9d4e04e109" diff --git a/services/docreader/pyproject.toml b/services/docreader/pyproject.toml new file mode 100644 index 0000000..7f017ec --- /dev/null +++ b/services/docreader/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "docreader" +version = "0.1.0" +description = "" +readme = "README.md" +requires-python = ">=3.12,<4.0" +dependencies = [ + "paddlepaddle (>=3.0.0,<4.0.0)", + "paddleocr (>=2.10.0,<3.0.0)", + "playwright (>=1.51.0,<2.0.0)", + "setuptools (>=79.0.0,<80.0.0)", + "textract (>=1.6.5,<2.0.0)", + "antiword (>=0.3.2,<0.4.0)" +] + + +[build-system] +requires = ["poetry-core>=2.0.0,<3.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/services/docreader/requirements.txt b/services/docreader/requirements.txt new file mode 100644 index 0000000..afef307 --- /dev/null +++ b/services/docreader/requirements.txt @@ -0,0 +1,32 @@ +grpcio +grpcio-tools +grpcio-health-checking +protobuf +python-docx +PyPDF2 +requests +Pillow +beautifulsoup4 +lxml +playwright +asyncio +urllib3 +markdownify +mistletoe +goose3[all] +paddleocr>=2.10.0,<3.0.0 +markdown +pypdf +cos-python-sdk-v5 +minio +textract +antiword +openai +ollama +pdfplumber + +--extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cpu/ +paddlepaddle>=3.0.0,<4.0.0 + +# --extra-index-url https://www.paddlepaddle.org.cn/packages/stable/cu126/ +# paddlepaddle-gpu==3.0.0 \ No newline at end of file diff --git a/services/docreader/scripts/generate_proto.sh b/services/docreader/scripts/generate_proto.sh new file mode 100755 index 0000000..86249db --- /dev/null +++ b/services/docreader/scripts/generate_proto.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -x + +# 设置目录 +PROTO_DIR="src/proto" +PYTHON_OUT="src/proto" +GO_OUT="src/proto" + +# 生成Python代码 +python3 -m grpc_tools.protoc -I${PROTO_DIR} \ + --python_out=${PYTHON_OUT} \ + --grpc_python_out=${PYTHON_OUT} \ + ${PROTO_DIR}/docreader.proto + +# 生成Go代码 +protoc -I${PROTO_DIR} --go_out=${GO_OUT} \ + --go_opt=paths=source_relative \ + --go-grpc_out=${GO_OUT} \ + --go-grpc_opt=paths=source_relative \ + ${PROTO_DIR}/docreader.proto + +# 修复Python导入问题（MacOS兼容版本） +if [ "$(uname)" == "Darwin" ]; then + # MacOS版本 + sed -i '' 's/import docreader_pb2/from . import docreader_pb2/g' ${PYTHON_OUT}/docreader_pb2_grpc.py +else + # Linux版本 + sed -i 's/import docreader_pb2/from . import docreader_pb2/g' ${PYTHON_OUT}/docreader_pb2_grpc.py +fi + +echo "Proto files generated successfully!" \ No newline at end of file diff --git a/services/docreader/src/__init__.py b/services/docreader/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/docreader/src/client/client.go b/services/docreader/src/client/client.go new file mode 100644 index 0000000..ee74246 --- /dev/null +++ b/services/docreader/src/client/client.go @@ -0,0 +1,115 @@ +package client + +import ( + "fmt" + "log" + "os" + "time" + + "github.com/Tencent/WeKnora/services/docreader/src/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/resolver" +) + +const ( + maxMessageSize = 50 * 1024 * 1024 // 50MB +) + +var ( + // Logger is the default logger used by the client + Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds) +) + +// ImageInfo 表示一个图片的信息 +type ImageInfo struct { + URL string // 图片URL（COS） + Caption string // 图片描述 + OCRText string // OCR提取的文本 + OriginalURL string // 原始图片URL + Start int // 图片在文本中的开始位置 + End int // 图片在文本中的结束位置 +} + +// Client represents a DocReader service client +type Client struct { + conn *grpc.ClientConn + proto.DocReaderClient + debug bool +} + +// NewClient creates a new DocReader client with the specified address +func NewClient(addr string) (*Client, error) { + Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr) + + // 设置消息大小限制 + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(maxMessageSize), + grpc.MaxCallSendMsgSize(maxMessageSize), + ), + } + resolver.SetDefaultScheme("dns") + + startTime := time.Now() + conn, err := grpc.Dial("dns:///"+addr, opts...) + if err != nil { + Logger.Printf("ERROR: Failed to connect to DocReader service: %v", err) + return nil, err + } + Logger.Printf("INFO: Successfully connected to DocReader service in %v", time.Since(startTime)) + + return &Client{ + conn: conn, + DocReaderClient: proto.NewDocReaderClient(conn), + debug: false, + }, nil +} + +// Close closes the client connection +func (c *Client) Close() error { + Logger.Printf("INFO: Closing DocReader client connection") + return c.conn.Close() +} + +// SetDebug enables or disables debug logging +func (c *Client) SetDebug(debug bool) { + c.debug = debug + Logger.Printf("INFO: Debug logging set to %v", debug) +} + +// Log logs a message with the appropriate level +func (c *Client) Log(level string, format string, args ...interface{}) { + if level == "DEBUG" && !c.debug { + return + } + Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...)) +} + +// GetImagesFromChunk 从一个Chunk中提取所有图片信息 +func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo { + if chunk == nil || len(chunk.Images) == 0 { + return nil + } + + images := make([]ImageInfo, 0, len(chunk.Images)) + for _, img := range chunk.Images { + images = append(images, ImageInfo{ + URL: img.Url, + Caption: img.Caption, + OCRText: img.OcrText, + OriginalURL: img.OriginalUrl, + Start: int(img.Start), + End: int(img.End), + }) + } + + return images +} + +// HasImagesInChunk 判断一个Chunk是否包含图片 +func HasImagesInChunk(chunk *proto.Chunk) bool { + return chunk != nil && len(chunk.Images) > 0 +} diff --git a/services/docreader/src/client/client_test.go b/services/docreader/src/client/client_test.go new file mode 100644 index 0000000..5551048 --- /dev/null +++ b/services/docreader/src/client/client_test.go @@ -0,0 +1,154 @@ +package client + +import ( + "context" + "log" + "os" + "testing" + "time" + + "github.com/Tencent/WeKnora/services/docreader/src/proto" +) + +func init() { + // 配置测试日志 + log.SetOutput(os.Stdout) + log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile) + log.Println("INFO: Initializing DocReader client tests") +} + +func TestReadFromURL(t *testing.T) { + log.Println("INFO: Starting TestReadFromURL") + + // 创建测试客户端 + log.Println("INFO: Creating test client") + client, err := NewClient("localhost:50051") + if err != nil { + log.Printf("ERROR: Failed to create client: %v", err) + t.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + // 启用调试日志 + client.SetDebug(true) + + // 测试 ReadFromURL 方法 + log.Println("INFO: Sending ReadFromURL request to server") + startTime := time.Now() + resp, err := client.ReadFromURL( + context.Background(), + &proto.ReadFromURLRequest{ + Url: "https://example.com", + Title: "test", + ReadConfig: &proto.ReadConfig{ + ChunkSize: 512, + ChunkOverlap: 50, + Separators: []string{"\n\n", "\n", "。"}, + EnableMultimodal: true, + }, + }, + ) + + requestDuration := time.Since(startTime) + if err != nil { + log.Printf("ERROR: ReadFromURL failed: %v", err) + t.Fatalf("ReadFromURL failed: %v", err) + } + log.Printf("INFO: ReadFromURL completed in %v", requestDuration) + + // 验证结果 + chunkCount := len(resp.Chunks) + log.Printf("INFO: Received %d chunks from URL parsing", chunkCount) + if chunkCount == 0 { + log.Println("WARN: Expected non-empty content but received none") + t.Error("Expected non-empty content") + } + + // 打印结果 + for i, chunk := range resp.Chunks { + if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块 + log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50)) + } else if i == 2 && chunkCount > 4 { + log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4) + } + } + + log.Println("INFO: TestReadFromURL completed successfully") +} + +func TestReadFromFileWithChunking(t *testing.T) { + log.Println("INFO: Starting TestReadFromFileWithChunking") + + // 创建测试客户端 + log.Println("INFO: Creating test client") + client, err := NewClient("localhost:50051") + if err != nil { + log.Printf("ERROR: Failed to create client: %v", err) + t.Fatalf("Failed to create client: %v", err) + } + defer client.Close() + + // 启用调试日志 + client.SetDebug(true) + + // 读取测试文件 + log.Println("INFO: Reading test file") + fileContent, err := os.ReadFile("../testdata/test.md") + if err != nil { + log.Printf("ERROR: Failed to read test file: %v", err) + t.Fatalf("Failed to read test file: %v", err) + } + log.Printf("INFO: Read test file, size: %d bytes", len(fileContent)) + + // 测试 ReadFromFile 方法，带分块参数 + log.Println("INFO: Sending ReadFromFile request to server") + startTime := time.Now() + resp, err := client.ReadFromFile( + context.Background(), + &proto.ReadFromFileRequest{ + FileContent: fileContent, + FileName: "test.md", + FileType: "md", + ReadConfig: &proto.ReadConfig{ + ChunkSize: 200, + ChunkOverlap: 50, + Separators: []string{"\n\n", "\n", "。"}, + EnableMultimodal: true, + }, + }, + ) + + requestDuration := time.Since(startTime) + if err != nil { + log.Printf("ERROR: ReadFromFile failed: %v", err) + t.Fatalf("ReadFromFile failed: %v", err) + } + log.Printf("INFO: ReadFromFile completed in %v", requestDuration) + + // 验证结果 + chunkCount := len(resp.Chunks) + log.Printf("INFO: Received %d chunks from file parsing", chunkCount) + if chunkCount == 0 { + log.Println("WARN: Expected non-empty content but received none") + t.Error("Expected non-empty content") + } + + // 打印结果 + for i, chunk := range resp.Chunks { + if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块 + log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50)) + } else if i == 2 && chunkCount > 4 { + log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4) + } + } + + log.Println("INFO: TestReadFromFileWithChunking completed successfully") +} + +// 截断字符串以供日志打印 +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "..." +} diff --git a/services/docreader/src/download_deps.py b/services/docreader/src/download_deps.py new file mode 100644 index 0000000..d24fbfa --- /dev/null +++ b/services/docreader/src/download_deps.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os +import logging +from paddleocr import PaddleOCR + +# 添加当前目录到Python路径 +current_dir = os.path.dirname(os.path.abspath(__file__)) +if current_dir not in sys.path: + sys.path.append(current_dir) + +# 导入ImageParser +from parser.image_parser import ImageParser + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +logger = logging.getLogger(__name__) + + +def init_ocr_model(): + """Initialize PaddleOCR model to pre-download and cache models""" + try: + logger.info("Initializing PaddleOCR model for pre-download...") + + # 使用与代码中相同的配置 + ocr_config = { + "use_gpu": False, + "text_det_limit_type": "max", + "text_det_limit_side_len": 960, + "use_doc_orientation_classify": True, # 启用文档方向分类 + "use_doc_unwarping": False, + "use_textline_orientation": True, # 启用文本行方向检测 + "text_recognition_model_name": "PP-OCRv4_server_rec", + "text_detection_model_name": "PP-OCRv4_server_det", + "text_det_thresh": 0.3, + "text_det_box_thresh": 0.6, + "text_det_unclip_ratio": 1.5, + "text_rec_score_thresh": 0.0, + "ocr_version": "PP-OCRv4", + "lang": "ch", + "show_log": False, + "use_dilation": True, + "det_db_score_mode": "slow", + } + + # 初始化PaddleOCR，这会触发模型下载和缓存 + ocr = PaddleOCR(**ocr_config) + logger.info("PaddleOCR model initialization completed successfully") + + # 测试OCR功能以确保模型正常工作 + import numpy as np + from PIL import Image + + # 创建一个简单的测试图像 + test_image = np.ones((100, 300, 3), dtype=np.uint8) * 255 + test_pil = Image.fromarray(test_image) + + # 执行一次OCR测试 + result = ocr.ocr(np.array(test_pil), cls=False) + logger.info("PaddleOCR test completed successfully") + + except Exception as e: + logger.error(f"Failed to initialize PaddleOCR model: {str(e)}") + raise diff --git a/services/docreader/src/parser/__init__.py b/services/docreader/src/parser/__init__.py new file mode 100644 index 0000000..c60a018 --- /dev/null +++ b/services/docreader/src/parser/__init__.py @@ -0,0 +1,42 @@ +""" +Parser module for WeKnora document processing system. + +This module provides document parsers for various file formats including: +- Microsoft Word documents (.doc, .docx) +- PDF documents +- Markdown files +- Plain text files +- Images with text content +- Web pages + +The parsers extract content from documents and can split them into +meaningful chunks for further processing and indexing. +""" + +from .base_parser import BaseParser, ParseResult +from .docx_parser import DocxParser +from .doc_parser import DocParser +from .pdf_parser import PDFParser +from .markdown_parser import MarkdownParser +from .text_parser import TextParser +from .image_parser import ImageParser +from .web_parser import WebParser +from .parser import Parser +from .config import ChunkingConfig +from .ocr_engine import OCREngine + +# Export public classes and modules +__all__ = [ + "BaseParser", # Base parser class that all format parsers inherit from + "DocxParser", # Parser for .docx files (modern Word documents) + "DocParser", # Parser for .doc files (legacy Word documents) + "PDFParser", # Parser for PDF documents + "MarkdownParser", # Parser for Markdown text files + "TextParser", # Parser for plain text files + "ImageParser", # Parser for images with text content + "WebParser", # Parser for web pages + "Parser", # Main parser factory that selects the appropriate parser + "ChunkingConfig", # Configuration for text chunking behavior + "ParseResult", # Standard result format returned by all parsers + "OCREngine", # OCR engine for extracting text from images +] diff --git a/services/docreader/src/parser/base_parser.py b/services/docreader/src/parser/base_parser.py new file mode 100644 index 0000000..e3af322 --- /dev/null +++ b/services/docreader/src/parser/base_parser.py @@ -0,0 +1,1247 @@ +# -*- coding: utf-8 -*- +import re +import os +import asyncio +from typing import List, Dict, Any, Optional, Tuple, Union +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +import logging +import sys +import traceback +import numpy as np +import time +import io +import json +from .ocr_engine import OCREngine +from .image_utils import image_to_base64 +from .config import ChunkingConfig +from .storage import create_storage +from PIL import Image + +# Add parent directory to Python path for src imports +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + +try: + from services.docreader.src.parser.caption import Caption +except ImportError: + # Fallback: try relative import + try: + from .caption import Caption + except ImportError: + # If both imports fail, set to None + Caption = None + logging.warning( + "Failed to import Caption, image captioning will be unavailable" + ) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +@dataclass +class Chunk: + """Chunk result""" + + content: str # Chunk content + seq: int # Chunk sequence number + start: int # Chunk start position + end: int # Chunk end position + images: List[Dict[str, Any]] = field(default_factory=list) # Images in the chunk + + +@dataclass +class ParseResult: + """Parse result""" + + text: str # Extracted text content + chunks: Optional[List[Chunk]] = None # Chunk results + title: Optional[str] = None # Document title (for URLs) + + +class BaseParser(ABC): + """Base parser interface""" + + # Class variable for shared OCR engine instance + _ocr_engine = None + _ocr_engine_failed = False + + @classmethod + def get_ocr_engine(cls, backend_type="paddle", **kwargs): + """Get OCR engine instance + + Args: + backend_type: OCR engine type, e.g. "paddle", "nanonets" + **kwargs: Arguments for the OCR engine + + Returns: + OCR engine instance or None + """ + if cls._ocr_engine is None and not cls._ocr_engine_failed: + try: + cls._ocr_engine = OCREngine.get_instance( + backend_type=backend_type, **kwargs + ) + if cls._ocr_engine is None: + cls._ocr_engine_failed = True + logger.error(f"Failed to initialize OCR engine ({backend_type})") + return None + logger.info(f"Successfully initialized OCR engine: {backend_type}") + except Exception as e: + cls._ocr_engine_failed = True + logger.error(f"Failed to initialize OCR engine: {str(e)}") + return None + return cls._ocr_engine + + def __init__( + self, + file_name: str = "", + file_type: str = None, + enable_multimodal: bool = True, + chunk_size: int = 1000, + chunk_overlap: int = 200, + separators: list = ["\n\n", "\n", "。"], + ocr_backend: str = "paddle", + ocr_config: dict = None, + max_image_size: int = 1920, # Maximum image size + max_concurrent_tasks: int = 5, # Max concurrent tasks + max_chunks: int = 1000, # Max number of returned chunks + chunking_config: ChunkingConfig = None, # Chunking configuration object + ): + """Initialize parser + + Args: + file_name: File name + file_type: File type, inferred from file_name if None + enable_multimodal: Whether to enable multimodal + chunk_size: Chunk size + chunk_overlap: Chunk overlap + separators: List of separators + ocr_backend: OCR engine type + ocr_config: OCR engine config + max_image_size: Maximum image size + max_concurrent_tasks: Max concurrent tasks + max_chunks: Max number of returned chunks + """ + # Storage client instance + self._storage = None + self.file_name = file_name + self.file_type = file_type or os.path.splitext(file_name)[1] + self.enable_multimodal = enable_multimodal + self.chunk_size = chunk_size + self.chunk_overlap = chunk_overlap + self.separators = separators + self.ocr_backend = os.getenv("OCR_BACKEND", ocr_backend) + self.ocr_config = ocr_config or {} + self.max_image_size = max_image_size + self.max_concurrent_tasks = max_concurrent_tasks + self.max_chunks = max_chunks + self.chunking_config = chunking_config + + logger.info( + f"Initializing {self.__class__.__name__} for file: {file_name}, type: {self.file_type}" + ) + logger.info( + f"Parser config: chunk_size={chunk_size}, " + f"overlap={chunk_overlap}, " + f"multimodal={enable_multimodal}, " + f"ocr_backend={ocr_backend}, " + f"max_chunks={max_chunks}" + ) + # Only initialize Caption service if multimodal is enabled + if self.enable_multimodal: + try: + self.caption_parser = Caption(self.chunking_config.vlm_config) + except Exception as e: + logger.warning(f"Failed to initialize Caption service: {str(e)}") + self.caption_parser = None + else: + self.caption_parser = None + + def perform_ocr(self, image): + """Execute OCR recognition on the image + + Args: + image: Image object (PIL.Image or numpy array) + + Returns: + Extracted text string + """ + start_time = time.time() + logger.info("Starting OCR recognition") + resized_image = None + + try: + # Resize image to avoid processing large images + resized_image = self._resize_image_if_needed(image) + + # Get OCR engine + ocr_engine = self.get_ocr_engine( + backend_type=self.ocr_backend, **self.ocr_config + ) + if ocr_engine is None: + logger.error( + f"OCR engine ({self.ocr_backend}) initialization failed or unavailable, " + "skipping OCR recognition" + ) + return "" + + # Execute OCR prediction + logger.info(f"Executing OCR prediction (using {self.ocr_backend} engine)") + # Add extra exception handling + try: + ocr_result = ocr_engine.predict(resized_image) + except RuntimeError as e: + # Handle common CUDA memory issues or other runtime errors + logger.error(f"OCR prediction runtime error: {str(e)}") + return "" + except Exception as e: + # Handle other prediction errors + logger.error(f"Unexpected OCR prediction error: {str(e)}") + return "" + + process_time = time.time() - start_time + logger.info(f"OCR recognition completed, time: {process_time:.2f} seconds") + return ocr_result + except Exception as e: + process_time = time.time() - start_time + logger.error( + f"OCR recognition error: {str(e)}, time: {process_time:.2f} seconds" + ) + return "" + finally: + # Release image resources + if resized_image is not image and hasattr(resized_image, "close"): + # Only close the new image we created, not the original image + resized_image.close() + + def _resize_image_if_needed(self, image): + """Resize image if it exceeds maximum size limit + + Args: + image: Image object (PIL.Image or numpy array) + + Returns: + Resized image object + """ + try: + # If it's a PIL Image + if hasattr(image, "size"): + width, height = image.size + if width > self.max_image_size or height > self.max_image_size: + logger.info(f"Resizing PIL image, original size: {width}x{height}") + scale = min( + self.max_image_size / width, self.max_image_size / height + ) + new_width = int(width * scale) + new_height = int(height * scale) + resized_image = image.resize((new_width, new_height)) + logger.info(f"Resized to: {new_width}x{new_height}") + return resized_image + else: + logger.info( + f"PIL image size {width}x{height} is within limits, no resizing needed" + ) + return image + # If it's a numpy array + elif hasattr(image, "shape"): + height, width = image.shape[:2] + if width > self.max_image_size or height > self.max_image_size: + logger.info( + f"Resizing numpy image, original size: {width}x{height}" + ) + scale = min( + self.max_image_size / width, self.max_image_size / height + ) + new_width = int(width * scale) + new_height = int(height * scale) + # Use PIL for resizing numpy arrays + pil_image = Image.fromarray(image) + resized_pil = pil_image.resize((new_width, new_height)) + resized_image = np.array(resized_pil) + logger.info(f"Resized to: {new_width}x{new_height}") + return resized_image + else: + logger.info( + f"Numpy image size {width}x{height} is within limits, no resizing needed" + ) + return image + else: + logger.warning(f"Unknown image type: {type(image)}, cannot resize") + return image + except Exception as e: + logger.error(f"Error resizing image: {str(e)}") + return image + + def process_image(self, image, image_url=None): + """Process image: first perform OCR, then get caption if text is available + + Args: + image: Image object (PIL.Image or numpy array) + image_url: Image URL (if uploaded) + + Returns: + tuple: (ocr_text, caption, image_url) + - ocr_text: OCR extracted text + - caption: Image description (if OCR has text) or empty string + - image_url: Image URL (if provided) + """ + logger.info("Starting image processing (OCR + optional caption)") + + # Resize image + image = self._resize_image_if_needed(image) + + # Perform OCR recognition + ocr_text = self.perform_ocr(image) + caption = "" + + if self.caption_parser: + logger.info( + f"OCR successfully extracted {len(ocr_text)} characters, continuing to get caption" + ) + # Convert image to base64 for caption generation + img_base64 = image_to_base64(image) + if img_base64: + caption = self.get_image_caption(img_base64) + if caption: + logger.info(f"Successfully obtained image caption: {caption}") + else: + logger.warning("Failed to get caption") + else: + logger.warning("Failed to convert image to base64") + caption = "" + else: + logger.info("Caption service not initialized, skipping caption retrieval") + + # Release image resources + del image + + return ocr_text, caption, image_url + + async def process_image_async(self, image, image_url=None): + """Asynchronously process image: first perform OCR, then get caption if text is available + + Args: + image: Image object (PIL.Image or numpy array) + image_url: Image URL (if uploaded) + + Returns: + tuple: (ocr_text, caption, image_url) + - ocr_text: OCR extracted text + - caption: Image description (if OCR has text) or empty string + - image_url: Image URL (if provided) + """ + logger.info("Starting asynchronous image processing (OCR + optional caption)") + resized_image = None + + try: + # Resize image + resized_image = self._resize_image_if_needed(image) + + # Perform OCR recognition (using run_in_executor to execute synchronous operations in the event loop) + loop = asyncio.get_event_loop() + try: + # Add timeout mechanism to avoid infinite blocking (30 seconds timeout) + ocr_task = loop.run_in_executor(None, self.perform_ocr, resized_image) + ocr_text = await asyncio.wait_for(ocr_task, timeout=30.0) + except asyncio.TimeoutError: + logger.error( + "OCR processing timed out (30 seconds), skipping this image" + ) + ocr_text = "" + except Exception as e: + logger.error(f"OCR processing error: {str(e)}") + ocr_text = "" + + logger.info( + f"OCR successfully extracted {len(ocr_text)} characters, continuing to get caption" + ) + caption = "" + if self.caption_parser: + try: + # Convert image to base64 for caption generation + img_base64 = image_to_base64(resized_image) + if img_base64: + # Add timeout to avoid blocking caption retrieval (30 seconds timeout) + caption_task = self.get_image_caption_async(img_base64) + image_data, caption = await asyncio.wait_for( + caption_task, timeout=30.0 + ) + if caption: + logger.info( + f"Successfully obtained image caption: {caption}" + ) + else: + logger.warning("Failed to get caption") + else: + logger.warning("Failed to convert image to base64") + caption = "" + except asyncio.TimeoutError: + logger.warning("Caption retrieval timed out, skipping") + except Exception as e: + logger.error(f"Failed to get caption: {str(e)}") + else: + logger.info( + "Caption service not initialized, skipping caption retrieval" + ) + + return ocr_text, caption, image_url + finally: + # Release image resources + if resized_image is not image and hasattr(resized_image, "close"): + # Only close the new image we created, not the original image + resized_image.close() + + async def process_with_limit(self, idx, image, url, semaphore): + """Function to process a single image using a semaphore""" + try: + logger.info(f"Waiting to process image {idx+1}") + async with semaphore: # Use semaphore to control concurrency + logger.info(f"Starting to process image {idx+1}") + result = await self.process_image_async(image, url) + logger.info(f"Completed processing image {idx+1}") + return result + except Exception as e: + logger.error(f"Error processing image {idx+1}: {str(e)}") + return ("", "", url) # Return empty result to avoid overall failure + finally: + # Manually release image resources + if hasattr(image, "close"): + image.close() + + async def process_multiple_images(self, images_data): + """Process multiple images concurrently + + Args: + images_data: List of (image, image_url) tuples + + Returns: + List of (ocr_text, caption, image_url) tuples + """ + logger.info(f"Starting concurrent processing of {len(images_data)} images") + + if not images_data: + logger.warning("No image data to process") + return [] + + # Set max concurrency, reduce concurrency to avoid resource contention + max_concurrency = min( + self.max_concurrent_tasks, 1 + ) # Reduce concurrency to prevent excessive memory usage + + # Use semaphore to limit concurrency + semaphore = asyncio.Semaphore(max_concurrency) + + # Store results to avoid overall failure due to task failure + results = [] + + # Create all tasks, but use semaphore to limit actual concurrency + tasks = [ + self.process_with_limit(i, img, url, semaphore) + for i, (img, url) in enumerate(images_data) + ] + + try: + # Execute all tasks, but set overall timeout + completed_results = await asyncio.gather(*tasks, return_exceptions=True) + + # Handle possible exception results + for i, result in enumerate(completed_results): + if isinstance(result, Exception): + logger.error( + f"Image {i+1} processing returned an exception: {str(result)}" + ) + # For exceptions, add empty results + if i < len(images_data): + results.append(("", "", images_data[i][1])) + else: + results.append(result) + except Exception as e: + logger.error(f"Error during concurrent image processing: {str(e)}") + # Add empty results for all images + results = [("", "", url) for _, url in images_data] + finally: + # Clean up references and trigger garbage collection + images_data.clear() + logger.info("Image processing resource cleanup complete") + + logger.info( + f"Completed concurrent processing of {len(results)}/{len(images_data)} images" + ) + return results + + def decode_bytes(self, content: bytes) -> str: + """Intelligently decode byte stream, supports multiple encodings + + Tries to decode in common encodings, if all fail, uses latin-1 as fallback + + Args: + content: Byte stream to decode + + Returns: + Decoded string + """ + logger.info(f"Attempting to decode bytes of length: {len(content)}") + # Common encodings, sorted by priority + encodings = ["utf-8", "gb18030", "gb2312", "gbk", "big5", "ascii", "latin-1"] + text = None + + # Try decoding with each encoding format + for encoding in encodings: + try: + text = content.decode(encoding) + logger.info(f"Successfully decoded content using {encoding} encoding") + break + except UnicodeDecodeError: + logger.info(f"Failed to decode using {encoding} encoding") + continue + + # If all encodings fail, use latin-1 as fallback + if text is None: + text = content.decode("latin-1") + logger.warning( + f"Unable to determine correct encoding, using latin-1 as fallback. " + f"This may cause character issues." + ) + + logger.info(f"Decoded text length: {len(text)} characters") + return text + + def get_image_caption(self, image_data: str) -> str: + """Get image description + + Args: + image_data: Image data (base64 encoded string or URL) + + Returns: + Image description + """ + start_time = time.time() + logger.info( + f"Getting caption for image: {image_data[:250]}..." + if len(image_data) > 250 + else f"Getting caption for image: {image_data}" + ) + caption = self.caption_parser.get_caption(image_data) + if caption: + logger.info( + f"Received caption of length: {len(caption)}, caption: {caption}," + f"cost: {time.time() - start_time} seconds" + ) + else: + logger.warning("Failed to get caption for image") + return caption + + async def get_image_caption_async(self, image_data: str) -> Tuple[str, str]: + """Asynchronously get image description + + Args: + image_data: Image data (base64 encoded string or URL) + + Returns: + Tuple[str, str]: Image data and corresponding description + """ + caption = self.get_image_caption(image_data) + return image_data, caption + + def __init_storage(self): + """Initialize storage client based on configuration""" + if self._storage is None: + storage_config = ( + self.chunking_config.storage_config if self.chunking_config else None + ) + self._storage = create_storage(storage_config) + logger.info( + f"Initialized storage client: {self._storage.__class__.__name__}" + ) + return self._storage + + def upload_file(self, file_path: str) -> str: + """Upload file to object storage + + Args: + file_path: File path + + Returns: + File URL + """ + logger.info(f"Uploading file: {file_path}") + try: + storage = self.__init_storage() + return storage.upload_file(file_path) + except Exception as e: + logger.error(f"Failed to upload file: {str(e)}") + return "" + + def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str: + """Upload bytes to object storage + + Args: + content: Byte content to upload + file_ext: File extension + + Returns: + File URL + """ + logger.info(f"Uploading bytes content, size: {len(content)} bytes") + try: + storage = self.__init_storage() + return storage.upload_bytes(content, file_ext) + except Exception as e: + logger.error(f"Failed to upload bytes to storage: {str(e)}") + traceback.print_exc() + return "" + + @abstractmethod + def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]: + """Parse document content + + Args: + content: Document content + + Returns: + Either a string containing the parsed text, or a tuple of (text, image_map) + where image_map is a dict mapping image URLs to Image objects + """ + pass + + def parse(self, content: bytes) -> ParseResult: + """Parse document content + + Args: + content: Document content + + Returns: + Parse result + """ + logger.info( + f"Parsing document with {self.__class__.__name__}, content size: {len(content)} bytes" + ) + parse_result = self.parse_into_text(content) + if isinstance(parse_result, tuple): + text, image_map = parse_result + else: + text = parse_result + image_map = {} + logger.info(f"Extracted {len(text)} characters of text from {self.file_name}") + logger.info(f"Beginning chunking process for text") + chunks = self.chunk_text(text) + logger.info(f"Created {len(chunks)} chunks from document") + + # Limit the number of returned chunks + if len(chunks) > self.max_chunks: + logger.warning( + f"Limiting chunks from {len(chunks)} to maximum {self.max_chunks}" + ) + chunks = chunks[: self.max_chunks] + + # If multimodal is enabled and file type is supported, process images in each chunk + if self.enable_multimodal: + # Get file extension and convert to lowercase + file_ext = ( + os.path.splitext(self.file_name)[1].lower() + if self.file_name + else (self.file_type.lower() if self.file_type else "") + ) + + # Define allowed file types for image processing + allowed_types = [ + ".pdf", # PDF files + ".md", + ".markdown", # Markdown files + ".doc", + ".docx", # Word documents + # Image files + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".webp", + ] + + if file_ext in allowed_types: + logger.info( + f"Processing images in each chunk for file type: {file_ext}" + ) + chunks = self.process_chunks_images(chunks, image_map) + else: + logger.info( + f"Skipping image processing for unsupported file type: {file_ext}" + ) + + # Include title if available (for WebParser) + title = getattr(self, 'extracted_title', None) + return ParseResult(text=text, chunks=chunks, title=title) + + def _split_into_units(self, text: str) -> List[str]: + """ + Args: + text: 文本内容 + + Returns: + 基本单元的列表 + """ + logger.info( + f"Splitting text into basic units with robust structure protection, text length: {len(text)}" + ) + + # 定义所有需要作为整体保护的结构模式 --- + table_pattern = r"(?m)(^\|.*\|[ \t]*\r?\n(?:[ \t]*\r?\n)?^\|\s*:?--+.*\r?\n(?:^\|.*\|\r?\n?)*)" + + # 其他需要保护的结构（代码块、公式块、行内元素） + code_block_pattern = r"```[\s\S]*?```" + math_block_pattern = r"\$\$[\s\S]*?\$\$" + inline_pattern = r"!\[.*?\]\(.*?\)|\[.*?\]\(.*?\)" + + # 查找所有受保护结构的位置 --- + protected_ranges = [] + for pattern in [ + table_pattern, + code_block_pattern, + math_block_pattern, + inline_pattern, + ]: + for match in re.finditer(pattern, text): + # 确保匹配到的不是空字符串，避免无效范围 + if match.group(0).strip(): + protected_ranges.append((match.start(), match.end())) + + # 按起始位置排序 + protected_ranges.sort(key=lambda x: x[0]) + logger.info( + f"Found {len(protected_ranges)} protected structures (tables, code, formulas, images, links)." + ) + + # 合并可能重叠的保护范围 --- + # 确保我们有一组不相交的、需要保护的文本块 + if protected_ranges: + merged_ranges = [] + current_start, current_end = protected_ranges[0] + + for next_start, next_end in protected_ranges[1:]: + if next_start < current_end: + # 如果下一个范围与当前范围重叠，则合并它们 + current_end = max(current_end, next_end) + else: + # 如果不重叠，则完成当前范围并开始一个新的范围 + merged_ranges.append((current_start, current_end)) + current_start, current_end = next_start, next_end + + merged_ranges.append((current_start, current_end)) + protected_ranges = merged_ranges + logger.info( + f"After merging overlaps, {len(protected_ranges)} protected ranges remain." + ) + + # 根据保护范围和分隔符来分割文本 --- + units = [] + last_end = 0 + + # 定义分隔符的正则表达式，通过加括号来保留分隔符本身 + separator_pattern = f"({'|'.join(re.escape(s) for s in self.separators)})" + + for start, end in protected_ranges: + # a. 处理受保护范围之前的文本 + if start > last_end: + pre_text = text[last_end:start] + # 对这部分非保护文本进行分割，并保留分隔符 + segments = re.split(separator_pattern, pre_text) + units.extend([s for s in segments if s]) # 添加所有非空部分 + + # b. 将整个受保护的块（例如，一个完整的表格）作为一个单独的、不可分割的单元添加 + protected_text = text[start:end] + units.append(protected_text) + + last_end = end + + # c. 处理最后一个受保护范围之后的文本 + if last_end < len(text): + post_text = text[last_end:] + segments = re.split(separator_pattern, post_text) + units.extend([s for s in segments if s]) # 添加所有非空部分 + + logger.info(f"Text splitting complete, created {len(units)} final basic units.") + return units + + def _find_complete_units(self, units: List[str], target_size: int) -> List[str]: + """Find a list of complete units that do not exceed the target size + + Args: + units: List of units + target_size: Target size + + Returns: + List of complete units + """ + logger.info(f"Finding complete units with target size: {target_size}") + result = [] + current_size = 0 + + for unit in units: + unit_size = len(unit) + if current_size + unit_size > target_size and result: + logger.info( + f"Reached target size limit at {current_size} characters, stopping" + ) + break + result.append(unit) + current_size += unit_size + logger.info( + f"Added unit of size {unit_size}, current total: {current_size}/{target_size}" + ) + + logger.info( + f"Found {len(result)} complete units totaling {current_size} characters" + ) + return result + + def chunk_text(self, text: str) -> List[Chunk]: + """Chunk text, preserving Markdown structure + + Args: + text: Text content + + Returns: + List of text chunks + """ + if not text: + logger.warning("Empty text provided for chunking, returning empty list") + return [] + + logger.info(f"Starting text chunking process, text length: {len(text)}") + logger.info( + f"Chunking parameters: size={self.chunk_size}, overlap={self.chunk_overlap}" + ) + + # Split text into basic units + units = self._split_into_units(text) + logger.info(f"Split text into {len(units)} basic units") + + chunks = [] + current_chunk = [] + current_size = 0 + current_start = 0 + + for i, unit in enumerate(units): + unit_size = len(unit) + logger.info(f"Processing unit {i+1}/{len(units)}, size: {unit_size}") + + # If current chunk plus new unit exceeds size limit, create new chunk + if current_size + unit_size > self.chunk_size and current_chunk: + chunk_text = "".join(current_chunk) + chunks.append( + Chunk( + seq=len(chunks), + content=chunk_text, + start=current_start, + end=current_start + len(chunk_text), + ) + ) + logger.info(f"Created chunk {len(chunks)}, size: {len(chunk_text)}") + + # Keep overlap, ensuring structure integrity + if self.chunk_overlap > 0: + # Calculate target overlap size + overlap_target = min(self.chunk_overlap, len(chunk_text)) + logger.info( + f"Calculating overlap with target size: {overlap_target}" + ) + + # Find complete units from the end + overlap_units = [] + overlap_size = 0 + + for u in reversed(current_chunk): + if overlap_size + len(u) > overlap_target: + logger.info( + f"Reached overlap target ({overlap_size}/{overlap_target})" + ) + break + overlap_units.insert(0, u) + overlap_size += len(u) + logger.info( + f"Added unit to overlap, current overlap size: {overlap_size}" + ) + + # Remove elements from overlap that are included in separators + start_index = 0 + for i, u in enumerate(overlap_units): + # Check if u is in separators + all_of_separator = True + for uu in u: + if uu not in self.separators: + all_of_separator = False + break + if all_of_separator: + # Remove the first element + start_index = i + 1 + overlap_size = overlap_size - len(u) + logger.info(f"Removed separator from overlap: '{u}'") + else: + break + + overlap_units = overlap_units[start_index:] + logger.info( + f"Final overlap: {len(overlap_units)} units, {overlap_size} characters" + ) + + current_chunk = overlap_units + current_size = overlap_size + # Update start position, considering overlap + current_start = current_start + len(chunk_text) - overlap_size + else: + logger.info("No overlap configured, starting fresh chunk") + current_chunk = [] + current_size = 0 + current_start = current_start + len(chunk_text) + + current_chunk.append(unit) + current_size += unit_size + logger.info( + f"Added unit to current chunk, now at {current_size}/{self.chunk_size} characters" + ) + + # Add the last chunk + if current_chunk: + chunk_text = "".join(current_chunk) + chunks.append( + Chunk( + seq=len(chunks), + content=chunk_text, + start=current_start, + end=current_start + len(chunk_text), + ) + ) + logger.info(f"Created final chunk {len(chunks)}, size: {len(chunk_text)}") + + logger.info(f"Chunking complete, created {len(chunks)} chunks from text") + return chunks + + def extract_images_from_chunk(self, chunk: Chunk) -> List[Dict[str, str]]: + """Extract image information from a chunk + + Args: + chunk: Document chunk + + Returns: + List of image information, each element contains image URL and match position + """ + logger.info(f"Extracting image information from Chunk #{chunk.seq}") + text = chunk.content + + # Regex to extract image information from text, supporting Markdown images and HTML images + img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)|]*src="([^"]+)" [^>]*>' + + # Extract image information + img_matches = list(re.finditer(img_pattern, text)) + logger.info(f"Chunk #{chunk.seq} found {len(img_matches)} images") + + images_info = [] + for match_idx, match in enumerate(img_matches): + # Process image URL + img_url = match.group(2) if match.group(2) else match.group(3) + alt_text = match.group(1) if match.group(1) else "" + + # Record image information + image_info = { + "original_url": img_url, + "start": match.start(), + "end": match.end(), + "alt_text": alt_text, + "match_text": text[match.start() : match.end()], + } + images_info.append(image_info) + + logger.info( + f"Image in Chunk #{chunk.seq} {match_idx+1}: " f"URL={img_url[:50]}..." + if len(img_url) > 50 + else f"Image in Chunk #{chunk.seq} {match_idx+1}: URL={img_url}" + ) + + return images_info + + async def download_and_upload_image(self, img_url: str): + """Download image and upload to object storage, if it's already an object storage path or local path, use directly + + Args: + img_url: Image URL or local path + + Returns: + tuple: (original URL, storage URL, image object), if failed returns (original URL, None, None) + """ + + try: + import requests + from PIL import Image + import io + + # Check if it's already a storage URL (COS or MinIO) + is_storage_url = any( + pattern in img_url + for pattern in ["cos", "myqcloud.com", "minio", ".s3."] + ) + if is_storage_url: + logger.info(f"Image already on COS: {img_url}, no need to re-upload") + try: + # Still need to get image object for OCR processing + # Get proxy settings from environment variables + http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") + https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") + proxies = {} + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + response = requests.get(img_url, timeout=5, proxies=proxies) + if response.status_code == 200: + image = Image.open(io.BytesIO(response.content)) + try: + return img_url, img_url, image + finally: + # Ensure image resources are also released after the function returns + # Image will be closed by the caller + pass + else: + logger.warning( + f"Failed to get storage image: {response.status_code}" + ) + return img_url, img_url, None + except Exception as e: + logger.error(f"Error getting storage image: {str(e)}") + return img_url, img_url, None + + # Check if it's a local file path + elif os.path.exists(img_url) and os.path.isfile(img_url): + logger.info(f"Using local image file: {img_url}") + image = None + try: + # Read local image + image = Image.open(img_url) + # Upload to storage + with open(img_url, "rb") as f: + content = f.read() + storage_url = self.upload_bytes(content) + logger.info( + f"Successfully uploaded local image to storage: {storage_url}" + ) + return img_url, storage_url, image + except Exception as e: + logger.error(f"Error processing local image: {str(e)}") + if image and hasattr(image, "close"): + image.close() + return img_url, None, None + + # Normal remote URL download handling + else: + # Get proxy settings from environment variables + http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") + https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") + proxies = {} + if http_proxy: + proxies["http"] = http_proxy + if https_proxy: + proxies["https"] = https_proxy + + logger.info( + f"Downloading image {img_url}, using proxy: {proxies if proxies else 'None'}" + ) + response = requests.get(img_url, timeout=5, proxies=proxies) + + if response.status_code == 200: + # Download successful, create image object + image = Image.open(io.BytesIO(response.content)) + try: + # Upload to storage using the method in BaseParser + storage_url = self.upload_bytes(response.content) + logger.info( + f"Successfully uploaded image to storage: {storage_url}" + ) + return img_url, storage_url, image + finally: + # Image will be closed by the caller + pass + else: + logger.warning(f"Failed to download image: {response.status_code}") + return img_url, None, None + + except Exception as e: + logger.error(f"Error downloading or processing image: {str(e)}") + return img_url, None, None + + async def process_chunk_images_async( + self, chunk, chunk_idx, total_chunks, image_map=None + ): + """Asynchronously process images in a single Chunk + + Args: + chunk: Chunk object to process + chunk_idx: Chunk index + total_chunks: Total number of chunks + image_map: Optional dictionary mapping image URLs to Image objects + + Returns: + Processed Chunk object + """ + + logger.info( + f"Starting to process images in Chunk #{chunk_idx+1}/{total_chunks}" + ) + + # Extract image information from the Chunk + images_info = self.extract_images_from_chunk(chunk) + if not images_info: + logger.info(f"Chunk #{chunk_idx+1} found no images") + return chunk + + # Prepare images that need to be downloaded and processed + images_to_process = [] + url_to_info_map = {} # Map URL to image information + + # Record all image URLs that need to be processed + for img_info in images_info: + url = img_info["original_url"] + url_to_info_map[url] = img_info + + results = [] + download_tasks = [] + for img_url in url_to_info_map.keys(): # Check if image is already in the image_map + if image_map and img_url in image_map: + logger.info(f"Image already in image_map: {img_url}, using cached object") + results.append((img_url, img_url, image_map[img_url])) + else: + download_task = self.download_and_upload_image(img_url) + download_tasks.append(download_task) + # Concurrent download and upload of images, ignore images that are already in the image_map + results.extend(await asyncio.gather(*download_tasks)) + + # Process download results, prepare for OCR processing + for orig_url, cos_url, image in results: + if cos_url and image: + img_info = url_to_info_map[orig_url] + img_info["cos_url"] = cos_url + images_to_process.append((image, cos_url)) + + # If no images were successfully downloaded and uploaded, return the original Chunk + if not images_to_process: + logger.info( + f"Chunk #{chunk_idx+1} found no successfully downloaded and uploaded images" + ) + return chunk + + # Concurrent processing of all images (OCR + caption) + logger.info( + f"Processing {len(images_to_process)} images in Chunk #{chunk_idx+1}" + ) + + # Concurrent processing of all images + processed_results = await self.process_multiple_images(images_to_process) + + # Process OCR and Caption results + for ocr_text, caption, img_url in processed_results: + # Find the corresponding original URL + for orig_url, info in url_to_info_map.items(): + if info.get("cos_url") == img_url: + info["ocr_text"] = ocr_text if ocr_text else "" + info["caption"] = caption if caption else "" + + if ocr_text: + logger.info( + f"Image OCR extracted {len(ocr_text)} characters: {img_url}" + ) + if caption: + logger.info(f"Obtained image description: '{caption}'") + break + + # Add processed image information to the Chunk + processed_images = [] + for img_info in images_info: + if "cos_url" in img_info: + processed_images.append(img_info) + + # Update image information in the Chunk + chunk.images = processed_images + + logger.info(f"Completed image processing in Chunk #{chunk_idx+1}") + return chunk + + def process_chunks_images(self, chunks: List[Chunk], image_map=None) -> List[Chunk]: + """Concurrent processing of images in all Chunks + + Args: + chunks: List of document chunks + + Returns: + List of processed document chunks + """ + logger.info( + f"Starting concurrent processing of images in all {len(chunks)} chunks" + ) + + if not chunks: + logger.warning("No chunks to process") + return chunks + + # Create and run all Chunk concurrent processing tasks + async def process_all_chunks(): + # Set max concurrency, reduce concurrency to avoid resource contention + max_concurrency = min(self.max_concurrent_tasks, 1) # Reduce concurrency + # Use semaphore to limit concurrency + semaphore = asyncio.Semaphore(max_concurrency) + + async def process_with_limit(chunk, idx, total): + """Use semaphore to control concurrent processing of Chunks""" + async with semaphore: + return await self.process_chunk_images_async( + chunk, idx, total, image_map + ) + + # Create tasks for all Chunks + tasks = [ + process_with_limit(chunk, idx, len(chunks)) + for idx, chunk in enumerate(chunks) + ] + + # Execute all tasks concurrently + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Handle possible exceptions + processed_chunks = [] + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.error(f"Error processing Chunk {i+1}: {str(result)}") + # Keep original Chunk + if i < len(chunks): + processed_chunks.append(chunks[i]) + else: + processed_chunks.append(result) + + return processed_chunks + + # Create event loop and run all tasks + try: + # Check if event loop already exists + try: + loop = asyncio.get_event_loop() + if loop.is_closed(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + except RuntimeError: + # If no event loop, create a new one + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + # Execute processing for all Chunks + processed_chunks = loop.run_until_complete(process_all_chunks()) + logger.info( + f"Successfully completed concurrent processing of {len(processed_chunks)}/{len(chunks)} chunks" + ) + + return processed_chunks + except Exception as e: + logger.error(f"Error during concurrent chunk processing: {str(e)}") + return chunks diff --git a/services/docreader/src/parser/caption.py b/services/docreader/src/parser/caption.py new file mode 100644 index 0000000..d84bf6d --- /dev/null +++ b/services/docreader/src/parser/caption.py @@ -0,0 +1,360 @@ +import json +import logging +import os +import time +from dataclasses import dataclass, field +from typing import List, Optional, Union + +import requests +import ollama + + +logger = logging.getLogger(__name__) + + +@dataclass +class ImageUrl: + """Image URL data structure for caption requests.""" + + url: Optional[str] = None + detail: Optional[str] = None + + +@dataclass +class Content: + """Content data structure that can contain text or image URL.""" + + type: Optional[str] = None + text: Optional[str] = None + image_url: Optional[ImageUrl] = None + + +@dataclass +class SystemMessage: + """System message for VLM model requests.""" + + role: Optional[str] = None + content: Optional[str] = None + + +@dataclass +class UserMessage: + """User message for VLM model requests, can contain multiple content items.""" + + role: Optional[str] = None + content: List[Content] = field(default_factory=list) + + +@dataclass +class CompletionRequest: + """Request structure for VLM model completion API.""" + + model: str + temperature: float + top_p: float + messages: List[Union[SystemMessage, UserMessage]] + user: str + + +@dataclass +class Model: + """Model identifier structure.""" + + id: str + + +@dataclass +class ModelsResp: + """Response structure for available models API.""" + + data: List[Model] = field(default_factory=list) + + +@dataclass +class Message: + """Message structure in API response.""" + + role: Optional[str] = None + content: Optional[str] = None + tool_calls: Optional[str] = None + + +@dataclass +class Choice: + """Choice structure in API response.""" + + message: Optional[Message] = None + + +@dataclass +class Usage: + """Token usage information in API response.""" + + prompt_tokens: Optional[int] = 0 + total_tokens: Optional[int] = 0 + completion_tokens: Optional[int] = 0 + + +@dataclass +class CaptionChatResp: + """Response structure for caption chat API.""" + + id: Optional[str] = None + created: Optional[int] = None + model: Optional[Model] = None + object: Optional[str] = None + choices: List[Choice] = field(default_factory=list) + usage: Optional[Usage] = None + + @staticmethod + def from_json(json_data: dict) -> "CaptionChatResp": + """ + Parse API response JSON into a CaptionChatResp object. + + Args: + json_data: The JSON response from the API + + Returns: + A parsed CaptionChatResp object + """ + logger.info("Parsing CaptionChatResp from JSON") + # Manually parse nested fields with safe field extraction + choices = [] + for choice in json_data.get("choices", []): + message_data = choice.get("message", {}) + message = Message( + role=message_data.get("role"), + content=message_data.get("content"), + tool_calls=message_data.get("tool_calls"), + ) + choices.append(Choice(message=message)) + + # Handle usage with safe field extraction + usage_data = json_data.get("usage", {}) + usage = None + if usage_data: + usage = Usage( + prompt_tokens=usage_data.get("prompt_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + completion_tokens=usage_data.get("completion_tokens", 0), + ) + + logger.info( + f"Parsed {len(choices)} choices and usage data: {usage is not None}" + ) + return CaptionChatResp( + id=json_data.get("id"), + created=json_data.get("created"), + model=json_data.get("model"), + object=json_data.get("object"), + choices=choices, + usage=usage, + ) + + def choice_data(self) -> str: + """ + Extract the content from the first choice in the response. + + Returns: + The content string from the first choice, or empty string if no choices + """ + if self.choices: + logger.info("Retrieving content from first choice") + return self.choices[0].message.content + logger.warning("No choices available in response") + return "" + + +class Caption: + """ + Service for generating captions for images using a Vision Language Model. + Uses an external API to process images and return textual descriptions. + """ + + def __init__(self, vlm_config=None): + """Initialize the Caption service with configuration from parameters or environment variables.""" + logger.info("Initializing Caption service") + self.prompt = """简单凝炼的描述图片的主要内容""" + + # Use provided VLM config if available, otherwise fall back to environment variables + if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"): + self.completion_url = vlm_config.get("base_url", "") + "/chat/completions" + self.model = vlm_config.get("model_name", "") + self.api_key = vlm_config.get("api_key", "") + self.interface_type = vlm_config.get("interface_type", "openai").lower() + else: + if os.getenv("VLM_MODEL_BASE_URL") == "" or os.getenv("VLM_MODEL_NAME") == "": + logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set") + return + self.completion_url = os.getenv("VLM_MODEL_BASE_URL") + "/chat/completions" + self.model = os.getenv("VLM_MODEL_NAME") + self.api_key = os.getenv("VLM_MODEL_API_KEY") + self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower() + + # 验证接口类型 + if self.interface_type not in ["ollama", "openai"]: + logger.warning(f"Unknown interface type: {self.interface_type}, defaulting to openai") + self.interface_type = "openai" + + logger.info( + f"Service configured with model: {self.model}, endpoint: {self.completion_url}, interface: {self.interface_type}" + ) + + def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]: + """ + Call the Caption API to generate a description for the given image. + + Args: + image_data: URL of the image or base64 encoded image data + + Returns: + CaptionChatResp object if successful, None otherwise + """ + logger.info(f"Calling Caption API for image captioning") + logger.info(f"Processing image data: {image_data[:50] if len(image_data) > 50 else image_data}") + + # 根据接口类型选择调用方式 + if self.interface_type == "ollama": + return self._call_ollama_api(image_data) + else: + return self._call_openai_api(image_data) + + def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]: + """Call Ollama API for image captioning using base64 encoded image data.""" + + host = self.completion_url.replace("/v1/chat/completions", "") + + client = ollama.Client( + host=host, + ) + + try: + logger.info(f"Calling Ollama API with model: {self.model}") + + # 调用Ollama API，使用images参数传递base64编码的图片 + response = client.generate( + model=self.model, + prompt="简单凝炼的描述图片的主要内容", + images=[image_base64], # image_base64是base64编码的图片数据 + options={"temperature": 0.1}, + stream=False, + ) + + # 构造响应对象 + caption_resp = CaptionChatResp( + id="ollama_response", + created=int(time.time()), + model=self.model, + object="chat.completion", + choices=[ + Choice( + message=Message( + role="assistant", + content=response.response + ) + ) + ] + ) + + logger.info("Successfully received response from Ollama API") + return caption_resp + + except Exception as e: + logger.error(f"Error calling Ollama API: {e}") + return None + + def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]: + """Call OpenAI-compatible API for image captioning.""" + logger.info(f"Calling OpenAI-compatible API with model: {self.model}") + + user_msg = UserMessage( + role="user", + content=[ + Content(type="text", text=self.prompt), + Content( + type="image_url", image_url=ImageUrl(url="data:image/png;base64," + image_base64, detail="auto") + ), + ], + ) + + gpt_req = CompletionRequest( + model=self.model, + temperature=0.3, + top_p=0.8, + messages=[user_msg], + user="abc", + ) + + headers = { + "Content-Type": "application/json", + "Accept": "text/event-stream", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + } + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + + try: + logger.info(f"Sending request to OpenAI-compatible API with model: {self.model}") + response = requests.post( + self.completion_url, + data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4), + headers=headers, + timeout=30, + ) + if response.status_code != 200: + logger.error( + f"OpenAI-compatible API returned non-200 status code: {response.status_code}" + ) + response.raise_for_status() + + logger.info( + f"Successfully received response from OpenAI-compatible API with status: {response.status_code}" + ) + logger.info(f"Converting response to CaptionChatResp object") + caption_resp = CaptionChatResp.from_json(response.json()) + + if caption_resp.usage: + logger.info( + f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, " + f"completion_tokens={caption_resp.usage.completion_tokens}" + ) + + return caption_resp + except requests.exceptions.Timeout: + logger.error(f"Timeout while calling OpenAI-compatible API after 30 seconds") + return None + except requests.exceptions.RequestException as e: + logger.error(f"Request error calling OpenAI-compatible API: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error calling OpenAI-compatible API: {e}") + return None + + def get_caption(self, image_data: str) -> str: + """ + Get a caption for the provided image data. + + Args: + image_data: URL of the image or base64 encoded image data + + Returns: + Caption text as string, or empty string if captioning failed + """ + logger.info("Getting caption for image") + if not image_data or self.completion_url is None: + logger.error("Image data is not set") + return "" + caption_resp = self._call_caption_api(image_data) + if caption_resp: + caption = caption_resp.choice_data() + caption_length = len(caption) + logger.info(f"Successfully generated caption of length {caption_length}") + logger.info( + f"Caption: {caption[:50]}..." + if caption_length > 50 + else f"Caption: {caption}" + ) + return caption + logger.warning("Failed to get caption from Caption API") + return "" diff --git a/services/docreader/src/parser/config.py b/services/docreader/src/parser/config.py new file mode 100644 index 0000000..85f9cb5 --- /dev/null +++ b/services/docreader/src/parser/config.py @@ -0,0 +1,21 @@ +from dataclasses import dataclass, field + + +@dataclass +class ChunkingConfig: + """ + Configuration for text chunking process. + Controls how documents are split into smaller pieces for processing. + """ + + chunk_size: int = 512 # Maximum size of each chunk in tokens/chars + chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks + separators: list = field( + default_factory=lambda: ["\n\n", "\n", "。"] + ) # Text separators in order of priority + enable_multimodal: bool = ( + False # Whether to enable multimodal processing (text + images) + ) + storage_config: dict = None # Preferred field name going forward + vlm_config: dict = None # VLM configuration for image captioning + diff --git a/services/docreader/src/parser/doc_parser.py b/services/docreader/src/parser/doc_parser.py new file mode 100644 index 0000000..71fc897 --- /dev/null +++ b/services/docreader/src/parser/doc_parser.py @@ -0,0 +1,315 @@ +import asyncio +import logging +import re +import tempfile +import os +import subprocess +import shutil +from io import BytesIO +from typing import Optional, List, Tuple +import textract +from PIL import Image +import zipfile +import xml.etree.ElementTree as ET + +from .base_parser import BaseParser +from .docx_parser import DocxParser, Docx + +logger = logging.getLogger(__name__) + + +class DocParser(BaseParser): + """DOC document parser""" + + def parse_into_text(self, content: bytes) -> str: + """Parse DOC document + + Args: + content: DOC document content + + Returns: + Parse result + """ + logger.info(f"Parsing DOC document, content size: {len(content)} bytes") + + # Save byte content as a temporary file + with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as temp_file: + temp_file_path = temp_file.name + temp_file.write(content) + temp_file.flush() + logger.info(f"Saved DOC content to temporary file: {temp_file_path}") + + try: + # First try to convert to docx format to extract images + if self.enable_multimodal: + logger.info("Multimodal enabled, attempting to extract images from DOC") + docx_content = self._convert_doc_to_docx(temp_file_path) + + if docx_content: + logger.info("Successfully converted DOC to DOCX, using DocxParser") + # Use existing DocxParser to parse the converted docx + docx_parser = DocxParser( + file_name=self.file_name, + file_type="docx", + enable_multimodal=self.enable_multimodal, + chunk_size=self.chunk_size, + chunk_overlap=self.chunk_overlap, + chunking_config=self.chunking_config, + separators=self.separators, + ) + text = docx_parser.parse_into_text(docx_content) + logger.info(f"Extracted {len(text)} characters using DocxParser") + + # Clean up temporary file + os.unlink(temp_file_path) + logger.info(f"Deleted temporary file: {temp_file_path}") + + return text + else: + logger.warning( + "Failed to convert DOC to DOCX, falling back to text-only extraction" + ) + + # If image extraction is not needed or conversion failed, try using antiword to extract text + try: + logger.info("Attempting to parse DOC file with antiword") + # Check if antiword is installed + antiword_path = self._find_antiword_path() + + if antiword_path: + # Use antiword to extract text directly + logger.info(f"Using antiword at {antiword_path} to extract text") + process = subprocess.Popen( + [antiword_path, temp_file_path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = process.communicate() + + if process.returncode == 0: + text = stdout.decode("utf-8", errors="ignore") + logger.info( + f"Successfully extracted {len(text)} characters using antiword" + ) + + # Clean up temporary file + os.unlink(temp_file_path) + logger.info(f"Deleted temporary file: {temp_file_path}") + + return text + else: + logger.warning( + f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}" + ) + else: + logger.warning("antiword not found, falling back to textract") + except Exception as e: + logger.warning( + f"Error using antiword: {str(e)}, falling back to textract" + ) + + # If antiword fails, try using textract + logger.info("Parsing DOC file with textract") + text = textract.process(temp_file_path, method="antiword").decode("utf-8") + logger.info( + f"Successfully extracted {len(text)} characters of text from DOC document using textract" + ) + + # Clean up temporary file + os.unlink(temp_file_path) + logger.info(f"Deleted temporary file: {temp_file_path}") + + return text + except Exception as e: + logger.error(f"Error parsing DOC document: {str(e)}") + # Ensure temporary file is cleaned up + if os.path.exists(temp_file_path): + os.unlink(temp_file_path) + logger.info(f"Deleted temporary file after error: {temp_file_path}") + return "" + + def _convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]: + """Convert DOC file to DOCX format + + Uses LibreOffice/OpenOffice for conversion + + Args: + doc_path: DOC file path + + Returns: + Byte stream of DOCX file content, or None if conversion fails + """ + logger.info(f"Converting DOC to DOCX: {doc_path}") + + # Create a temporary directory to store the converted file + temp_dir = tempfile.mkdtemp() + docx_path = os.path.join(temp_dir, "converted.docx") + + try: + # Check if LibreOffice or OpenOffice is installed + soffice_path = self._find_soffice_path() + if not soffice_path: + logger.error( + "LibreOffice/OpenOffice not found, cannot convert DOC to DOCX" + ) + return None + + # Execute conversion command + logger.info(f"Using {soffice_path} to convert DOC to DOCX") + cmd = [ + soffice_path, + "--headless", + "--convert-to", + "docx", + "--outdir", + temp_dir, + doc_path, + ] + + logger.info(f"Running command: {' '.join(cmd)}") + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = process.communicate() + + if process.returncode != 0: + logger.error( + f"Error converting DOC to DOCX: {stderr.decode('utf-8', errors='ignore')}" + ) + return None + + # Find the converted file + for file in os.listdir(temp_dir): + if file.endswith(".docx"): + converted_file = os.path.join(temp_dir, file) + logger.info(f"Found converted file: {converted_file}") + + # Read the converted file content + with open(converted_file, "rb") as f: + docx_content = f.read() + + logger.info( + f"Successfully read converted DOCX file, size: {len(docx_content)} bytes" + ) + return docx_content + + logger.error("No DOCX file found after conversion") + return None + + except Exception as e: + logger.error(f"Error during DOC to DOCX conversion: {str(e)}") + return None + finally: + # Clean up temporary directory + try: + shutil.rmtree(temp_dir) + logger.info(f"Cleaned up temporary directory: {temp_dir}") + except Exception as e: + logger.warning(f"Failed to clean up temporary directory: {str(e)}") + + def _find_soffice_path(self) -> Optional[str]: + """Find LibreOffice/OpenOffice executable path + + Returns: + Executable path, or None if not found + """ + # Common LibreOffice/OpenOffice executable paths + possible_paths = [ + # Linux + "/usr/bin/soffice", + "/usr/lib/libreoffice/program/soffice", + "/opt/libreoffice25.2/program/soffice", + # macOS + "/Applications/LibreOffice.app/Contents/MacOS/soffice", + # Windows + "C:\\Program Files\\LibreOffice\\program\\soffice.exe", + "C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe", + ] + + # Check if path is set in environment variable + if os.environ.get("LIBREOFFICE_PATH"): + possible_paths.insert(0, os.environ.get("LIBREOFFICE_PATH")) + + for path in possible_paths: + if os.path.exists(path): + logger.info(f"Found LibreOffice/OpenOffice at: {path}") + return path + + # Try to find in PATH + try: + result = subprocess.run( + ["which", "soffice"], capture_output=True, text=True + ) + if result.returncode == 0 and result.stdout.strip(): + path = result.stdout.strip() + logger.info(f"Found LibreOffice/OpenOffice in PATH: {path}") + return path + except Exception: + pass + + logger.warning("LibreOffice/OpenOffice not found") + return None + + def _find_antiword_path(self) -> Optional[str]: + """Find antiword executable path + + Returns: + Executable path, or None if not found + """ + # Common antiword executable paths + possible_paths = [ + # Linux/macOS + "/usr/bin/antiword", + "/usr/local/bin/antiword", + # Windows + "C:\\Program Files\\Antiword\\antiword.exe", + "C:\\Program Files (x86)\\Antiword\\antiword.exe", + ] + + # Check if path is set in environment variable + if os.environ.get("ANTIWORD_PATH"): + possible_paths.insert(0, os.environ.get("ANTIWORD_PATH")) + + for path in possible_paths: + if os.path.exists(path): + logger.info(f"Found antiword at: {path}") + return path + + # Try to find in PATH + try: + result = subprocess.run( + ["which", "antiword"], capture_output=True, text=True + ) + if result.returncode == 0 and result.stdout.strip(): + path = result.stdout.strip() + logger.info(f"Found antiword in PATH: {path}") + return path + except Exception: + pass + + logger.warning("antiword not found") + return None + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + logger.info("Running DocParser in standalone mode") + + file_name = "/path/to/your/test.doc" + logger.info(f"Processing file: {file_name}") + + doc_parser = DocParser( + file_name, enable_multimodal=True, chunk_size=512, chunk_overlap=60 + ) + logger.info("Parser initialized, starting processing") + + with open(file_name, "rb") as f: + content = f.read() + + text = doc_parser.parse_into_text(content) + logger.info(f"Processing complete, extracted text length: {len(text)}") + logger.info(f"Sample text: {text[:200]}...") diff --git a/services/docreader/src/parser/docx_parser.py b/services/docreader/src/parser/docx_parser.py new file mode 100644 index 0000000..ee0cb90 --- /dev/null +++ b/services/docreader/src/parser/docx_parser.py @@ -0,0 +1,1489 @@ +import logging +import tempfile +import os +import sys +import time +from io import BytesIO +from typing import Optional, Dict, Any, Tuple, List, Union +from dataclasses import dataclass, field +from PIL import Image +from docx import Document +from docx.image.exceptions import ( + UnrecognizedImageError, + UnexpectedEndOfFileError, + InvalidImageStreamError, +) +from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed +import tempfile +import threading +import traceback +from multiprocessing import Manager +import re + +from .base_parser import BaseParser + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# Add thread local storage to track the processing status of each thread +thread_local = threading.local() + + +class ImageData: + """Represents a processed image of document content""" + local_path: str = "" + object: Image.Image = None + url: str = "" + + +@dataclass +class LineData: + """Represents a processed line of document content with associated images""" + + text: str = "" # Extracted text content + images: List[ImageData] = field(default_factory=list) # List of images or image paths + extra_info: str = "" # Placeholder for additional info (currently unused) + page_num: int = 0 # Page number + content_sequence: List[Tuple[str, Any]] = field( + default_factory=list + ) # Sequence of content items (text/images) + + +class DocxParser(BaseParser): + """DOCX document parser""" + + def __init__( + self, + file_name: str = "", + file_type: str = None, + enable_multimodal: bool = True, + chunk_size: int = 1000, + chunk_overlap: int = 200, + separators: list = ["\n\n", "\n", "。"], + ocr_backend: str = "paddle", + ocr_config: dict = None, + max_image_size: int = 1920, + max_concurrent_tasks: int = 5, + max_pages: int = 100, # Maximum number of pages to process, default to 50 pages + chunking_config=None, + ): + """Initialize DOCX document parser + + Args: + file_name: File name + file_type: File type, if None, infer from file name + enable_multimodal: Whether to enable multimodal processing + chunk_size: Chunk size + chunk_overlap: Chunk overlap + separators: List of separators + ocr_backend: OCR engine type + ocr_config: OCR engine configuration + max_image_size: Maximum image size limit + max_concurrent_tasks: Maximum number of concurrent tasks + max_pages: Maximum number of pages to process, if more than this, only process the first max_pages pages + """ + super().__init__( + file_name=file_name, + file_type=file_type, + enable_multimodal=enable_multimodal, + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + separators=separators, + ocr_backend=ocr_backend, + ocr_config=ocr_config, + max_image_size=max_image_size, + max_concurrent_tasks=max_concurrent_tasks, + chunking_config=chunking_config, + ) + self.max_pages = max_pages + logger.info(f"DocxParser initialized with max_pages={max_pages}") + + def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]: + """Parse DOCX document, extract text content and image Markdown links + + Args: + content: DOCX document content + + Returns: + Tuple of (parsed_text, image_map) where image_map maps image URLs to Image objects + All LineData objects are used internally but not returned directly through this interface + """ + logger.info(f"Parsing DOCX document, content size: {len(content)} bytes") + logger.info(f"Max pages limit set to: {self.max_pages}") + logger.info("Converting DOCX content to sections and tables") + + start_time = time.time() + # Use concurrent processing to handle the document + max_workers = min( + 4, os.cpu_count() or 2 + ) # Reduce thread count to avoid excessive memory consumption + logger.info(f"Setting max_workers to {max_workers} for document processing") + + try: + logger.info(f"Starting Docx processing with max_pages={self.max_pages}") + docx_processor = Docx( + max_image_size=self.max_image_size, + enable_multimodal=self.enable_multimodal, + upload_file=self.upload_file, + ) + all_lines, tables = docx_processor( + binary=content, + max_workers=max_workers, + to_page=self.max_pages, + ) + processing_time = time.time() - start_time + logger.info( + f"Docx processing completed in {processing_time:.2f}s, " + f"extracted {len(all_lines)} sections and {len(tables)} tables" + ) + + logger.info("Processing document sections") + section_start_time = time.time() + + text_parts = [] + image_parts = {} + + for sec_idx, line in enumerate(all_lines): + try: + if line.text is not None and line.text != "": + text_parts.append(line.text) + if sec_idx < 3 or sec_idx % 50 == 0: + logger.info( + f"Added section {sec_idx+1} text: {line.text[:50]}..." + if len(line.text) > 50 + else f"Added section {sec_idx+1} text: {line.text}" + ) + if line.images: + for image_data in line.images: + if image_data.url: + image_parts[image_data.url] = image_data.object + except Exception as e: + logger.error(f"Error processing section {sec_idx+1}: {str(e)}") + logger.error(f"Detailed stack trace: {traceback.format_exc()}") + continue + + # Combine text + section_processing_time = time.time() - section_start_time + logger.info( + f"Section processing completed in {section_processing_time:.2f}s" + ) + logger.info("Combining all text parts") + text = "\n\n".join([part for part in text_parts if part]) + + # Check if the generated text is empty + if not text: + logger.warning("Generated text is empty, trying alternative method") + return self._parse_using_simple_method(content) + + total_processing_time = time.time() - start_time + logger.info( + f"Parsing complete in {total_processing_time:.2f}s, generated {len(text)} characters of text " + ) + + return text, image_parts + except Exception as e: + logger.error(f"Error parsing DOCX document: {str(e)}") + logger.error(f"Detailed stack trace: {traceback.format_exc()}") + fallback_text = self._parse_using_simple_method(content) + return fallback_text, {} + + def _parse_using_simple_method(self, content: bytes) -> str: + """Parse document using a simplified method, as a fallback + + Args: + content: Document content + + Returns: + Parsed text + """ + logger.info("Attempting to parse document using simplified method") + start_time = time.time() + try: + doc = Document(BytesIO(content)) + logger.info( + f"Successfully loaded document in simplified method, " + f"contains {len(doc.paragraphs)} paragraphs and {len(doc.tables)} tables" + ) + text_parts = [] + + # Extract paragraph text + para_count = len(doc.paragraphs) + logger.info(f"Extracting text from {para_count} paragraphs") + para_with_text = 0 + for i, para in enumerate(doc.paragraphs): + if i % 100 == 0: + logger.info(f"Processing paragraph {i+1}/{para_count}") + if para.text.strip(): + text_parts.append(para.text.strip()) + para_with_text += 1 + + logger.info(f"Extracted text from {para_with_text}/{para_count} paragraphs") + + # Extract table text + table_count = len(doc.tables) + logger.info(f"Extracting text from {table_count} tables") + tables_with_content = 0 + rows_processed = 0 + for i, table in enumerate(doc.tables): + if i % 10 == 0: + logger.info(f"Processing table {i+1}/{table_count}") + + table_has_content = False + for row in table.rows: + rows_processed += 1 + row_text = " | ".join( + [cell.text.strip() for cell in row.cells if cell.text.strip()] + ) + if row_text: + text_parts.append(row_text) + table_has_content = True + + if table_has_content: + tables_with_content += 1 + + logger.info( + f"Extracted content from {tables_with_content}/{table_count} tables, " + f"processed {rows_processed} rows" + ) + + # Combine text + result_text = "\n\n".join(text_parts) + processing_time = time.time() - start_time + logger.info( + f"Simplified parsing complete in {processing_time:.2f}s, " + f"generated {len(result_text)} characters of text" + ) + + # If the result is still empty, return an error message + if not result_text: + logger.warning("No text extracted using simplified method") + return "", {} + + return result_text, {} + except Exception as backup_error: + processing_time = time.time() - start_time + logger.error( + f"Simplified parsing failed after {processing_time:.2f}s: {str(backup_error)}" + ) + logger.error(f"Detailed traceback: {traceback.format_exc()}") + return "", {} + + +class Docx: + def __init__(self, max_image_size=1920, enable_multimodal=False, upload_file=None): + logger.info("Initializing DOCX processor") + self.max_image_size = max_image_size # Maximum image size limit + self.picture_cache = ( + {} + ) # Image cache to avoid processing the same image repeatedly + self.enable_multimodal = enable_multimodal + self.upload_file = upload_file + + def get_picture(self, document, paragraph) -> Optional[Image.Image]: + logger.info("Extracting image from paragraph") + img = paragraph._element.xpath(".//pic:pic") + if not img: + logger.info("No image found in paragraph") + return None + img = img[0] + try: + embed = img.xpath(".//a:blip/@r:embed")[0] + related_part = document.part.related_parts[embed] + logger.info(f"Found embedded image with ID: {embed}") + + try: + image_blob = related_part.image.blob + except UnrecognizedImageError: + logger.warning("Unrecognized image format. Skipping image.") + return None + except UnexpectedEndOfFileError: + logger.warning( + "EOF was unexpectedly encountered while reading an image stream. Skipping image." + ) + return None + except InvalidImageStreamError: + logger.warning( + "The recognized image stream appears to be corrupted. Skipping image." + ) + return None + + try: + logger.info("Converting image blob to PIL Image") + image = Image.open(BytesIO(image_blob)).convert("RGBA") + logger.info( + f"Successfully extracted image, size: {image.width}x{image.height}" + ) + return image + except Exception as e: + logger.error(f"Failed to open image: {str(e)}") + return None + except Exception as e: + logger.error(f"Error extracting image: {str(e)}") + return None + + def _identify_page_paragraph_mapping(self, max_page=100000): + """Identify the paragraph range included on each page + + Args: + max_page: Maximum number of pages to process + + Returns: + dict: Mapping of page numbers to lists of paragraph indices + """ + start_time = time.time() + logger.info(f"Identifying page to paragraph mapping (max_page={max_page})") + page_to_paragraphs = {} + current_page = 0 + + # Initialize page 0 + page_to_paragraphs[current_page] = [] + + # Record the total number of paragraphs processed + total_paragraphs = len(self.doc.paragraphs) + logger.info(f"Total paragraphs to map: {total_paragraphs}") + + # Heuristic method: estimate the number of paragraphs per page + # For large documents, using a heuristic can reduce XML parsing overhead + if total_paragraphs > 1000: + logger.info("Large document detected, using heuristic paragraph mapping") + estimated_paras_per_page = ( + 25 # Estimate approximately 25 paragraphs per page + ) + + # Create an estimated page mapping + for p_idx in range(total_paragraphs): + est_page = p_idx // estimated_paras_per_page + if est_page > max_page: + logger.info( + f"Reached max page limit ({max_page}) at paragraph {p_idx}, stopping paragraph mapping" + ) + break + + if est_page not in page_to_paragraphs: + page_to_paragraphs[est_page] = [] + + page_to_paragraphs[est_page].append(p_idx) + + if p_idx > 0 and p_idx % 1000 == 0: + logger.info( + f"Heuristic mapping: processed {p_idx}/{total_paragraphs} paragraphs" + ) + + mapping_time = time.time() - start_time + logger.info( + f"Created heuristic mapping with {len(page_to_paragraphs)} pages in {mapping_time:.2f}s" + ) + return page_to_paragraphs + + # Standard method: iterate through all paragraphs to find page breaks + logger.info("Using standard paragraph mapping method") + page_breaks_found = 0 + for p_idx, p in enumerate(self.doc.paragraphs): + # Add the current paragraph to the current page + page_to_paragraphs[current_page].append(p_idx) + + # Log every 100 paragraphs + if p_idx > 0 and p_idx % 100 == 0: + logger.info( + f"Processed {p_idx}/{total_paragraphs} paragraphs in page mapping" + ) + + # Check for page breaks + page_break_found = False + + # Method 1: Check for lastRenderedPageBreak + for run in p.runs: + if "lastRenderedPageBreak" in run._element.xml: + page_break_found = True + break + + if "w:br" in run._element.xml and 'type="page"' in run._element.xml: + page_break_found = True + break + + # Method 2: Check sectPr element (section break, usually indicates a new page) + if not page_break_found and p._element.xpath(".//w:sectPr"): + page_break_found = True + + # If a page break is found, create a new page + if page_break_found: + page_breaks_found += 1 + current_page += 1 + if current_page > max_page: + logger.info( + f"Reached max page limit ({max_page}), stopping page mapping" + ) + break + + # Initialize the paragraph list for the new page + if current_page not in page_to_paragraphs: + page_to_paragraphs[current_page] = [] + + if page_breaks_found % 10 == 0: + logger.info( + f"Found {page_breaks_found} page breaks so far, current page: {current_page}" + ) + + # Handle potential empty page mappings + empty_pages = [page for page, paras in page_to_paragraphs.items() if not paras] + if empty_pages: + logger.info(f"Removing {len(empty_pages)} empty pages from mapping") + for page in empty_pages: + del page_to_paragraphs[page] + + mapping_time = time.time() - start_time + logger.info( + f"Created paragraph mapping with {len(page_to_paragraphs)} pages in {mapping_time:.2f}s" + ) + + # Check the validity of the result + if not page_to_paragraphs: + logger.warning("No valid page mapping created, using fallback method") + # All paragraphs are on page 0 + page_to_paragraphs[0] = list(range(total_paragraphs)) + + # Log page distribution statistics + page_sizes = [len(paragraphs) for paragraphs in page_to_paragraphs.values()] + if page_sizes: + avg_paragraphs = sum(page_sizes) / len(page_sizes) + min_paragraphs = min(page_sizes) + max_paragraphs = max(page_sizes) + logger.info( + f"Page statistics: avg={avg_paragraphs:.1f}, " + f"min={min_paragraphs}, max={max_paragraphs} paragraphs per page" + ) + + return page_to_paragraphs + + + def __call__( + self, + binary: Optional[bytes] = None, + from_page: int = 0, + to_page: int = 100000, + max_workers: Optional[int] = None, + ) -> Tuple[List[LineData], List[Any]]: + """ + Process DOCX document, supporting concurrent processing of each page + + Args: + binary: DOCX document binary content + from_page: Starting page number + to_page: Ending page number + max_workers: Maximum number of workers, default to None (system decides) + + Returns: + tuple: (List of LineData objects with document content, List of tables) + """ + logger.info("Processing DOCX document") + + # Check CPU core count to determine parallel strategy + cpu_count = os.cpu_count() or 2 + logger.info(f"System has {cpu_count} CPU cores available") + + # Load document + self.doc = self._load_document(binary) + if not self.doc: + return [], [] + + # Identify page structure + self.para_page_mapping = self._identify_page_paragraph_mapping(to_page) + logger.info( + f"Identified page to paragraph mapping for {len(self.para_page_mapping)} pages" + ) + + # Apply page limits + pages_to_process = self._apply_page_limit( + self.para_page_mapping, from_page, to_page + ) + if not pages_to_process: + logger.warning("No pages to process after applying page limits!") + return [], [] + + # Initialize shared resources + self._init_shared_resources() + + # Process document content + self._process_document( + binary, + pages_to_process, + from_page, + to_page, + max_workers, + ) + + # Process tables + tbls = self._process_tables() + + # Clean up document resources + self.doc = None + + logger.info( + f"Document processing complete, " + f"extracted {len(self.all_lines)} text sections and {len(tbls)} tables" + ) + return self.all_lines, tbls + + def _load_document(self, binary): + """Load document + + Args: + binary: Document binary content + + Returns: + Document: Document object, or None (if loading fails) + """ + try: + doc = Document(BytesIO(binary)) + logger.info("Successfully loaded document from binary content") + return doc + except Exception as e: + logger.error(f"Failed to load DOCX document: {str(e)}") + return None + + def _init_shared_resources(self): + """Initialize shared resources""" + # Create shared resource locks to protect data structures shared between threads + self.lines_lock = threading.Lock() + + # Initialize result containers + self.all_lines = [] + + def _get_request_id(self): + """Get current request ID""" + current_request_id = None + try: + from utils.request import get_request_id + + current_request_id = get_request_id() + logger.info( + f"Getting current request ID: {current_request_id} to pass to processing threads" + ) + except Exception as e: + logger.warning(f"Failed to get current request ID: {str(e)}") + return current_request_id + + def _apply_page_limit(self, para_page_mapping, from_page, to_page): + """Apply page limits, return the list of pages to process + + Args: + para_page_mapping: Mapping of pages to paragraphs + from_page: Starting page number + to_page: Ending page number + + Returns: + list: List of pages to process + """ + # Add page limits + total_pages = len(para_page_mapping) + if total_pages > to_page: + logger.info( + f"Document has {total_pages} pages, limiting processing to first {to_page} pages" + ) + logger.info(f"Setting to_page limit to {to_page}") + else: + logger.info( + f"Document has {total_pages} pages, processing all pages (limit: {to_page})" + ) + + # Filter out pages outside the range + all_pages = sorted(para_page_mapping.keys()) + pages_to_process = [p for p in all_pages if from_page <= p < to_page] + + # Output the actual number of pages processed for debugging + if pages_to_process: + logger.info( + f"Will process {len(pages_to_process)} pages " + f"from page {from_page} to page {min(to_page, pages_to_process[-1] if pages_to_process else from_page)}" + ) + + if len(pages_to_process) < len(all_pages): + logger.info( + f"Skipping {len(all_pages) - len(pages_to_process)} pages due to page limit" + ) + + # Log detailed page index information + if len(pages_to_process) <= 10: + logger.info(f"Pages to process: {pages_to_process}") + else: + logger.info( + f"First 5 pages to process: {pages_to_process[:5]}, last 5: {pages_to_process[-5:]}" + ) + + return pages_to_process + + + def _process_document( + self, + binary, + pages_to_process, + from_page, + to_page, + max_workers, + ): + """Process large documents, using multiprocessing + + Args: + binary: Document binary content + pages_to_process: List of pages to process + from_page: Starting page number + to_page: Ending page number + max_workers: Maximum number of workers + """ + # If the number of pages is too large, process in batches to reduce memory consumption + cpu_count = os.cpu_count() or 2 + + # Check if the document contains images to optimize processing speed + doc_contains_images = self._check_document_has_images() + + # Optimize process count: dynamically adjust based on number of pages and CPU cores + if max_workers is None: + max_workers = self._calculate_optimal_workers( + doc_contains_images, pages_to_process, cpu_count + ) + + temp_file_path = self._prepare_document_sharing(binary) + + # Prepare multiprocess processing arguments + args_list = self._prepare_multiprocess_args( + pages_to_process, + from_page, + to_page, + doc_contains_images, + temp_file_path, + ) + + # Execute multiprocess tasks + self._execute_multiprocess_tasks(args_list, max_workers) + + # Clean up temporary file + self._cleanup_temp_file(temp_file_path) + + def _check_document_has_images(self): + """Check if the document contains images + + Returns: + bool: Whether the document contains images + """ + doc_contains_images = False + if hasattr(self.doc, "inline_shapes") and len(self.doc.inline_shapes) > 0: + doc_contains_images = True + logger.info( + f"Document contains {len(self.doc.inline_shapes)} inline images" + ) + return doc_contains_images + + def _calculate_optimal_workers( + self, doc_contains_images, pages_to_process, cpu_count + ): + """Calculate the optimal number of workers + + Args: + doc_contains_images: Whether the document contains images + pages_to_process: List of pages to process + cpu_count: Number of CPU cores + + Returns: + int: Optimal number of workers + """ + # If no images or few pages, use fewer processes to avoid overhead + if not doc_contains_images or len(pages_to_process) < cpu_count: + max_workers = min(len(pages_to_process), max(1, cpu_count - 1)) + else: + max_workers = min(len(pages_to_process), cpu_count) + logger.info(f"Automatically set worker count to {max_workers}") + return max_workers + + def _prepare_document_sharing(self, binary): + """Prepare document sharing method + + Args: + binary: Document binary content + + Returns: + str: Temporary file path, or None if not using + """ + + temp_file = tempfile.NamedTemporaryFile(delete=False) + temp_file_path = temp_file.name + temp_file.write(binary) + temp_file.close() + return temp_file_path + + def _prepare_multiprocess_args( + self, + pages_to_process, + from_page, + to_page, + doc_contains_images, + temp_file_path, + ): + """Prepare a list of arguments for multiprocess processing + + Args: + pages_to_process: List of pages to process + from_page: Starting page number + to_page: Ending page number + doc_contains_images: Whether the document contains images + temp_file_path: Temporary file path + + Returns: + list: List of arguments + """ + args_list = [] + for page_num in pages_to_process: + args_list.append( + ( + page_num, + self.para_page_mapping[page_num], + from_page, + to_page, + doc_contains_images, + self.max_image_size, + temp_file_path, + self.enable_multimodal, + ) + ) + + return args_list + + def _execute_multiprocess_tasks(self, args_list, max_workers): + """Execute multiprocess tasks + + Args: + args_list: List of arguments + max_workers: Maximum number of workers + """ + # Use a shared manager to share data + with Manager() as manager: + # Create shared data structures + self.all_lines = manager.list() + + logger.info( + f"Processing {len(args_list)} pages using {max_workers} processes" + ) + + # Use ProcessPoolExecutor to truly implement multi-core parallelization + batch_start_time = time.time() + with ProcessPoolExecutor(max_workers=max_workers) as executor: + logger.info(f"Started ProcessPoolExecutor with {max_workers} workers") + + # Submit all tasks + future_to_idx = { + executor.submit(process_page_multiprocess, *args): i + for i, args in enumerate(args_list) + } + logger.info( + f"Submitted {len(future_to_idx)} processing tasks to process pool" + ) + + # Collect results + self._collect_process_results( + future_to_idx, args_list, batch_start_time + ) + + def _collect_process_results(self, future_to_idx, args_list, batch_start_time): + """Collect multiprocess processing results + + Args: + future_to_idx: Mapping of Future to index + args_list: List of arguments + batch_start_time: Batch start time + + Returns: + List[LineData]: Processed results as LineData objects + """ + # Collect results + completed_count = 0 + results = [] + temp_img_paths = set() # Collect all temporary image paths + + for future in as_completed(future_to_idx): + idx = future_to_idx[future] + page_num = args_list[idx][0] + try: + page_lines = future.result() + + # Collect temporary image paths for later cleanup + for line in page_lines: + for image_data in line.images: + if image_data.local_path and image_data.local_path.startswith("/tmp/docx_img_"): + temp_img_paths.add(image_data.local_path) + + results.extend(page_lines) + completed_count += 1 + + if completed_count % max( + 1, len(args_list) // 10 + ) == 0 or completed_count == len(args_list): + elapsed_ms = int((time.time() - batch_start_time) * 1000) + progress_pct = int((completed_count / len(args_list)) * 100) + logger.info( + f"Progress: {completed_count}/{len(args_list)} pages processed " + f"({progress_pct}%, elapsed: {elapsed_ms}ms)" + ) + + except Exception as e: + logger.error(f"Error processing page {page_num}: {str(e)}") + logger.error( + f"Detailed traceback for page {page_num}: {traceback.format_exc()}" + ) + + # Process completion + processing_elapsed_ms = int((time.time() - batch_start_time) * 1000) + logger.info(f"All processing completed in {processing_elapsed_ms}ms") + + # Process results + self._process_multiprocess_results(results) + + # Clean up temporary image files + self._cleanup_temp_image_files(temp_img_paths) + + def _process_multiprocess_results(self, results: List[LineData]): + """Process multiprocess results + + Args: + results: List of processed LineData results + """ + lines = list(results) + + # Process images - must be handled in the main process for upload + # If images are being processed, they need to be handled in the main process for upload + image_upload_start = time.time() + + # Count total images to process + images_to_process = [] + processed_lines = [] + for i, line_data in enumerate(lines): + # Check if there are images + if line_data.images and len(line_data.images) > 0: + images_to_process.append(i) + logger.info( + f"Found line {i} with {len(line_data.images)} images to process" + ) + + # Process images if needed + image_url_map = {} # Map from image path to URL + if images_to_process: + logger.info( + f"Found {len(images_to_process)} lines with images to process in main process" + ) + + # First, create a mapping of image paths to uploaded URLs + for line_idx in images_to_process: + line_data = lines[line_idx] + image_paths = line_data.images + page_num = line_data.page_num + + # Process all image data objects + for image_data in image_paths: + if image_data.local_path and os.path.exists(image_data.local_path) and image_data.local_path not in image_url_map: + try: + # Upload the image if it doesn't have a URL yet + if not image_data.url: + image_url = self.upload_file(image_data.local_path) + if image_url: + # Store the URL in the ImageData object + image_data.url = image_url + # Add image URL as Markdown format + markdown_image = f"![]({image_url})" + image_url_map[image_data.local_path] = markdown_image + logger.info( + f"Added image URL for {image_data.local_path}: {image_url}" + ) + else: + logger.warning(f"Failed to upload image: {image_data.local_path}") + else: + # Already has a URL, use it + markdown_image = f"![]({image_data.url})" + image_url_map[image_data.local_path] = markdown_image + logger.info( + f"Using existing URL for image {image_data.local_path}: {image_data.url}" + ) + except Exception as e: + logger.error( + f"Error processing image from page {page_num}: {str(e)}" + ) + + image_upload_elapsed = time.time() - image_upload_start + logger.info( + f"Finished uploading {len(image_url_map)} images in {image_upload_elapsed:.2f}s" + ) + + # Process content in original sequence order + for line_data in lines: + processed_content = [] + if line_data.content_sequence: # Check if we have processed_content + processed_content = line_data.content_sequence + page_num = line_data.page_num + + # Reconstruct text with images in original positions + combined_parts = [] + for content_type, content in processed_content: + if content_type == "text": + combined_parts.append(content) + elif content_type == "image": + # For ImageData objects, use the URL + if isinstance(content, str) and content in image_url_map: + combined_parts.append(image_url_map[content]) + elif hasattr(content, 'local_path') and content.local_path in image_url_map: + combined_parts.append(image_url_map[content.local_path]) + + # Create the final text with proper ordering + final_text = "\n\n".join(part for part in combined_parts if part) + processed_lines.append(LineData(text=final_text, page_num=page_num, images=line_data.images)) + else: + processed_lines = lines + + # Sort results by page number + sorted_lines = sorted(processed_lines, key=lambda x: x.page_num) + self.all_lines = sorted_lines + + logger.info( + f"Finished processing {len(self.all_lines)} lines with interleaved images and text" + ) + + def _cleanup_temp_image_files(self, temp_paths): + """Clean up temporary image files created by multiprocessing + + Args: + temp_paths: Set of temporary file paths + """ + if not temp_paths: + return + + logger.info(f"Cleaning up {len(temp_paths)} temporary image files") + deleted_count = 0 + error_count = 0 + + for path in temp_paths: + try: + if os.path.exists(path): + os.unlink(path) + deleted_count += 1 + # Delete temporary directory (if empty) + try: + temp_dir = os.path.dirname(path) + if temp_dir.startswith("/tmp/docx_img_") and os.path.exists( + temp_dir + ): + os.rmdir(temp_dir) + except OSError: + # If directory is not empty, ignore error + pass + except Exception as e: + logger.error(f"Failed to delete temp file {path}: {str(e)}") + error_count += 1 + + logger.info( + f"Temporary file cleanup: deleted {deleted_count}, errors {error_count}" + ) + + def _cleanup_temp_file(self, temp_file_path): + """Clean up temporary file + + Args: + temp_file_path: Temporary file path + """ + if temp_file_path and os.path.exists(temp_file_path): + try: + os.unlink(temp_file_path) + logger.info(f"Removed temporary file: {temp_file_path}") + except Exception as e: + logger.error(f"Failed to remove temporary file: {str(e)}") + + def _process_tables(self): + """Process tables in the document + + Returns: + list: List of tables + """ + tbls = [] + table_count = len(self.doc.tables) + if table_count > 0: + logger.info(f"Processing {table_count} tables") + for tb_idx, tb in enumerate(self.doc.tables): + if tb_idx % 10 == 0: # Log only every 10 tables to reduce log volume + logger.info(f"Processing table {tb_idx+1}/{table_count}") + + # Optimize: Check if table is empty + if len(tb.rows) == 0 or all(len(r.cells) == 0 for r in tb.rows): + logger.info(f"Skipping empty table {tb_idx+1}") + continue + + table_html = self._convert_table_to_html(tb) + # Still using tuple format for tables as they are handled differently + tbls.append(((None, table_html), "")) + + return tbls + + def _convert_table_to_html(self, table): + """Convert table to HTML + + Args: + table: Table object + + Returns: + str: HTML formatted table + """ + html = "" + for r in table.rows: + html += "" + i = 0 + while i < len(r.cells): + span = 1 + c = r.cells[i] + for j in range(i + 1, len(r.cells)): + if c.text == r.cells[j].text: + span += 1 + i = j + i += 1 + html += ( + f"" + if span == 1 + else f"" + ) + html += "" + html += "
{c.text}{c.text}
" + return html + + def _safe_concat_images(self, images): + """Safely concatenate image lists + + Args: + images: List of images + + Returns: + Image: Concatenated image, or the first image (if concatenation fails) + """ + if not images: + return None + + if len(images) == 1: + return images[0] + + try: + logger.info(f"Attempting to concatenate {len(images)} images") + from PIL import Image + + # Calculate the size of the concatenated image + total_width = max(img.width for img in images if hasattr(img, "width")) + total_height = sum(img.height for img in images if hasattr(img, "height")) + + if total_width <= 0 or total_height <= 0: + logger.warning("Invalid image size, returning the first image") + return images[0] + + # Create a new image + new_image = Image.new("RGBA", (total_width, total_height), (0, 0, 0, 0)) + + # Paste images one by one + y_offset = 0 + for img in images: + if not hasattr(img, "width") or not hasattr(img, "height"): + continue + + new_image.paste(img, (0, y_offset)) + y_offset += img.height + + logger.info( + f"Successfully concatenated images, final size: {total_width}x{total_height}" + ) + return new_image + except Exception as e: + logger.error(f"Failed to concatenate images: {str(e)}") + logger.error(f"Detailed error: {traceback.format_exc()}") + # If concatenation fails, return the first image + return images[0] + + +def _save_image_to_temp(logger, image, page_num, img_idx): + """Save image to a temporary file to pass between processes + + Args: + logger: Logger + image: PIL image object + page_num: Page number + img_idx: Image index + + Returns: + str: Temporary file path, or None (if saving fails) + """ + if not image: + return None + + import tempfile + import os + + try: + # Create a temporary file + temp_dir = tempfile.mkdtemp(prefix="docx_img_") + temp_file_path = os.path.join(temp_dir, f"page_{page_num}_img_{img_idx}.png") + + # Save the image + image.save(temp_file_path, format="PNG") + logger.info( + f"[PID:{os.getpid()}] Saved image to temporary file: {temp_file_path}" + ) + + return temp_file_path + except Exception as e: + logger.error(f"[PID:{os.getpid()}] Failed to save image to temp file: {str(e)}") + return None + + +def process_page_multiprocess( + page_num: int, + paragraphs: List[int], + from_page: int, + to_page: int, + doc_contains_images: bool, + max_image_size: int, + temp_file_path: Optional[str], + enable_multimodal: bool, +) -> List[LineData]: + """Page processing function specifically designed for multiprocessing + + Args: + page_num: Page number + paragraphs: List of paragraph indices + from_page: Starting page number + to_page: Ending page number + doc_contains_images: Whether the document contains images + max_image_size: Maximum image size + doc_binary: Document binary content + temp_file_path: Temporary file path, if using + enable_multimodal: Whether to enable multimodal processing + + Returns: + list: List of processed result lines + """ + try: + # Set process-level logging + process_logger = logging.getLogger(__name__) + + # If outside processing range, do not process + if page_num < from_page or page_num >= to_page: + process_logger.info( + f"[PID:{os.getpid()}] Skipping page {page_num} (out of requested range)" + ) + return [] + + process_logger.info( + f"[PID:{os.getpid()}] Processing page {page_num} with {len(paragraphs)} paragraphs, " + f"enable_multimodal={enable_multimodal}" + ) + start_time = time.time() + + # Load document in the process + doc = _load_document_in_process(process_logger, page_num, temp_file_path) + if not doc: + return [] + + # If paragraph indices are empty, return empty result + if not paragraphs: + process_logger.info( + f"[PID:{os.getpid()}] No paragraphs to process for page {page_num}" + ) + return [] + + # Extract page content + combined_text, image_objects, content_sequence = _extract_page_content_in_process( + process_logger, doc, page_num, paragraphs, enable_multimodal, max_image_size + ) + + # Process content sequence to maintain order between processes + processed_content = [] + temp_image_index = 0 + image_data_list = [] + + if enable_multimodal: + # First pass: save all images to temporary files + for i, image_object in enumerate(image_objects): + img_path = _save_image_to_temp(process_logger, image_object, page_num, i) + if img_path: + # Create ImageData object + image_data = ImageData() + image_data.local_path = img_path + image_data.object = image_object + image_data_list.append(image_data) + + process_logger.info( + f"[PID:{os.getpid()}] Saved {len(image_data_list)} images to temp files for page {page_num}" + ) + + # Second pass: reconstruct the content sequence with image data objects + for content_type, content in content_sequence: + if content_type == "text": + processed_content.append(("text", content)) + else: # image + if temp_image_index < len(image_data_list): + processed_content.append( + ("image", image_data_list[temp_image_index]) + ) + temp_image_index += 1 + + # Create result line with the ordered content sequence + line_data = LineData( + text=combined_text, + images=image_data_list, + page_num=page_num, + content_sequence=processed_content, + ) + page_lines = [line_data] + + processing_time = time.time() - start_time + process_logger.info( + f"[PID:{os.getpid()}] Page {page_num} processing completed in {processing_time:.2f}s" + ) + + return page_lines + + except Exception as e: + process_logger = logging.getLogger(__name__) + process_logger.error( + f"[PID:{os.getpid()}] Error processing page {page_num}: {str(e)}" + ) + process_logger.error(f"[PID:{os.getpid()}] Traceback: {traceback.format_exc()}") + return [] + + +def _load_document_in_process(logger, page_num, temp_file_path): + """Load document in a process + + Args: + logger: Logger + page_num: Page number + temp_file_path: Temporary file path + + Returns: + Document: Loaded document object, or None (if loading fails) + """ + logger.info(f"[PID:{os.getpid()}] Loading document in process for page {page_num}") + try: + # Load document from temporary file + if temp_file_path is not None and os.path.exists(temp_file_path): + doc = Document(temp_file_path) + logger.info( + f"[PID:{os.getpid()}] Loaded document from temp file: {temp_file_path}" + ) + else: + logger.error(f"[PID:{os.getpid()}] No document source provided") + return None + return doc + + except Exception as e: + logger.error(f"[PID:{os.getpid()}] Failed to load document: {str(e)}") + logger.error(f"[PID:{os.getpid()}] Error traceback: {traceback.format_exc()}") + return None + + +def _extract_page_content_in_process( + logger, + doc, + page_num: int, + paragraphs: List[int], + enable_multimodal: bool, + max_image_size: int, +) -> Tuple[str, List[Any], List[Tuple[str, Any]]]: + """Extract page content in a process + + Args: + logger: Logger + doc: Document object + page_num: Page number + paragraphs: List of paragraph indices + enable_multimodal: Whether to enable multimodal processing + max_image_size: Maximum image size + + Returns: + tuple: (Extracted text, List of extracted images, Content sequence) + """ + logger.info( + f"[PID:{os.getpid()}] Page {page_num}: Processing {len(paragraphs)} paragraphs, " + f"enable_multimodal={enable_multimodal}" + ) + + # Instead of separate collections, track content in paragraph sequence + content_sequence = [] + current_text = "" + + processed_paragraphs = 0 + paragraphs_with_text = 0 + paragraphs_with_images = 0 + + for para_idx in paragraphs: + if para_idx >= len(doc.paragraphs): + logger.warning( + f"[PID:{os.getpid()}] Paragraph index {para_idx} out of range" + ) + continue + + paragraph = doc.paragraphs[para_idx] + processed_paragraphs += 1 + + # Extract text content + text = paragraph.text.strip() + if text: + # Clean text + cleaned_text = re.sub(r"\u3000", " ", text).strip() + current_text += cleaned_text + "\n" + paragraphs_with_text += 1 + + # Process image - if multimodal processing is enabled + if enable_multimodal: + image_object = _extract_image_in_process( + logger, doc, paragraph, page_num, para_idx, max_image_size + ) + if image_object: + # If we have accumulated text, add it to sequence first + if current_text: + content_sequence.append(("text", current_text)) + current_text = "" + + # Add image to sequence + content_sequence.append(("image", image_object)) + paragraphs_with_images += 1 + + if processed_paragraphs % 50 == 0: + logger.info( + f"[PID:{os.getpid()}] " + f"Page {page_num}: Processed {processed_paragraphs}/{len(paragraphs)} paragraphs" + ) + + # Add any remaining text + if current_text: + content_sequence.append(("text", current_text)) + + logger.info( + f"[PID:{os.getpid()}] Page {page_num}: Completed content extraction, " + f"found {paragraphs_with_text} paragraphs with text, " + f"{paragraphs_with_images} with images, " + f"total content items: {len(content_sequence)}" + ) + + # Extract text and images in their original sequence + text_parts = [] + images = [] + + # Split content sequence into text and images + for content_type, content in content_sequence: + if content_type == "text": + text_parts.append(content) + else: # image + images.append(content) + + combined_text = "\n\n".join(text_parts) if text_parts else "" + + return combined_text, images, content_sequence + + +def _extract_image_in_process( + logger, doc, paragraph, page_num, para_idx, max_image_size +): + """Extract image from a paragraph in a process + + Args: + logger: Logger + doc: Document object + paragraph: Paragraph object + page_num: Page number + para_idx: Paragraph index + max_image_size: Maximum image size + + Returns: + Image: Extracted image object, or None + """ + try: + # Attempt to extract image + img = paragraph._element.xpath(".//pic:pic") + if not img: + return None + + img = img[0] + logger.info( + f"[PID:{os.getpid()}] Page {page_num}: Found pic element in paragraph {para_idx}" + ) + + try: + # Extract image ID and related part + embed = img.xpath(".//a:blip/@r:embed") + if not embed: + logger.warning( + f"[PID:{os.getpid()}] Page {page_num}: No embed attribute found in image" + ) + return None + + embed = embed[0] + if embed not in doc.part.related_parts: + logger.warning( + f"[PID:{os.getpid()}] Page {page_num}: Embed ID {embed} not found in related parts" + ) + return None + + related_part = doc.part.related_parts[embed] + logger.info(f"[PID:{os.getpid()}] Found embedded image with ID: {embed}") + + # Attempt to get image data + try: + image_blob = related_part.image.blob + logger.info( + f"[PID:{os.getpid()}] Successfully extracted image blob, size: {len(image_blob)} bytes" + ) + except Exception as blob_error: + logger.warning( + f"[PID:{os.getpid()}] Error extracting image blob: {str(blob_error)}" + ) + return None + + # Convert data to PIL image + try: + image = Image.open(BytesIO(image_blob)).convert("RGBA") + + # Check image size + if hasattr(image, "width") and hasattr(image, "height"): + logger.info( + f"[PID:{os.getpid()}] Successfully created image object, " + f"size: {image.width}x{image.height}" + ) + + # Skip small images (usually decorative elements) + if image.width < 50 or image.height < 50: + logger.info( + f"[PID:{os.getpid()}] " + f"Skipping small image ({image.width}x{image.height})" + ) + return None + + # Scale large images + if image.width > max_image_size or image.height > max_image_size: + scale = min( + max_image_size / image.width, max_image_size / image.height + ) + new_width = int(image.width * scale) + new_height = int(image.height * scale) + resized_image = image.resize((new_width, new_height)) + logger.info( + f"[PID:{os.getpid()}] Resized image to {new_width}x{new_height}" + ) + return resized_image + + logger.info(f"[PID:{os.getpid()}] Found image in paragraph {para_idx}") + return image + except Exception as e: + logger.error( + f"[PID:{os.getpid()}] Failed to create image from blob: {str(e)}" + ) + logger.error( + f"[PID:{os.getpid()}] Error traceback: {traceback.format_exc()}" + ) + return None + except Exception as e: + logger.error(f"[PID:{os.getpid()}] Error extracting image: {str(e)}") + logger.error( + f"[PID:{os.getpid()}] Error traceback: {traceback.format_exc()}" + ) + return None + except Exception as e: + logger.error(f"[PID:{os.getpid()}] Error processing image: {str(e)}") + logger.error(f"[PID:{os.getpid()}] Error traceback: {traceback.format_exc()}") + return None diff --git a/services/docreader/src/parser/image_parser.py b/services/docreader/src/parser/image_parser.py new file mode 100644 index 0000000..4ebbcee --- /dev/null +++ b/services/docreader/src/parser/image_parser.py @@ -0,0 +1,68 @@ +import logging +import os +import asyncio +from PIL import Image +import io +from typing import Dict, Any, Tuple, Union +from .base_parser import BaseParser, ParseResult +import numpy as np + +# Set up logger for this module +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +class ImageParser(BaseParser): + """ + Parser for image files with OCR capability. + Extracts text from images and generates captions. + + This parser handles image processing by: + 1. Uploading the image to storage + 2. Generating a descriptive caption + 3. Performing OCR to extract text content + 4. Returning a combined result with both text and image reference + """ + + def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]: + """ + Parse image content, upload the image and return Markdown reference along with image map. + + Args: + content: Raw image data (bytes) + + Returns: + Tuple of (markdown_text, image_map) where image_map maps image URLs to PIL Image objects + """ + logger.info(f"Parsing image content, size: {len(content)} bytes") + image_map = {} + + try: + # Upload image to storage service + logger.info("Uploading image to storage") + _, ext = os.path.splitext(self.file_name) + image_url = self.upload_bytes(content, file_ext=ext) + if not image_url: + logger.error("Failed to upload image to storage") + return "", {} + logger.info( + f"Successfully uploaded image, URL: {image_url[:50]}..." + if len(image_url) > 50 + else f"Successfully uploaded image, URL: {image_url}" + ) + + # Create image object and add to map + try: + from PIL import Image + import io + image = Image.open(io.BytesIO(content)) + image_map[image_url] = image + logger.info(f"Added image to image_map for URL: {image_url}") + except Exception as img_err: + logger.error(f"Error creating image object: {str(img_err)}") + + markdown_text = f"![{self.file_name}]({image_url})" + return markdown_text, image_map + + except Exception as e: + logger.error(f"Error parsing image: {str(e)}") + return "", {} diff --git a/services/docreader/src/parser/image_utils.py b/services/docreader/src/parser/image_utils.py new file mode 100644 index 0000000..55cb474 --- /dev/null +++ b/services/docreader/src/parser/image_utils.py @@ -0,0 +1,43 @@ +import base64 +import io +import logging +from typing import Union +from PIL import Image +import numpy as np + +logger = logging.getLogger(__name__) + +def image_to_base64(image: Union[str, bytes, Image.Image, np.ndarray]) -> str: + """Convert image to base64 encoded string + + Args: + image: Image file path, bytes, PIL Image object, or numpy array + + Returns: + Base64 encoded image string, or empty string if conversion fails + """ + try: + if isinstance(image, str): + # It's a file path + with open(image, "rb") as image_file: + return base64.b64encode(image_file.read()).decode("utf-8") + elif isinstance(image, bytes): + # It's bytes data + return base64.b64encode(image).decode("utf-8") + elif isinstance(image, Image.Image): + # It's a PIL Image + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return base64.b64encode(buffer.getvalue()).decode("utf-8") + elif isinstance(image, np.ndarray): + # It's a numpy array + pil_image = Image.fromarray(image) + buffer = io.BytesIO() + pil_image.save(buffer, format="PNG") + return base64.b64encode(buffer.getvalue()).decode("utf-8") + else: + logger.error(f"Unsupported image type: {type(image)}") + return "" + except Exception as e: + logger.error(f"Error converting image to base64: {str(e)}") + return "" diff --git a/services/docreader/src/parser/markdown_parser.py b/services/docreader/src/parser/markdown_parser.py new file mode 100644 index 0000000..330d5a0 --- /dev/null +++ b/services/docreader/src/parser/markdown_parser.py @@ -0,0 +1,33 @@ +import asyncio +import re +import logging +import numpy as np +import os # Import os module to get environment variables +from typing import Dict, List, Optional, Tuple, Union, Any +from .base_parser import BaseParser + +# Get logger object +logger = logging.getLogger(__name__) + + +class MarkdownParser(BaseParser): + """Markdown document parser""" + + def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]: + """Parse Markdown document, only extract text content, do not process images + + Args: + content: Markdown document content + + Returns: + Parsed text result + """ + logger.info(f"Parsing Markdown document, content size: {len(content)} bytes") + + # Convert byte content to string using universal decoding method + text = self.decode_bytes(content) + logger.info(f"Decoded Markdown content, text length: {len(text)} characters") + + logger.info(f"Markdown parsing complete, extracted {len(text)} characters of text") + return text + diff --git a/services/docreader/src/parser/ocr_engine.py b/services/docreader/src/parser/ocr_engine.py new file mode 100644 index 0000000..13c3e88 --- /dev/null +++ b/services/docreader/src/parser/ocr_engine.py @@ -0,0 +1,278 @@ +import os +import logging +import base64 +from typing import Optional, Union, Dict, Any +from abc import ABC, abstractmethod +from PIL import Image +import io +import numpy as np +from .image_utils import image_to_base64 + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +class OCRBackend(ABC): + """Base class for OCR backends""" + + @abstractmethod + def predict(self, image: Union[str, bytes, Image.Image]) -> str: + """Extract text from an image + + Args: + image: Image file path, bytes, or PIL Image object + + Returns: + Extracted text + """ + pass + +class PaddleOCRBackend(OCRBackend): + """PaddleOCR backend implementation""" + + def __init__(self, **kwargs): + """Initialize PaddleOCR backend""" + self.ocr = None + try: + import os + import paddle + + # Set PaddlePaddle to use CPU and disable GPU + os.environ['CUDA_VISIBLE_DEVICES'] = '' + paddle.set_device('cpu') + + # 尝试检测CPU是否支持AVX指令集 + try: + import subprocess + import platform + + # 检测CPU是否支持AVX + if platform.system() == "Linux": + try: + result = subprocess.run(['grep', '-o', 'avx', '/proc/cpuinfo'], + capture_output=True, text=True, timeout=5) + has_avx = 'avx' in result.stdout.lower() + if not has_avx: + logger.warning("CPU does not support AVX instructions, using compatibility mode") + # 进一步限制指令集使用 + os.environ['FLAGS_use_avx2'] = '0' + os.environ['FLAGS_use_avx'] = '1' + except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError): + logger.warning("Could not detect AVX support, using compatibility mode") + os.environ['FLAGS_use_avx2'] = '0' + os.environ['FLAGS_use_avx'] = '1' + except Exception as e: + logger.warning(f"Error detecting CPU capabilities: {e}, using compatibility mode") + os.environ['FLAGS_use_avx2'] = '0' + os.environ['FLAGS_use_avx'] = '1' + + from paddleocr import PaddleOCR + # OCR configuration with text orientation classification enabled + ocr_config = { + "use_gpu": False, + "text_det_limit_type": "max", + "text_det_limit_side_len": 960, + "use_doc_orientation_classify": True, # 启用文档方向分类 + "use_doc_unwarping": False, + "use_textline_orientation": True, # 启用文本行方向检测 + "text_recognition_model_name": "PP-OCRv4_server_rec", + "text_detection_model_name": "PP-OCRv4_server_det", + "text_det_thresh": 0.3, + "text_det_box_thresh": 0.6, + "text_det_unclip_ratio": 1.5, + "text_rec_score_thresh": 0.0, + "ocr_version": "PP-OCRv4", + "lang": "ch", + "show_log": False, + "use_dilation": True, # improves accuracy + "det_db_score_mode": "slow", # improves accuracy + } + + self.ocr = PaddleOCR(**ocr_config) + logger.info("PaddleOCR engine initialized successfully") + + except ImportError as e: + logger.error(f"Failed to import paddleocr: {str(e)}. Please install it with 'pip install paddleocr'") + except OSError as e: + if "Illegal instruction" in str(e) or "core dumped" in str(e): + logger.error(f"PaddlePaddle crashed due to CPU instruction set incompatibility: {str(e)}") + logger.error("This usually happens when the CPU doesn't support AVX instructions.") + logger.error("Please try installing a CPU-only version of PaddlePaddle or use a different OCR backend.") + else: + logger.error(f"Failed to initialize PaddleOCR due to OS error: {str(e)}") + except Exception as e: + logger.error(f"Failed to initialize PaddleOCR: {str(e)}") + + def predict(self, image): + """Perform OCR recognition on the image + + Args: + image: Image object (PIL.Image or numpy array) + + Returns: + Extracted text string + """ + try: + # Ensure image is in RGB format + if hasattr(image, "convert") and image.mode != "RGB": + image = image.convert("RGB") + + # Convert to numpy array if needed + if hasattr(image, "convert"): + image_array = np.array(image) + else: + image_array = image + + # Perform OCR + ocr_result = self.ocr.ocr(image_array, cls=False) + + # Extract text + ocr_text = "" + if ocr_result and ocr_result[0]: + for line in ocr_result[0]: + if line and len(line) >= 2: + text = line[1][0] if line[1] else "" + if text: + ocr_text += text + " " + + text_length = len(ocr_text.strip()) + if text_length > 0: + logger.info(f"OCR extracted {text_length} characters") + return ocr_text.strip() + else: + logger.warning("OCR returned empty result") + return "" + + except Exception as e: + logger.error(f"OCR recognition error: {str(e)}") + return "" + +class NanonetsOCRBackend(OCRBackend): + """Nanonets OCR backend implementation using OpenAI API format""" + + def __init__(self, **kwargs): + """Initialize Nanonets OCR backend + + Args: + api_key: API key for OpenAI API + base_url: Base URL for OpenAI API + model: Model name + """ + try: + from openai import OpenAI + self.api_key = kwargs.get("api_key", "123") + self.base_url = kwargs.get("base_url", "http://localhost:8000/v1") + self.model = kwargs.get("model", "nanonets/Nanonets-OCR-s") + self.temperature = kwargs.get("temperature", 0.0) + self.max_tokens = kwargs.get("max_tokens", 15000) + + self.client = OpenAI(api_key=self.api_key, base_url=self.base_url) + self.prompt = """ +## 任务说明 + +请从上传的文档中提取文字内容，严格按自然阅读顺序（从上到下，从左到右）输出，并遵循以下格式规范。 + +### 1. **文本处理** + +* 按正常阅读顺序提取文字，语句流畅自然。 + +### 2. **表格** + +* 所有表格统一转换为 **Markdown 表格格式**。 +* 内容保持清晰、对齐整齐，便于阅读。 + +### 3. **公式** + +* 所有公式转换为 **LaTeX 格式**，使用 `$$公式$$` 包裹。 + +### 4. **图片** + +* 忽略图片信息 + +### 5. **链接** + +* 不要猜测或补全不确定的链接地址。 +""" + logger.info(f"Nanonets OCR engine initialized with model: {self.model}") + except ImportError: + logger.error("Failed to import openai. Please install it with 'pip install openai'") + self.client = None + except Exception as e: + logger.error(f"Failed to initialize Nanonets OCR: {str(e)}") + self.client = None + + def predict(self, image: Union[str, bytes, Image.Image]) -> str: + """Extract text from an image using Nanonets OCR + + Args: + image: Image file path, bytes, or PIL Image object + + Returns: + Extracted text + """ + if self.client is None: + logger.error("Nanonets OCR client not initialized") + return "" + + try: + # Encode image to base64 + img_base64 = image_to_base64(image) + if not img_base64: + return "" + + # Call Nanonets OCR API + logger.info(f"Calling Nanonets OCR API with model: {self.model}") + response = self.client.chat.completions.create( + model=self.model, + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{img_base64}"}, + }, + { + "type": "text", + "text": self.prompt, + }, + ], + } + ], + temperature=self.temperature, + max_tokens=self.max_tokens + ) + + return response.choices[0].message.content + except Exception as e: + logger.error(f"Nanonets OCR prediction error: {str(e)}") + return "" + +class OCREngine: + """OCR Engine factory class""" + + _instance = None + + @classmethod + def get_instance(cls, backend_type="paddle", **kwargs) -> Optional[OCRBackend]: + """Get OCR engine instance + + Args: + backend_type: OCR backend type, one of: "paddle", "nanonets" + **kwargs: Additional arguments for the backend + + Returns: + OCR engine instance or None if initialization fails + """ + if cls._instance is None: + logger.info(f"Initializing OCR engine with backend: {backend_type}") + + if backend_type.lower() == "paddle": + cls._instance = PaddleOCRBackend(**kwargs) + elif backend_type.lower() == "nanonets": + cls._instance = NanonetsOCRBackend(**kwargs) + else: + logger.error(f"Unknown OCR backend type: {backend_type}") + return None + + return cls._instance + diff --git a/services/docreader/src/parser/parser.py b/services/docreader/src/parser/parser.py new file mode 100644 index 0000000..8e1668d --- /dev/null +++ b/services/docreader/src/parser/parser.py @@ -0,0 +1,206 @@ +import logging +from dataclasses import dataclass, field +from typing import Dict, Any, Optional, Type + +from .base_parser import BaseParser, ParseResult +from .docx_parser import DocxParser +from .doc_parser import DocParser +from .pdf_parser import PDFParser +from .markdown_parser import MarkdownParser +from .text_parser import TextParser +from .image_parser import ImageParser +from .web_parser import WebParser +from .config import ChunkingConfig +import traceback + +logger = logging.getLogger(__name__) + +@dataclass +class Chunk: + """ + Represents a single text chunk with associated metadata. + Basic unit for document processing and embedding. + """ + + content: str # Text content of the chunk + metadata: Dict[str, Any] = None # Associated metadata (source, page number, etc.) + + +class Parser: + """ + Document parser facade that integrates all specialized parsers. + Provides a unified interface for parsing various document types. + """ + + def __init__(self): + logger.info("Initializing document parser") + # Initialize all parser types + self.parsers: Dict[str, Type[BaseParser]] = { + "docx": DocxParser, + "doc": DocParser, + "pdf": PDFParser, + "md": MarkdownParser, + "txt": TextParser, + "jpg": ImageParser, + "jpeg": ImageParser, + "png": ImageParser, + "gif": ImageParser, + "bmp": ImageParser, + "tiff": ImageParser, + "webp": ImageParser, + "markdown": MarkdownParser, + } + logger.info( + "Parser initialized with %d parsers: %s", + len(self.parsers), + ", ".join(self.parsers.keys()), + ) + + + def get_parser(self, file_type: str) -> Optional[Type[BaseParser]]: + """ + Get parser class for the specified file type. + + Args: + file_type: The file extension or type identifier + + Returns: + Parser class for the file type, or None if unsupported + """ + file_type = file_type.lower() + parser = self.parsers.get(file_type) + if parser: + logger.info(f"Found parser for file type: {file_type}") + else: + logger.warning(f"No parser found for file type: {file_type}") + return parser + + def parse_file( + self, + file_name: str, + file_type: str, + content: bytes, + config: ChunkingConfig, + ) -> Optional[ParseResult]: + """ + Parse file content using appropriate parser based on file type. + + Args: + file_name: Name of the file being parsed + file_type: Type/extension of the file + content: Raw file content as bytes + config: Configuration for chunking process + + Returns: + ParseResult containing chunks and metadata, or None if parsing failed + """ + logger.info(f"Parsing file: {file_name} with type: {file_type}") + logger.info( + f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, " + f"multimodal={config.enable_multimodal}" + ) + + parser_instance = None + + try: + # Get appropriate parser for file type + cls = self.get_parser(file_type) + if cls is None: + logger.error(f"Unsupported file type: {file_type}") + return None + + # Parse file content + logger.info(f"Creating parser instance for {file_type} file") + parser_instance = cls( + file_name=file_name, + file_type=file_type, + chunk_size=config.chunk_size, + chunk_overlap=config.chunk_overlap, + separators=config.separators, + enable_multimodal=config.enable_multimodal, + max_image_size=1920, # Limit image size to 1920px + max_concurrent_tasks=5, # Limit concurrent tasks to 5 + chunking_config=config, # Pass the entire chunking config + ) + + logger.info(f"Starting to parse file content, size: {len(content)} bytes") + result = parser_instance.parse(content) + + if result: + logger.info( + f"Successfully parsed file {file_name}, generated {len(result.chunks)} chunks" + ) + if result.chunks and len(result.chunks) > 0: + logger.info( + f"First chunk content length: {len(result.chunks[0].content)}" + ) + else: + logger.warning(f"Parser returned empty chunks for file: {file_name}") + else: + logger.warning(f"Parser returned None result for file: {file_name}") + + # Return parse results + return result + + except Exception as e: + logger.error(f"Error parsing file {file_name}: {str(e)}") + logger.info(f"Detailed traceback: {traceback.format_exc()}") + return None + + def parse_url( + self, url: str, title: str, config: ChunkingConfig + ) -> Optional[ParseResult]: + """ + Parse content from a URL using the WebParser. + + Args: + url: URL to parse + title: Title of the webpage (for metadata) + config: Configuration for chunking process + + Returns: + ParseResult containing chunks and metadata, or None if parsing failed + """ + logger.info(f"Parsing URL: {url}, title: {title}") + logger.info( + f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, " + f"multimodal={config.enable_multimodal}" + ) + + parser_instance = None + + try: + # Create web parser instance + logger.info("Creating WebParser instance") + parser_instance = WebParser( + title=title, + chunk_size=config.chunk_size, + chunk_overlap=config.chunk_overlap, + separators=config.separators, + enable_multimodal=config.enable_multimodal, + max_image_size=1920, # Limit image size + max_concurrent_tasks=5, # Limit concurrent tasks + chunking_config=config, + ) + + logger.info(f"Starting to parse URL content") + result = parser_instance.parse(url) + + if result: + logger.info( + f"Successfully parsed URL, generated {len(result.chunks)} chunks" + ) + logger.info( + f"First chunk content length: {len(result.chunks[0].content) if result.chunks else 0}" + ) + else: + logger.warning(f"Parser returned empty result for URL: {url}") + + # Return parse results + return result + + except Exception as e: + logger.error(f"Error parsing URL {url}: {str(e)}") + logger.info(f"Detailed traceback: {traceback.format_exc()}") + return None + diff --git a/services/docreader/src/parser/pdf_parser.py b/services/docreader/src/parser/pdf_parser.py new file mode 100644 index 0000000..94d9f9a --- /dev/null +++ b/services/docreader/src/parser/pdf_parser.py @@ -0,0 +1,113 @@ +import logging +import os +import io +from typing import Any, List, Iterator, Optional, Mapping, Tuple, Dict, Union + +import pdfplumber +import tempfile +from .base_parser import BaseParser + +logger = logging.getLogger(__name__) + +class PDFParser(BaseParser): + """ + PDF Document Parser + + This parser handles PDF documents by extracting text content. + It uses the pypdf library for simple text extraction. + """ + def _convert_table_to_markdown(self, table_data: list) -> str: + + if not table_data or not table_data[0]: return "" + def clean_cell(cell): + if cell is None: return "" + return str(cell).replace("\n", "
") + try: + markdown = "" + header = [clean_cell(cell) for cell in table_data[0]] + markdown += "| " + " | ".join(header) + " |\n" + markdown += "| " + " | ".join(["---"] * len(header)) + " |\n" + for row in table_data[1:]: + if not row: continue + body_row = [clean_cell(cell) for cell in row] + if len(body_row) != len(header): + logger.warning(f"Skipping malformed table row: {body_row}") + continue + markdown += "| " + " | ".join(body_row) + " |\n" + return markdown + except Exception as e: + logger.error(f"Error converting table to markdown: {e}") + return "" + def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]: + + logger.info(f"Parsing PDF with pdfplumber, content size: {len(content)} bytes") + + all_page_content = [] + + + temp_pdf = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") + temp_pdf_path = temp_pdf.name + + try: + temp_pdf.write(content) + temp_pdf.close() + logger.info(f"PDF content written to temporary file: {temp_pdf_path}") + + with pdfplumber.open(temp_pdf_path) as pdf: + logger.info(f"PDF has {len(pdf.pages)} pages") + + for page_num, page in enumerate(pdf.pages): + page_content_parts = [] + + # Try-fallback strategy for table detection + default_settings = { "vertical_strategy": "lines", "horizontal_strategy": "lines" } + found_tables = page.find_tables(default_settings) + if not found_tables: + logger.info(f"Page {page_num+1}: Default strategy found no tables. Trying fallback strategy.") + fallback_settings = { "vertical_strategy": "text", "horizontal_strategy": "lines" } + found_tables = page.find_tables(fallback_settings) + + table_bboxes = [table.bbox for table in found_tables] + # Define a filter function that keeps objects NOT inside any table bbox. + def not_within_bboxes(obj): + """Check if an object is outside all table bounding boxes.""" + for bbox in table_bboxes: + # Check if the object's vertical center is within a bbox + if bbox[1] <= (obj["top"] + obj["bottom"]) / 2 <= bbox[3]: + return False # It's inside a table, so we DON'T keep it + return True # It's outside all tables, so we DO keep it + + # that contains only the non-table text. + non_table_page = page.filter(not_within_bboxes) + + # Now, extract text from this filtered page view. + text = non_table_page.extract_text(x_tolerance=2) + if text: + page_content_parts.append(text) + + # Process and append the structured Markdown tables + if found_tables: + logger.info(f"Found {len(found_tables)} tables on page {page_num + 1}") + for table in found_tables: + markdown_table = self._convert_table_to_markdown(table.extract()) + page_content_parts.append(f"\n\n{markdown_table}\n\n") + + + all_page_content.append("".join(page_content_parts)) + + final_text = "\n\n--- Page Break ---\n\n".join(all_page_content) + logger.info(f"PDF parsing complete. Extracted {len(final_text)} text chars.") + + return final_text + + except Exception as e: + logger.error(f"Failed to parse PDF document: {str(e)}") + return "" + finally: + # This block is GUARANTEED to execute, preventing resource leaks. + if os.path.exists(temp_pdf_path): + try: + os.remove(temp_pdf_path) + logger.info(f"Temporary file cleaned up: {temp_pdf_path}") + except OSError as e: + logger.error(f"Error removing temporary file {temp_pdf_path}: {e}") diff --git a/services/docreader/src/parser/storage.py b/services/docreader/src/parser/storage.py new file mode 100644 index 0000000..33cb9a2 --- /dev/null +++ b/services/docreader/src/parser/storage.py @@ -0,0 +1,360 @@ +# -*- coding: utf-8 -*- +import os +import uuid +import logging +import io +import traceback +from abc import ABC, abstractmethod +from typing import Tuple, Optional + +from qcloud_cos import CosConfig, CosS3Client +from minio import Minio + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class Storage(ABC): + """Abstract base class for object storage operations""" + + @abstractmethod + def upload_file(self, file_path: str) -> str: + """Upload file to object storage + + Args: + file_path: File path + + Returns: + File URL + """ + pass + + @abstractmethod + def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str: + """Upload bytes to object storage + + Args: + content: Byte content to upload + file_ext: File extension + + Returns: + File URL + """ + pass + + +class CosStorage(Storage): + """Tencent Cloud COS storage implementation""" + + def __init__(self, storage_config=None): + """Initialize COS storage + + Args: + storage_config: Storage configuration + """ + self.storage_config = storage_config + self.client, self.bucket_name, self.region, self.prefix = self._init_cos_client() + + def _init_cos_client(self): + """Initialize Tencent Cloud COS client""" + try: + # Use provided COS config if available, otherwise fall back to environment variables + if self.storage_config and self.storage_config.get("access_key_id") != "": + cos_config = self.storage_config + secret_id = cos_config.get("access_key_id") + secret_key = cos_config.get("secret_access_key") + region = cos_config.get("region") + bucket_name = cos_config.get("bucket_name") + appid = cos_config.get("app_id") + prefix = cos_config.get("path_prefix", "") + else: + # Get COS configuration from environment variables + secret_id = os.getenv("COS_SECRET_ID") + secret_key = os.getenv("COS_SECRET_KEY") + region = os.getenv("COS_REGION") + bucket_name = os.getenv("COS_BUCKET_NAME") + appid = os.getenv("COS_APP_ID") + prefix = os.getenv("COS_PATH_PREFIX") + + enable_old_domain = ( + os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true" + ) + + if not all([secret_id, secret_key, region, bucket_name, appid]): + logger.error( + "Incomplete COS configuration, missing required environment variables" + f"secret_id: {secret_id}, secret_key: {secret_key}, region: {region}, bucket_name: {bucket_name}, appid: {appid}" + ) + return None, None, None, None + + # Initialize COS configuration + logger.info( + f"Initializing COS client with region: {region}, bucket: {bucket_name}" + ) + config = CosConfig( + Appid=appid, + Region=region, + SecretId=secret_id, + SecretKey=secret_key, + EnableOldDomain=enable_old_domain, + ) + + # Create client + client = CosS3Client(config) + return client, bucket_name, region, prefix + except Exception as e: + logger.error(f"Failed to initialize COS client: {str(e)}") + return None, None, None, None + + def _get_download_url(self, bucket_name, region, object_key): + """Generate COS object URL + + Args: + bucket_name: Bucket name + region: Region + object_key: Object key + + Returns: + File URL + """ + return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}" + + + def upload_file(self, file_path: str) -> str: + """Upload file to Tencent Cloud COS + + Args: + file_path: File path + + Returns: + File URL + """ + logger.info(f"Uploading file to COS: {file_path}") + try: + if not self.client: + return "" + + # Generate object key, use UUID to avoid conflicts + file_name = os.path.basename(file_path) + object_key = ( + f"{self.prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}" + ) + logger.info(f"Generated object key: {object_key}") + + # Upload file + logger.info("Attempting to upload file to COS") + response = self.client.upload_file( + Bucket=self.bucket_name, LocalFilePath=file_path, Key=object_key + ) + + # Get file URL + file_url = self._get_download_url(self.bucket_name, self.region, object_key) + + logger.info(f"Successfully uploaded file to COS: {file_url}") + return file_url + + except Exception as e: + logger.error(f"Failed to upload file to COS: {str(e)}") + return "" + + def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str: + """Upload bytes to Tencent Cloud COS + + Args: + content: Byte content to upload + file_ext: File extension + + Returns: + File URL + """ + try: + logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes") + if not self.client: + return "" + + object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.prefix else f"images/{uuid.uuid4().hex}{file_ext}" + logger.info(f"Generated object key: {object_key}") + self.client.put_object(Bucket=self.bucket_name, Body=content, Key=object_key) + file_url = self._get_download_url(self.bucket_name, self.region, object_key) + logger.info(f"Successfully uploaded bytes to COS: {file_url}") + return file_url + except Exception as e: + logger.error(f"Failed to upload bytes to COS: {str(e)}") + traceback.print_exc() + return "" + + +class MinioStorage(Storage): + """MinIO storage implementation""" + + def __init__(self, storage_config=None): + """Initialize MinIO storage + + Args: + storage_config: Storage configuration + """ + self.storage_config = storage_config + self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = self._init_minio_client() + + def _init_minio_client(self): + """Initialize MinIO client from environment variables or injected config. + + If storage_config.path_prefix contains JSON from server (for minio case), + prefer those values to override envs. + """ + try: + endpoint = os.getenv("MINIO_ENDPOINT") + use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true" + if self.storage_config and self.storage_config.get("bucket_name"): + storage_config = self.storage_config + bucket_name = storage_config.get("bucket_name") + path_prefix = storage_config.get("path_prefix").strip().strip("/") + access_key = storage_config.get("access_key_id") + secret_key = storage_config.get("secret_access_key") + else: + access_key = os.getenv("MINIO_ACCESS_KEY_ID") + secret_key = os.getenv("MINIO_SECRET_ACCESS_KEY") + bucket_name = os.getenv("MINIO_BUCKET_NAME") + path_prefix = os.getenv("MINIO_PATH_PREFIX", "").strip().strip("/") + + if not all([endpoint, access_key, secret_key, bucket_name]): + logger.error("Incomplete MinIO configuration, missing required environment variables") + return None, None, None, None, None + + # Initialize client + client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl) + + # Ensure bucket exists + found = client.bucket_exists(bucket_name) + if not found: + client.make_bucket(bucket_name) + policy = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}' % (bucket_name, bucket_name) + client.set_bucket_policy(bucket_name, policy) + + return client, bucket_name, use_ssl, endpoint, path_prefix + except Exception as e: + logger.error(f"Failed to initialize MinIO client: {str(e)}") + return None, None, None, None, None + + def _get_download_url(self, bucket_name: str, object_key: str, use_ssl: bool, endpoint: str, public_endpoint: str = None): + """Construct a public URL for MinIO object. + + If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint. + """ + if public_endpoint: + base = public_endpoint + else: + scheme = "https" if use_ssl else "http" + base = f"{scheme}://{endpoint}" + # Path-style URL for MinIO + return f"{base}/{bucket_name}/{object_key}" + + def upload_file(self, file_path: str) -> str: + """Upload file to MinIO + + Args: + file_path: File path + + Returns: + File URL + """ + logger.info(f"Uploading file to MinIO: {file_path}") + try: + if not self.client: + return "" + + # Generate object key, use UUID to avoid conflicts + file_name = os.path.basename(file_path) + object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}" if self.path_prefix else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}" + logger.info(f"Generated MinIO object key: {object_key}") + + # Upload file + logger.info("Attempting to upload file to MinIO") + with open(file_path, 'rb') as file_data: + file_size = os.path.getsize(file_path) + self.client.put_object( + bucket_name=self.bucket_name, + object_name=object_key, + data=file_data, + length=file_size, + content_type='application/octet-stream' + ) + + # Get file URL + file_url = self._get_download_url( + self.bucket_name, + object_key, + self.use_ssl, + self.endpoint, + os.getenv("MINIO_PUBLIC_ENDPOINT", None) + ) + + logger.info(f"Successfully uploaded file to MinIO: {file_url}") + return file_url + + except Exception as e: + logger.error(f"Failed to upload file to MinIO: {str(e)}") + return "" + + def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str: + """Upload bytes to MinIO + + Args: + content: Byte content to upload + file_ext: File extension + + Returns: + File URL + """ + try: + logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes") + if not self.client: + return "" + + object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.path_prefix else f"images/{uuid.uuid4().hex}{file_ext}" + logger.info(f"Generated MinIO object key: {object_key}") + self.client.put_object( + self.bucket_name, + object_key, + data=io.BytesIO(content), + length=len(content), + content_type="application/octet-stream" + ) + file_url = self._get_download_url( + self.bucket_name, + object_key, + self.use_ssl, + self.endpoint, + os.getenv("MINIO_PUBLIC_ENDPOINT", None) + ) + logger.info(f"Successfully uploaded bytes to MinIO: {file_url}") + return file_url + except Exception as e: + logger.error(f"Failed to upload bytes to MinIO: {str(e)}") + traceback.print_exc() + return "" + + +def create_storage(storage_config=None) -> Storage: + """Create a storage instance based on configuration or environment variables + + Args: + storage_config: Storage configuration dictionary + + Returns: + Storage instance + """ + storage_type = os.getenv("STORAGE_TYPE", "cos").lower() + + if storage_config: + storage_type = str(storage_config.get("provider", storage_type)).lower() + + logger.info(f"Creating {storage_type} storage instance") + + if storage_type == "minio": + return MinioStorage(storage_config) + elif storage_type == "cos": + # Default to COS + return CosStorage(storage_config) + else: + return None \ No newline at end of file diff --git a/services/docreader/src/parser/text_parser.py b/services/docreader/src/parser/text_parser.py new file mode 100644 index 0000000..0bd0dd7 --- /dev/null +++ b/services/docreader/src/parser/text_parser.py @@ -0,0 +1,58 @@ +import logging +from .base_parser import BaseParser +from typing import Dict, Any, Tuple, Union + +logger = logging.getLogger(__name__) + + +class TextParser(BaseParser): + """ + Text document parser for processing plain text files. + This parser handles text extraction and chunking from plain text documents. + """ + + def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]: + """ + Parse text document content by decoding bytes to string. + + This is a straightforward parser that simply converts the binary content + to text using appropriate character encoding. + + Args: + content: Raw document content as bytes + + Returns: + Parsed text content as string + """ + logger.info(f"Parsing text document, content size: {len(content)} bytes") + text = self.decode_bytes(content) + logger.info( + f"Successfully parsed text document, extracted {len(text)} characters" + ) + return text + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + logger.info("Running TextParser in standalone mode") + + # Sample text for testing + text = """## 标题1 + ![alt text](image.png) + ## 标题2 + ![alt text](image2.png) + ## 标题3 + ![alt text](image3.png)""" + logger.info(f"Test text content: {text}") + + # Define separators for text splitting + seperators = ["\n\n", "\n", "。"] + parser = TextParser(separators=seperators) + logger.info("Splitting text into units") + units = parser._split_into_units(text) + logger.info(f"Split text into {len(units)} units") + logger.info(f"Units: {units}") diff --git a/services/docreader/src/parser/web_parser.py b/services/docreader/src/parser/web_parser.py new file mode 100644 index 0000000..2d53383 --- /dev/null +++ b/services/docreader/src/parser/web_parser.py @@ -0,0 +1,133 @@ +from typing import Any, Optional, Tuple, Dict, Union +import os + +from playwright.async_api import async_playwright +from bs4 import BeautifulSoup +from .base_parser import BaseParser, ParseResult +import logging +import asyncio + +logger = logging.getLogger(__name__) + + +class WebParser(BaseParser): + """Web page parser""" + + def __init__(self, title: str, **kwargs): + self.title = title + self.extracted_title = None # Store extracted title from HTML + self.proxy = os.environ.get("WEB_PROXY", "") + super().__init__(file_name=title, **kwargs) + logger.info(f"Initialized WebParser with title: {title}") + + async def scrape(self, url: str) -> Any: + logger.info(f"Starting web page scraping for URL: {url}") + try: + async with async_playwright() as p: + kwargs = {} + if self.proxy: + kwargs["proxy"] = {"server": self.proxy} + logger.info("Launching WebKit browser") + browser = await p.webkit.launch(**kwargs) + page = await browser.new_page() + + logger.info(f"Navigating to URL: {url}") + try: + await page.goto(url, timeout=30000) + logger.info("Initial page load complete") + except Exception as e: + logger.error(f"Error navigating to URL: {str(e)}") + await browser.close() + return BeautifulSoup( + "", "html.parser" + ) # Return empty soup on navigation error + + logger.info("Retrieving page HTML content") + content = await page.content() + logger.info(f"Retrieved {len(content)} bytes of HTML content") + + await browser.close() + logger.info("Browser closed") + + # Parse HTML content with BeautifulSoup + logger.info("Parsing HTML with BeautifulSoup") + soup = BeautifulSoup(content, "html.parser") + logger.info("Successfully parsed HTML content") + return soup + + except Exception as e: + logger.error(f"Failed to scrape web page: {str(e)}") + # Return empty BeautifulSoup object on error + return BeautifulSoup("", "html.parser") + + def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]: + """Parse web page + + Args: + content: Web page content + + Returns: + Parse result + """ + logger.info("Starting web page parsing") + + # Call async method synchronously + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + try: + # Run async method + # Handle content possibly being a string + if isinstance(content, bytes): + url = self.decode_bytes(content) + logger.info(f"Decoded URL from bytes: {url}") + else: + url = content + logger.info(f"Using content as URL directly: {url}") + + logger.info(f"Scraping web page: {url}") + soup = loop.run_until_complete(self.scrape(url)) + + # Extract page text + logger.info("Extracting text from web page") + text = soup.get_text("\n") + logger.info(f"Extracted {len(text)} characters of text from URL: {url}") + + # Get title, usually in or <h1> tag + if self.title != "": + title = self.title + logger.info(f"Using provided title: {title}") + else: + title = soup.title.string if soup.title else None + logger.info(f"Found title tag: {title}") + + if not title: # If <title> tag does not exist or is empty, try <h1> tag + h1_tag = soup.find("h1") + if h1_tag: + title = h1_tag.get_text() + logger.info(f"Using h1 tag as title: {title}") + else: + title = "Untitled Web Page" + logger.info("No title found, using default") + + logger.info(f"Web page title: {title}") + # Store the extracted title for later use + self.extracted_title = title.strip() if title else "" + text = "\n".join( + (line.strip() for line in text.splitlines() if line.strip()) + ) + + result = title + "\n\n" + text + logger.info( + f"Web page parsing complete, total content: {len(result)} characters" + ) + return result + + except Exception as e: + logger.error(f"Error parsing web page: {str(e)}") + return f"Error parsing web page: {str(e)}" + + finally: + # Close event loop + logger.info("Closing event loop") + loop.close() diff --git a/services/docreader/src/proto/docreader.pb.go b/services/docreader/src/proto/docreader.pb.go new file mode 100644 index 0000000..cc917b3 --- /dev/null +++ b/services/docreader/src/proto/docreader.pb.go @@ -0,0 +1,827 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.6 +// protoc v5.29.3 +// source: docreader.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// 对象存储提供方 +type StorageProvider int32 + +const ( + StorageProvider_STORAGE_PROVIDER_UNSPECIFIED StorageProvider = 0 + StorageProvider_COS StorageProvider = 1 // 腾讯云 COS + StorageProvider_MINIO StorageProvider = 2 // MinIO/S3 兼容 +) + +// Enum value maps for StorageProvider. +var ( + StorageProvider_name = map[int32]string{ + 0: "STORAGE_PROVIDER_UNSPECIFIED", + 1: "COS", + 2: "MINIO", + } + StorageProvider_value = map[string]int32{ + "STORAGE_PROVIDER_UNSPECIFIED": 0, + "COS": 1, + "MINIO": 2, + } +) + +func (x StorageProvider) Enum() *StorageProvider { + p := new(StorageProvider) + *p = x + return p +} + +func (x StorageProvider) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (StorageProvider) Descriptor() protoreflect.EnumDescriptor { + return file_docreader_proto_enumTypes[0].Descriptor() +} + +func (StorageProvider) Type() protoreflect.EnumType { + return &file_docreader_proto_enumTypes[0] +} + +func (x StorageProvider) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use StorageProvider.Descriptor instead. +func (StorageProvider) EnumDescriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{0} +} + +// 通用对象存储配置，兼容 COS 与 MinIO +type StorageConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Provider StorageProvider `protobuf:"varint,1,opt,name=provider,proto3,enum=docreader.StorageProvider" json:"provider,omitempty"` // 存储提供方 + Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` // 区域（COS 使用） + BucketName string `protobuf:"bytes,3,opt,name=bucket_name,json=bucketName,proto3" json:"bucket_name,omitempty"` // 桶名 + AccessKeyId string `protobuf:"bytes,4,opt,name=access_key_id,json=accessKeyId,proto3" json:"access_key_id,omitempty"` // 访问密钥 ID（MinIO/S3 使用） + SecretAccessKey string `protobuf:"bytes,5,opt,name=secret_access_key,json=secretAccessKey,proto3" json:"secret_access_key,omitempty"` // 访问密钥 Secret（MinIO/S3 使用） + AppId string `protobuf:"bytes,6,opt,name=app_id,json=appId,proto3" json:"app_id,omitempty"` // 应用 ID（COS 使用） + PathPrefix string `protobuf:"bytes,7,opt,name=path_prefix,json=pathPrefix,proto3" json:"path_prefix,omitempty"` // 路径前缀 + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StorageConfig) Reset() { + *x = StorageConfig{} + mi := &file_docreader_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StorageConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StorageConfig) ProtoMessage() {} + +func (x *StorageConfig) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StorageConfig.ProtoReflect.Descriptor instead. +func (*StorageConfig) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{0} +} + +func (x *StorageConfig) GetProvider() StorageProvider { + if x != nil { + return x.Provider + } + return StorageProvider_STORAGE_PROVIDER_UNSPECIFIED +} + +func (x *StorageConfig) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *StorageConfig) GetBucketName() string { + if x != nil { + return x.BucketName + } + return "" +} + +func (x *StorageConfig) GetAccessKeyId() string { + if x != nil { + return x.AccessKeyId + } + return "" +} + +func (x *StorageConfig) GetSecretAccessKey() string { + if x != nil { + return x.SecretAccessKey + } + return "" +} + +func (x *StorageConfig) GetAppId() string { + if x != nil { + return x.AppId + } + return "" +} + +func (x *StorageConfig) GetPathPrefix() string { + if x != nil { + return x.PathPrefix + } + return "" +} + +// VLM 配置 +type VLMConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + ModelName string `protobuf:"bytes,1,opt,name=model_name,json=modelName,proto3" json:"model_name,omitempty"` // VLM Model Name + BaseUrl string `protobuf:"bytes,2,opt,name=base_url,json=baseUrl,proto3" json:"base_url,omitempty"` // VLM Base URL + ApiKey string `protobuf:"bytes,3,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"` // VLM API Key + InterfaceType string `protobuf:"bytes,4,opt,name=interface_type,json=interfaceType,proto3" json:"interface_type,omitempty"` // VLM Interface Type: "ollama" or "openai" + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VLMConfig) Reset() { + *x = VLMConfig{} + mi := &file_docreader_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VLMConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VLMConfig) ProtoMessage() {} + +func (x *VLMConfig) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VLMConfig.ProtoReflect.Descriptor instead. +func (*VLMConfig) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{1} +} + +func (x *VLMConfig) GetModelName() string { + if x != nil { + return x.ModelName + } + return "" +} + +func (x *VLMConfig) GetBaseUrl() string { + if x != nil { + return x.BaseUrl + } + return "" +} + +func (x *VLMConfig) GetApiKey() string { + if x != nil { + return x.ApiKey + } + return "" +} + +func (x *VLMConfig) GetInterfaceType() string { + if x != nil { + return x.InterfaceType + } + return "" +} + +type ReadConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + ChunkSize int32 `protobuf:"varint,1,opt,name=chunk_size,json=chunkSize,proto3" json:"chunk_size,omitempty"` // 分块大小 + ChunkOverlap int32 `protobuf:"varint,2,opt,name=chunk_overlap,json=chunkOverlap,proto3" json:"chunk_overlap,omitempty"` // 分块重叠 + Separators []string `protobuf:"bytes,3,rep,name=separators,proto3" json:"separators,omitempty"` // 分隔符 + EnableMultimodal bool `protobuf:"varint,4,opt,name=enable_multimodal,json=enableMultimodal,proto3" json:"enable_multimodal,omitempty"` // 多模态处理 + StorageConfig *StorageConfig `protobuf:"bytes,5,opt,name=storage_config,json=storageConfig,proto3" json:"storage_config,omitempty"` // 对象存储配置（通用） + VlmConfig *VLMConfig `protobuf:"bytes,6,opt,name=vlm_config,json=vlmConfig,proto3" json:"vlm_config,omitempty"` // VLM 配置 + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadConfig) Reset() { + *x = ReadConfig{} + mi := &file_docreader_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadConfig) ProtoMessage() {} + +func (x *ReadConfig) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadConfig.ProtoReflect.Descriptor instead. +func (*ReadConfig) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{2} +} + +func (x *ReadConfig) GetChunkSize() int32 { + if x != nil { + return x.ChunkSize + } + return 0 +} + +func (x *ReadConfig) GetChunkOverlap() int32 { + if x != nil { + return x.ChunkOverlap + } + return 0 +} + +func (x *ReadConfig) GetSeparators() []string { + if x != nil { + return x.Separators + } + return nil +} + +func (x *ReadConfig) GetEnableMultimodal() bool { + if x != nil { + return x.EnableMultimodal + } + return false +} + +func (x *ReadConfig) GetStorageConfig() *StorageConfig { + if x != nil { + return x.StorageConfig + } + return nil +} + +func (x *ReadConfig) GetVlmConfig() *VLMConfig { + if x != nil { + return x.VlmConfig + } + return nil +} + +// 从文件读取文档请求 +type ReadFromFileRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"` // 文件内容 + FileName string `protobuf:"bytes,2,opt,name=file_name,json=fileName,proto3" json:"file_name,omitempty"` // 文件名 + FileType string `protobuf:"bytes,3,opt,name=file_type,json=fileType,proto3" json:"file_type,omitempty"` // 文件类型 + ReadConfig *ReadConfig `protobuf:"bytes,4,opt,name=read_config,json=readConfig,proto3" json:"read_config,omitempty"` + RequestId string `protobuf:"bytes,5,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadFromFileRequest) Reset() { + *x = ReadFromFileRequest{} + mi := &file_docreader_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadFromFileRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadFromFileRequest) ProtoMessage() {} + +func (x *ReadFromFileRequest) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadFromFileRequest.ProtoReflect.Descriptor instead. +func (*ReadFromFileRequest) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{3} +} + +func (x *ReadFromFileRequest) GetFileContent() []byte { + if x != nil { + return x.FileContent + } + return nil +} + +func (x *ReadFromFileRequest) GetFileName() string { + if x != nil { + return x.FileName + } + return "" +} + +func (x *ReadFromFileRequest) GetFileType() string { + if x != nil { + return x.FileType + } + return "" +} + +func (x *ReadFromFileRequest) GetReadConfig() *ReadConfig { + if x != nil { + return x.ReadConfig + } + return nil +} + +func (x *ReadFromFileRequest) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +// 从URL读取文档请求 +type ReadFromURLRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // 文档URL + Title string `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"` // 标题 + ReadConfig *ReadConfig `protobuf:"bytes,3,opt,name=read_config,json=readConfig,proto3" json:"read_config,omitempty"` + RequestId string `protobuf:"bytes,4,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadFromURLRequest) Reset() { + *x = ReadFromURLRequest{} + mi := &file_docreader_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadFromURLRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadFromURLRequest) ProtoMessage() {} + +func (x *ReadFromURLRequest) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadFromURLRequest.ProtoReflect.Descriptor instead. +func (*ReadFromURLRequest) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{4} +} + +func (x *ReadFromURLRequest) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *ReadFromURLRequest) GetTitle() string { + if x != nil { + return x.Title + } + return "" +} + +func (x *ReadFromURLRequest) GetReadConfig() *ReadConfig { + if x != nil { + return x.ReadConfig + } + return nil +} + +func (x *ReadFromURLRequest) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + +// 图片信息 +type Image struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // 图片URL + Caption string `protobuf:"bytes,2,opt,name=caption,proto3" json:"caption,omitempty"` // 图片描述 + OcrText string `protobuf:"bytes,3,opt,name=ocr_text,json=ocrText,proto3" json:"ocr_text,omitempty"` // OCR提取的文本内容 + OriginalUrl string `protobuf:"bytes,4,opt,name=original_url,json=originalUrl,proto3" json:"original_url,omitempty"` // 原始图片URL + Start int32 `protobuf:"varint,5,opt,name=start,proto3" json:"start,omitempty"` // 图片在文本中的开始位置 + End int32 `protobuf:"varint,6,opt,name=end,proto3" json:"end,omitempty"` // 图片在文本中的结束位置 + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Image) Reset() { + *x = Image{} + mi := &file_docreader_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Image) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Image) ProtoMessage() {} + +func (x *Image) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Image.ProtoReflect.Descriptor instead. +func (*Image) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{5} +} + +func (x *Image) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *Image) GetCaption() string { + if x != nil { + return x.Caption + } + return "" +} + +func (x *Image) GetOcrText() string { + if x != nil { + return x.OcrText + } + return "" +} + +func (x *Image) GetOriginalUrl() string { + if x != nil { + return x.OriginalUrl + } + return "" +} + +func (x *Image) GetStart() int32 { + if x != nil { + return x.Start + } + return 0 +} + +func (x *Image) GetEnd() int32 { + if x != nil { + return x.End + } + return 0 +} + +type Chunk struct { + state protoimpl.MessageState `protogen:"open.v1"` + Content string `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` // 块内容 + Seq int32 `protobuf:"varint,2,opt,name=seq,proto3" json:"seq,omitempty"` // 块在文档中的次序 + Start int32 `protobuf:"varint,3,opt,name=start,proto3" json:"start,omitempty"` // 块在文档中的起始位置 + End int32 `protobuf:"varint,4,opt,name=end,proto3" json:"end,omitempty"` // 块在文档中的结束位置 + Images []*Image `protobuf:"bytes,5,rep,name=images,proto3" json:"images,omitempty"` // 块中包含的图片信息 + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Chunk) Reset() { + *x = Chunk{} + mi := &file_docreader_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Chunk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Chunk) ProtoMessage() {} + +func (x *Chunk) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Chunk.ProtoReflect.Descriptor instead. +func (*Chunk) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{6} +} + +func (x *Chunk) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +func (x *Chunk) GetSeq() int32 { + if x != nil { + return x.Seq + } + return 0 +} + +func (x *Chunk) GetStart() int32 { + if x != nil { + return x.Start + } + return 0 +} + +func (x *Chunk) GetEnd() int32 { + if x != nil { + return x.End + } + return 0 +} + +func (x *Chunk) GetImages() []*Image { + if x != nil { + return x.Images + } + return nil +} + +// 从URL读取文档响应 +type ReadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Chunks []*Chunk `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"` // 文档分块 + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` // 错误信息 + Title string `protobuf:"bytes,3,opt,name=title,proto3" json:"title,omitempty"` // 文档标题 (仅从URL读取时返回) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReadResponse) Reset() { + *x = ReadResponse{} + mi := &file_docreader_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadResponse) ProtoMessage() {} + +func (x *ReadResponse) ProtoReflect() protoreflect.Message { + mi := &file_docreader_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadResponse.ProtoReflect.Descriptor instead. +func (*ReadResponse) Descriptor() ([]byte, []int) { + return file_docreader_proto_rawDescGZIP(), []int{7} +} + +func (x *ReadResponse) GetChunks() []*Chunk { + if x != nil { + return x.Chunks + } + return nil +} + +func (x *ReadResponse) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *ReadResponse) GetTitle() string { + if x != nil { + return x.Title + } + return "" +} + +var File_docreader_proto protoreflect.FileDescriptor + +const file_docreader_proto_rawDesc = "" + + "\n" + + "\x0fdocreader.proto\x12\tdocreader\"\x88\x02\n" + + "\rStorageConfig\x126\n" + + "\bprovider\x18\x01 \x01(\x0e2\x1a.docreader.StorageProviderR\bprovider\x12\x16\n" + + "\x06region\x18\x02 \x01(\tR\x06region\x12\x1f\n" + + "\vbucket_name\x18\x03 \x01(\tR\n" + + "bucketName\x12\"\n" + + "\raccess_key_id\x18\x04 \x01(\tR\vaccessKeyId\x12*\n" + + "\x11secret_access_key\x18\x05 \x01(\tR\x0fsecretAccessKey\x12\x15\n" + + "\x06app_id\x18\x06 \x01(\tR\x05appId\x12\x1f\n" + + "\vpath_prefix\x18\a \x01(\tR\n" + + "pathPrefix\"\x85\x01\n" + + "\tVLMConfig\x12\x1d\n" + + "\n" + + "model_name\x18\x01 \x01(\tR\tmodelName\x12\x19\n" + + "\bbase_url\x18\x02 \x01(\tR\abaseUrl\x12\x17\n" + + "\aapi_key\x18\x03 \x01(\tR\x06apiKey\x12%\n" + + "\x0einterface_type\x18\x04 \x01(\tR\rinterfaceType\"\x93\x02\n" + + "\n" + + "ReadConfig\x12\x1d\n" + + "\n" + + "chunk_size\x18\x01 \x01(\x05R\tchunkSize\x12#\n" + + "\rchunk_overlap\x18\x02 \x01(\x05R\fchunkOverlap\x12\x1e\n" + + "\n" + + "separators\x18\x03 \x03(\tR\n" + + "separators\x12+\n" + + "\x11enable_multimodal\x18\x04 \x01(\bR\x10enableMultimodal\x12?\n" + + "\x0estorage_config\x18\x05 \x01(\v2\x18.docreader.StorageConfigR\rstorageConfig\x123\n" + + "\n" + + "vlm_config\x18\x06 \x01(\v2\x14.docreader.VLMConfigR\tvlmConfig\"\xc9\x01\n" + + "\x13ReadFromFileRequest\x12!\n" + + "\ffile_content\x18\x01 \x01(\fR\vfileContent\x12\x1b\n" + + "\tfile_name\x18\x02 \x01(\tR\bfileName\x12\x1b\n" + + "\tfile_type\x18\x03 \x01(\tR\bfileType\x126\n" + + "\vread_config\x18\x04 \x01(\v2\x15.docreader.ReadConfigR\n" + + "readConfig\x12\x1d\n" + + "\n" + + "request_id\x18\x05 \x01(\tR\trequestId\"\x93\x01\n" + + "\x12ReadFromURLRequest\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12\x14\n" + + "\x05title\x18\x02 \x01(\tR\x05title\x126\n" + + "\vread_config\x18\x03 \x01(\v2\x15.docreader.ReadConfigR\n" + + "readConfig\x12\x1d\n" + + "\n" + + "request_id\x18\x04 \x01(\tR\trequestId\"\x99\x01\n" + + "\x05Image\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12\x18\n" + + "\acaption\x18\x02 \x01(\tR\acaption\x12\x19\n" + + "\bocr_text\x18\x03 \x01(\tR\aocrText\x12!\n" + + "\foriginal_url\x18\x04 \x01(\tR\voriginalUrl\x12\x14\n" + + "\x05start\x18\x05 \x01(\x05R\x05start\x12\x10\n" + + "\x03end\x18\x06 \x01(\x05R\x03end\"\x85\x01\n" + + "\x05Chunk\x12\x18\n" + + "\acontent\x18\x01 \x01(\tR\acontent\x12\x10\n" + + "\x03seq\x18\x02 \x01(\x05R\x03seq\x12\x14\n" + + "\x05start\x18\x03 \x01(\x05R\x05start\x12\x10\n" + + "\x03end\x18\x04 \x01(\x05R\x03end\x12(\n" + + "\x06images\x18\x05 \x03(\v2\x10.docreader.ImageR\x06images\"N\n" + + "\fReadResponse\x12(\n" + + "\x06chunks\x18\x01 \x03(\v2\x10.docreader.ChunkR\x06chunks\x12\x14\n" + + "\x05error\x18\x02 \x01(\tR\x05error*G\n" + + "\x0fStorageProvider\x12 \n" + + "\x1cSTORAGE_PROVIDER_UNSPECIFIED\x10\x00\x12\a\n" + + "\x03COS\x10\x01\x12\t\n" + + "\x05MINIO\x10\x022\x9f\x01\n" + + "\tDocReader\x12I\n" + + "\fReadFromFile\x12\x1e.docreader.ReadFromFileRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n" + + "\vReadFromURL\x12\x1d.docreader.ReadFromURLRequest\x1a\x17.docreader.ReadResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3" + +var ( + file_docreader_proto_rawDescOnce sync.Once + file_docreader_proto_rawDescData []byte +) + +func file_docreader_proto_rawDescGZIP() []byte { + file_docreader_proto_rawDescOnce.Do(func() { + file_docreader_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc))) + }) + return file_docreader_proto_rawDescData +} + +var file_docreader_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_docreader_proto_goTypes = []any{ + (StorageProvider)(0), // 0: docreader.StorageProvider + (*StorageConfig)(nil), // 1: docreader.StorageConfig + (*VLMConfig)(nil), // 2: docreader.VLMConfig + (*ReadConfig)(nil), // 3: docreader.ReadConfig + (*ReadFromFileRequest)(nil), // 4: docreader.ReadFromFileRequest + (*ReadFromURLRequest)(nil), // 5: docreader.ReadFromURLRequest + (*Image)(nil), // 6: docreader.Image + (*Chunk)(nil), // 7: docreader.Chunk + (*ReadResponse)(nil), // 8: docreader.ReadResponse +} +var file_docreader_proto_depIdxs = []int32{ + 0, // 0: docreader.StorageConfig.provider:type_name -> docreader.StorageProvider + 1, // 1: docreader.ReadConfig.storage_config:type_name -> docreader.StorageConfig + 2, // 2: docreader.ReadConfig.vlm_config:type_name -> docreader.VLMConfig + 3, // 3: docreader.ReadFromFileRequest.read_config:type_name -> docreader.ReadConfig + 3, // 4: docreader.ReadFromURLRequest.read_config:type_name -> docreader.ReadConfig + 6, // 5: docreader.Chunk.images:type_name -> docreader.Image + 7, // 6: docreader.ReadResponse.chunks:type_name -> docreader.Chunk + 4, // 7: docreader.DocReader.ReadFromFile:input_type -> docreader.ReadFromFileRequest + 5, // 8: docreader.DocReader.ReadFromURL:input_type -> docreader.ReadFromURLRequest + 8, // 9: docreader.DocReader.ReadFromFile:output_type -> docreader.ReadResponse + 8, // 10: docreader.DocReader.ReadFromURL:output_type -> docreader.ReadResponse + 9, // [9:11] is the sub-list for method output_type + 7, // [7:9] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_docreader_proto_init() } +func file_docreader_proto_init() { + if File_docreader_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_docreader_proto_goTypes, + DependencyIndexes: file_docreader_proto_depIdxs, + EnumInfos: file_docreader_proto_enumTypes, + MessageInfos: file_docreader_proto_msgTypes, + }.Build() + File_docreader_proto = out.File + file_docreader_proto_goTypes = nil + file_docreader_proto_depIdxs = nil +} diff --git a/services/docreader/src/proto/docreader.proto b/services/docreader/src/proto/docreader.proto new file mode 100644 index 0000000..b799c82 --- /dev/null +++ b/services/docreader/src/proto/docreader.proto @@ -0,0 +1,90 @@ +syntax = "proto3"; + +package docreader; + +option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto"; + +// 文档读取服务 +service DocReader { + // 从文件读取文档 + rpc ReadFromFile(ReadFromFileRequest) returns (ReadResponse) {} + // 从URL读取文档 + rpc ReadFromURL(ReadFromURLRequest) returns (ReadResponse) {} +} + +// 对象存储提供方 +enum StorageProvider { + STORAGE_PROVIDER_UNSPECIFIED = 0; + COS = 1; // 腾讯云 COS + MINIO = 2; // MinIO/S3 兼容 +} + +// 通用对象存储配置，兼容 COS 与 MinIO +message StorageConfig { + StorageProvider provider = 1; // 存储提供方 + string region = 2; // 区域（COS 使用） + string bucket_name = 3; // 桶名 + string access_key_id = 4; // 访问密钥 ID（MinIO/S3 使用） + string secret_access_key = 5; // 访问密钥 Secret（MinIO/S3 使用） + string app_id = 6; // 应用 ID（COS 使用） + string path_prefix = 7; // 路径前缀 +} + +// VLM 配置 +message VLMConfig { + string model_name = 1; // VLM Model Name + string base_url = 2; // VLM Base URL + string api_key = 3; // VLM API Key + string interface_type = 4; // VLM Interface Type: "ollama" or "openai" +} + +message ReadConfig { + int32 chunk_size = 1; // 分块大小 + int32 chunk_overlap = 2; // 分块重叠 + repeated string separators = 3; // 分隔符 + bool enable_multimodal = 4; // 多模态处理 + StorageConfig storage_config = 5; // 对象存储配置（通用） + VLMConfig vlm_config = 6; // VLM 配置 +} + +// 从文件读取文档请求 +message ReadFromFileRequest { + bytes file_content = 1; // 文件内容 + string file_name = 2; // 文件名 + string file_type = 3; // 文件类型 + ReadConfig read_config = 4; + string request_id = 5; +} + +// 从URL读取文档请求 +message ReadFromURLRequest { + string url = 1; // 文档URL + string title = 2; // 标题 + ReadConfig read_config = 3; + string request_id = 4; +} + +// 图片信息 +message Image { + string url = 1; // 图片URL + string caption = 2; // 图片描述 + string ocr_text = 3; // OCR提取的文本内容 + string original_url = 4; // 原始图片URL + int32 start = 5; // 图片在文本中的开始位置 + int32 end = 6; // 图片在文本中的结束位置 +} + +message Chunk { + string content = 1; // 块内容 + int32 seq = 2; // 块在文档中的次序 + int32 start = 3; // 块在文档中的起始位置 + int32 end = 4; // 块在文档中的结束位置 + repeated Image images = 5; // 块中包含的图片信息 +} + +// 从URL读取文档响应 +message ReadResponse { + repeated Chunk chunks = 1; // 文档分块 + string error = 2; // 错误信息 + string title = 3; // 文档标题 (仅从URL读取时返回) +} \ No newline at end of file diff --git a/services/docreader/src/proto/docreader_grpc.pb.go b/services/docreader/src/proto/docreader_grpc.pb.go new file mode 100644 index 0000000..79bd7ef --- /dev/null +++ b/services/docreader/src/proto/docreader_grpc.pb.go @@ -0,0 +1,167 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v5.29.3 +// source: docreader.proto + +package proto + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + DocReader_ReadFromFile_FullMethodName = "/docreader.DocReader/ReadFromFile" + DocReader_ReadFromURL_FullMethodName = "/docreader.DocReader/ReadFromURL" +) + +// DocReaderClient is the client API for DocReader service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// 文档读取服务 +type DocReaderClient interface { + // 从文件读取文档 + ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error) + // 从URL读取文档 + ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error) +} + +type docReaderClient struct { + cc grpc.ClientConnInterface +} + +func NewDocReaderClient(cc grpc.ClientConnInterface) DocReaderClient { + return &docReaderClient{cc} +} + +func (c *docReaderClient) ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReadResponse) + err := c.cc.Invoke(ctx, DocReader_ReadFromFile_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReadResponse) + err := c.cc.Invoke(ctx, DocReader_ReadFromURL_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// DocReaderServer is the server API for DocReader service. +// All implementations must embed UnimplementedDocReaderServer +// for forward compatibility. +// +// 文档读取服务 +type DocReaderServer interface { + // 从文件读取文档 + ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error) + // 从URL读取文档 + ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error) + mustEmbedUnimplementedDocReaderServer() +} + +// UnimplementedDocReaderServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedDocReaderServer struct{} + +func (UnimplementedDocReaderServer) ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReadFromFile not implemented") +} +func (UnimplementedDocReaderServer) ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReadFromURL not implemented") +} +func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {} +func (UnimplementedDocReaderServer) testEmbeddedByValue() {} + +// UnsafeDocReaderServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DocReaderServer will +// result in compilation errors. +type UnsafeDocReaderServer interface { + mustEmbedUnimplementedDocReaderServer() +} + +func RegisterDocReaderServer(s grpc.ServiceRegistrar, srv DocReaderServer) { + // If the following call pancis, it indicates UnimplementedDocReaderServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&DocReader_ServiceDesc, srv) +} + +func _DocReader_ReadFromFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReadFromFileRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DocReaderServer).ReadFromFile(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DocReader_ReadFromFile_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DocReaderServer).ReadFromFile(ctx, req.(*ReadFromFileRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DocReader_ReadFromURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReadFromURLRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DocReaderServer).ReadFromURL(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: DocReader_ReadFromURL_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DocReaderServer).ReadFromURL(ctx, req.(*ReadFromURLRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// DocReader_ServiceDesc is the grpc.ServiceDesc for DocReader service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DocReader_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "docreader.DocReader", + HandlerType: (*DocReaderServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ReadFromFile", + Handler: _DocReader_ReadFromFile_Handler, + }, + { + MethodName: "ReadFromURL", + Handler: _DocReader_ReadFromURL_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "docreader.proto", +} diff --git a/services/docreader/src/proto/docreader_pb2_grpc.py b/services/docreader/src/proto/docreader_pb2_grpc.py new file mode 100644 index 0000000..76780b3 --- /dev/null +++ b/services/docreader/src/proto/docreader_pb2_grpc.py @@ -0,0 +1,145 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc +import warnings + +import docreader_pb2 as docreader__pb2 + +GRPC_GENERATED_VERSION = '1.74.0' +GRPC_VERSION = grpc.__version__ +_version_not_supported = False + +try: + from grpc._utilities import first_version_is_lower + _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) +except ImportError: + _version_not_supported = True + +if _version_not_supported: + raise RuntimeError( + f'The grpc package installed is at version {GRPC_VERSION},' + + f' but the generated code in docreader_pb2_grpc.py depends on' + + f' grpcio>={GRPC_GENERATED_VERSION}.' + + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' + ) + + +class DocReaderStub(object): + """文档读取服务 + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.ReadFromFile = channel.unary_unary( + '/docreader.DocReader/ReadFromFile', + request_serializer=docreader__pb2.ReadFromFileRequest.SerializeToString, + response_deserializer=docreader__pb2.ReadResponse.FromString, + _registered_method=True) + self.ReadFromURL = channel.unary_unary( + '/docreader.DocReader/ReadFromURL', + request_serializer=docreader__pb2.ReadFromURLRequest.SerializeToString, + response_deserializer=docreader__pb2.ReadResponse.FromString, + _registered_method=True) + + +class DocReaderServicer(object): + """文档读取服务 + """ + + def ReadFromFile(self, request, context): + """从文件读取文档 + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ReadFromURL(self, request, context): + """从URL读取文档 + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_DocReaderServicer_to_server(servicer, server): + rpc_method_handlers = { + 'ReadFromFile': grpc.unary_unary_rpc_method_handler( + servicer.ReadFromFile, + request_deserializer=docreader__pb2.ReadFromFileRequest.FromString, + response_serializer=docreader__pb2.ReadResponse.SerializeToString, + ), + 'ReadFromURL': grpc.unary_unary_rpc_method_handler( + servicer.ReadFromURL, + request_deserializer=docreader__pb2.ReadFromURLRequest.FromString, + response_serializer=docreader__pb2.ReadResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'docreader.DocReader', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('docreader.DocReader', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class DocReader(object): + """文档读取服务 + """ + + @staticmethod + def ReadFromFile(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/docreader.DocReader/ReadFromFile', + docreader__pb2.ReadFromFileRequest.SerializeToString, + docreader__pb2.ReadResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + + @staticmethod + def ReadFromURL(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/docreader.DocReader/ReadFromURL', + docreader__pb2.ReadFromURLRequest.SerializeToString, + docreader__pb2.ReadResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/services/docreader/src/server/server.py b/services/docreader/src/server/server.py new file mode 100644 index 0000000..11a7b96 --- /dev/null +++ b/services/docreader/src/server/server.py @@ -0,0 +1,378 @@ +import os +import sys +import logging +from concurrent import futures +import traceback +import grpc +import uuid +import atexit +from grpc_health.v1 import health_pb2_grpc +from grpc_health.v1.health import HealthServicer + +# Add parent directory to Python path +current_dir = os.path.dirname(os.path.abspath(__file__)) +parent_dir = os.path.dirname(current_dir) +if parent_dir not in sys.path: + sys.path.insert(0, parent_dir) + +from proto.docreader_pb2 import ReadResponse, Chunk, Image +from proto import docreader_pb2_grpc +from parser import Parser, OCREngine +from parser.config import ChunkingConfig +from utils.request import request_id_context, init_logging_request_id + +# --- Encoding utilities: sanitize strings to valid UTF-8 and (optionally) multi-encoding read --- +import re +from typing import Optional + +try: + # Optional dependency for charset detection; install via `pip install charset-normalizer` + from charset_normalizer import from_bytes as _cn_from_bytes # type: ignore +except Exception: # pragma: no cover + _cn_from_bytes = None # type: ignore + +# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values and cannot be encoded to UTF-8 +_SURROGATE_RE = re.compile(r"[\ud800-\udfff]") + +def to_valid_utf8_text(s: Optional[str]) -> str: + """Return a UTF-8 safe string for protobuf. + + - Replace any surrogate code points with U+FFFD + - Re-encode with errors='replace' to ensure valid UTF-8 + """ + if not s: + return "" + s = _SURROGATE_RE.sub("\uFFFD", s) + return s.encode("utf-8", errors="replace").decode("utf-8") + +def read_text_with_fallback(file_path: str) -> str: + """Read text from file supporting multiple encodings with graceful fallback. + + This server currently receives bytes over gRPC and delegates decoding to the parser. + This helper is provided for future local-file reads if needed. + """ + with open(file_path, "rb") as f: + raw = f.read() + if _cn_from_bytes is not None: + try: + result = _cn_from_bytes(raw).best() + if result: + return str(result) + except Exception: + pass + for enc in ("utf-8", "gb18030", "latin-1"): + try: + return raw.decode(enc, errors="replace") + except UnicodeDecodeError: + continue + return raw.decode("utf-8", errors="replace") + +# Ensure no existing handlers +for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + +# Configure logging - use stdout +handler = logging.StreamHandler(sys.stdout) +logging.root.addHandler(handler) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logger.info("Initializing server logging") + +# Initialize request ID logging +init_logging_request_id() + +# Set max message size to 50MB +MAX_MESSAGE_LENGTH = 50 * 1024 * 1024 + + +parser = Parser() + +class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer): + def __init__(self): + super().__init__() + self.parser = Parser() + + def ReadFromFile(self, request, context): + # Get or generate request ID + request_id = ( + request.request_id + if hasattr(request, "request_id") and request.request_id + else str(uuid.uuid4()) + ) + + # Use request ID context + with request_id_context(request_id): + try: + # Get file type + file_type = ( + request.file_type or os.path.splitext(request.file_name)[1][1:] + ) + logger.info( + f"Received ReadFromFile request for file: {request.file_name}, type: {file_type}" + ) + logger.info(f"File content size: {len(request.file_content)} bytes") + + # Create chunking config + chunk_size = request.read_config.chunk_size or 512 + chunk_overlap = request.read_config.chunk_overlap or 50 + separators = request.read_config.separators or ["\n\n", "\n", "。"] + enable_multimodal = request.read_config.enable_multimodal or False + + logger.info( + f"Using chunking config: size={chunk_size}, overlap={chunk_overlap}, " + f"multimodal={enable_multimodal}" + ) + + # Get Storage and VLM config from request + storage_config = None + vlm_config = None + + sc = request.read_config.storage_config + # Keep parser-side key name as cos_config for backward compatibility + storage_config = { + 'provider': 'minio' if sc.provider == 2 else 'cos', + 'region': sc.region, + 'bucket_name': sc.bucket_name, + 'access_key_id': sc.access_key_id, + 'secret_access_key': sc.secret_access_key, + 'app_id': sc.app_id, + 'path_prefix': sc.path_prefix, + } + logger.info(f"Using Storage config: provider={storage_config.get('provider')}, bucket={storage_config['bucket_name']}") + + vlm_config = { + 'model_name': request.read_config.vlm_config.model_name, + 'base_url': request.read_config.vlm_config.base_url, + 'api_key': request.read_config.vlm_config.api_key or '', + 'interface_type': request.read_config.vlm_config.interface_type or 'openai', + } + logger.info(f"Using VLM config: model={vlm_config['model_name']}, " + f"base_url={vlm_config['base_url']}, " + f"interface_type={vlm_config['interface_type']}") + + chunking_config = ChunkingConfig( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + separators=separators, + enable_multimodal=enable_multimodal, + storage_config=storage_config, + vlm_config=vlm_config, + ) + + # Parse file + logger.info(f"Starting file parsing process") + result = self.parser.parse_file( + request.file_name, file_type, request.file_content, chunking_config + ) + + if not result: + error_msg = "Failed to parse file" + logger.error(error_msg) + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(error_msg) + return ReadResponse() + + # Convert to protobuf message + logger.info( + f"Successfully parsed file {request.file_name}, returning {len(result.chunks)} chunks" + ) + + # Build response, including image info + response = ReadResponse( + chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks] + ) + logger.info(f"Response size: {response.ByteSize()} bytes") + return response + + except Exception as e: + error_msg = f"Error reading file: {str(e)}" + logger.error(error_msg) + logger.info(f"Detailed traceback: {traceback.format_exc()}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + return ReadResponse(error=str(e)) + + def ReadFromURL(self, request, context): + # Get or generate request ID + request_id = ( + request.request_id + if hasattr(request, "request_id") and request.request_id + else str(uuid.uuid4()) + ) + + # Use request ID context + with request_id_context(request_id): + try: + logger.info(f"Received ReadFromURL request for URL: {request.url}") + + # Create chunking config + chunk_size = request.read_config.chunk_size or 512 + chunk_overlap = request.read_config.chunk_overlap or 50 + separators = request.read_config.separators or ["\n\n", "\n", "。"] + enable_multimodal = request.read_config.enable_multimodal or False + + logger.info( + f"Using chunking config: size={chunk_size}, overlap={chunk_overlap}, " + f"multimodal={enable_multimodal}" + ) + + # Get Storage and VLM config from request + storage_config = None + vlm_config = None + + sc = request.read_config.storage_config + storage_config = { + 'provider': 'minio' if sc.provider == 2 else 'cos', + 'region': sc.region, + 'bucket_name': sc.bucket_name, + 'access_key_id': sc.access_key_id, + 'secret_access_key': sc.secret_access_key, + 'app_id': sc.app_id, + 'path_prefix': sc.path_prefix, + } + logger.info(f"Using Storage config: provider={storage_config.get('provider')}, bucket={storage_config['bucket_name']}") + + vlm_config = { + 'model_name': request.read_config.vlm_config.model_name, + 'base_url': request.read_config.vlm_config.base_url, + 'api_key': request.read_config.vlm_config.api_key or '', + 'interface_type': request.read_config.vlm_config.interface_type or 'openai', + } + logger.info(f"Using VLM config: model={vlm_config['model_name']}, " + f"base_url={vlm_config['base_url']}, " + f"interface_type={vlm_config['interface_type']}") + + chunking_config = ChunkingConfig( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + separators=separators, + enable_multimodal=enable_multimodal, + storage_config=storage_config, + vlm_config=vlm_config, + ) + + # Parse URL + logger.info(f"Starting URL parsing process") + result = self.parser.parse_url(request.url, request.title, chunking_config) + if not result: + error_msg = "Failed to parse URL" + logger.error(error_msg) + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(error_msg) + return ReadResponse(error=error_msg) + + # Convert to protobuf message, including image info and title + logger.info( + f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks, title: {result.title}" + ) + + response = ReadResponse( + chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks], + title=result.title if result.title else "" + ) + logger.info(f"Response size: {response.ByteSize()} bytes") + return response + + except Exception as e: + error_msg = f"Error reading URL: {str(e)}" + logger.error(error_msg) + logger.info(f"Detailed traceback: {traceback.format_exc()}") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + return ReadResponse(error=str(e)) + + def _convert_chunk_to_proto(self, chunk): + """Convert internal Chunk object to protobuf Chunk message + Ensures all string fields are valid UTF-8 for protobuf (no lone surrogates). + """ + # Clean helper for strings + _c = to_valid_utf8_text + + proto_chunk = Chunk( + content=_c(getattr(chunk, "content", None)), + seq=getattr(chunk, "seq", 0), + start=getattr(chunk, "start", 0), + end=getattr(chunk, "end", 0), + ) + + # If chunk has images attribute and is not empty, add image info + if hasattr(chunk, "images") and chunk.images: + logger.info(f"Adding {len(chunk.images)} images to chunk {getattr(chunk, 'seq', 0)}") + for img_info in chunk.images: + # img_info expected as dict + proto_image = Image( + url=_c(img_info.get("cos_url", "")), + caption=_c(img_info.get("caption", "")), + ocr_text=_c(img_info.get("ocr_text", "")), + original_url=_c(img_info.get("original_url", "")), + start=int(img_info.get("start", 0) or 0), + end=int(img_info.get("end", 0) or 0), + ) + proto_chunk.images.append(proto_image) + + return proto_chunk + +def init_ocr_engine(ocr_backend, ocr_config): + """Initialize OCR engine""" + try: + logger.info(f"Initializing OCR engine with backend: {ocr_backend}") + ocr_engine = OCREngine.get_instance(backend_type=ocr_backend, **ocr_config) + if ocr_engine: + logger.info("OCR engine initialized successfully") + return True + else: + logger.error("OCR engine initialization failed") + return False + except Exception as e: + logger.error(f"Error initializing OCR engine: {str(e)}") + return False + + +def serve(): + + init_ocr_engine(os.getenv("OCR_BACKEND", "paddle"), { + "OCR_API_BASE_URL": os.getenv("OCR_API_BASE_URL", ""), + }) + + # Set max number of worker threads + max_workers = int(os.environ.get("GRPC_MAX_WORKERS", "4")) + logger.info(f"Starting DocReader service with {max_workers} worker threads") + + # Get port number + port = os.environ.get("GRPC_PORT", "50051") + + # Create server + server = grpc.server( + futures.ThreadPoolExecutor(max_workers=max_workers), + options=[ + ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH), + ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH), + ], + ) + + # Register services + docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server) + + # Register health check service + health_servicer = HealthServicer() + health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server) + + # Set listen address + server.add_insecure_port(f"[::]:{port}") + + # Start service + server.start() + + logger.info(f"Server started on port {port}") + logger.info("Server is ready to accept connections") + + try: + # Wait for service termination + server.wait_for_termination() + except KeyboardInterrupt: + logger.info("Received termination signal, shutting down server") + server.stop(0) + +if __name__ == "__main__": + serve() diff --git a/services/docreader/src/testdata/images/test_text.png b/services/docreader/src/testdata/images/test_text.png new file mode 100644 index 0000000..2d3dd01 Binary files /dev/null and b/services/docreader/src/testdata/images/test_text.png differ diff --git a/services/docreader/src/testdata/test.html b/services/docreader/src/testdata/test.html new file mode 100644 index 0000000..f0cce25 --- /dev/null +++ b/services/docreader/src/testdata/test.html @@ -0,0 +1,56 @@ +<!DOCTYPE html> +<html lang="zh-CN"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>测试 HTML 文档 + + +

测试 HTML 文档

+ +

这是一个测试 HTML 文档，用于测试 HTML 解析功能。

+ +

包含图片

+ 测试图片 + +

包含链接

+

这是一个测试链接。

+ +

包含代码块

+ 

+def hello_world():
+    print("Hello, World!")
+
+ +

包含表格

+ + + + + + + + + + + + + + + + + +
表头1表头2
内容1内容2
内容3内容4
+ +

测试分块功能

+

这部分内容用于测试分块功能，确保 HTML 结构在分块时保持完整。

+
    +
  • 第一块内容
    • +
  • 第二块内容
    • +
  • 第三块内容
    • +
+ +

测试重叠功能

+

这部分内容可能会在分块时与前后块重叠，以确保上下文的连续性。

+ + \ No newline at end of file diff --git a/services/docreader/src/testdata/test.md b/services/docreader/src/testdata/test.md new file mode 100644 index 0000000..bc51c4b --- /dev/null +++ b/services/docreader/src/testdata/test.md @@ -0,0 +1,37 @@ +# 测试 Markdown 文档 + +这是一个测试 Markdown 文档，用于测试 Markdown 解析功能。 + +## 包含图片 + +![测试图片](https://geektutu.com/post/quick-go-protobuf/go-protobuf.jpg) + +## 包含链接 + +这是一个[测试链接](https://example.com)。 + +## 包含代码块 + +```python +def hello_world(): + print("Hello, World!") +``` + +## 包含表格 + +| 表头1 | 表头2 | +|-------|-------| +| 内容1 | 内容2 | +| 内容3 | 内容4 | + +## 测试分块功能 + +这部分内容用于测试分块功能，确保 Markdown 结构在分块时保持完整。 + +- 第一块内容 +- 第二块内容 +- 第三块内容 + +## 测试重叠功能 + +这部分内容可能会在分块时与前后块重叠，以确保上下文的连续性。 \ No newline at end of file diff --git a/services/docreader/src/testdata/test.txt b/services/docreader/src/testdata/test.txt new file mode 100644 index 0000000..bf6be04 --- /dev/null +++ b/services/docreader/src/testdata/test.txt @@ -0,0 +1,16 @@ +这是一个测试文档 +包含多行内容 +用于测试文档解析功能 + +这个文档包含以下内容： +1. 基本文本内容 +2. 多行段落 +3. 列表项 + +测试分块功能： +- 第一块内容 +- 第二块内容 +- 第三块内容 + +测试重叠功能： +这部分内容可能会在分块时与前后块重叠，以确保上下文的连续性。 \ No newline at end of file diff --git a/services/docreader/src/testdata/test_download.txt b/services/docreader/src/testdata/test_download.txt new file mode 100644 index 0000000..67bca7a --- /dev/null +++ b/services/docreader/src/testdata/test_download.txt @@ -0,0 +1,19 @@ +这是一个测试文档 +包含多行内容 +用于测试文档解析功能 + +这个文档包含以下内容： +1. 基本文本内容 +2. 多行段落 +3. 列表项 + +测试分块功能： +- 第一块内容 +- 第二块内容 +- 第三块内容 + +测试重叠功能： +这部分内容可能会在分块时与前后块重叠，以确保上下文的连续性。 + + +test \ No newline at end of file diff --git a/services/docreader/src/utils/__init__.py b/services/docreader/src/utils/__init__.py new file mode 100644 index 0000000..f1c7702 --- /dev/null +++ b/services/docreader/src/utils/__init__.py @@ -0,0 +1,83 @@ +# +# Copyright 2024 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re +import logging + +# 配置日志 +logger = logging.getLogger(__name__) + + +def singleton(cls, *args, **kw): + instances = {} + + def _singleton(): + key = str(cls) + str(os.getpid()) + if key not in instances: + logger.info(f"Creating new singleton instance with key: {key}") + instances[key] = cls(*args, **kw) + else: + logger.info(f"Returning existing singleton instance with key: {key}") + return instances[key] + + return _singleton + + +def rmSpace(txt): + logger.info(f"Removing spaces from text of length: {len(txt)}") + txt = re.sub(r"([^a-z0-9.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE) + return re.sub(r"([^ ]) +([^a-z0-9.,\(<])", r"\1\2", txt, flags=re.IGNORECASE) + + +def findMaxDt(fnm): + m = "1970-01-01 00:00:00" + logger.info(f"Finding maximum date in file: {fnm}") + try: + with open(fnm, "r") as f: + while True: + l = f.readline() + if not l: + break + l = l.strip("\n") + if l == "nan": + continue + if l > m: + m = l + logger.info(f"Maximum date found: {m}") + except Exception as e: + logger.error(f"Error reading file {fnm} for max date: {str(e)}") + return m + + +def findMaxTm(fnm): + m = 0 + logger.info(f"Finding maximum time in file: {fnm}") + try: + with open(fnm, "r") as f: + while True: + l = f.readline() + if not l: + break + l = l.strip("\n") + if l == "nan": + continue + if int(l) > m: + m = int(l) + logger.info(f"Maximum time found: {m}") + except Exception as e: + logger.error(f"Error reading file {fnm} for max time: {str(e)}") + return m diff --git a/services/docreader/src/utils/request.py b/services/docreader/src/utils/request.py new file mode 100644 index 0000000..867892c --- /dev/null +++ b/services/docreader/src/utils/request.py @@ -0,0 +1,149 @@ +from contextvars import ContextVar +import logging +import uuid +import contextlib +import time +from typing import Optional +from logging import LogRecord + +# 配置日志 +logger = logging.getLogger(__name__) + +# 定义上下文变量 +request_id_var = ContextVar("request_id", default=None) +_request_start_time_ctx = ContextVar("request_start_time", default=None) + + +def set_request_id(request_id: str) -> None: + """设置当前上下文的请求ID""" + request_id_var.set(request_id) + + +def get_request_id() -> Optional[str]: + """获取当前上下文的请求ID""" + return request_id_var.get() + + +class MillisecondFormatter(logging.Formatter): + """自定义日志格式化器，只显示毫秒级时间戳(3位数字)而不是微秒(6位)""" + + def formatTime(self, record, datefmt=None): + """重写formatTime方法，将微秒格式化为毫秒""" + # 先获取标准的格式化时间 + result = super().formatTime(record, datefmt) + + # 如果使用了包含.%f的格式，则将微秒(6位)截断为毫秒(3位) + if datefmt and ".%f" in datefmt: + # 格式化的时间字符串应该在最后有6位微秒数 + parts = result.split('.') + if len(parts) > 1 and len(parts[1]) >= 6: + # 只保留前3位作为毫秒 + millis = parts[1][:3] + result = f"{parts[0]}.{millis}" + + return result + + +def init_logging_request_id(): + """ + Initialize logging to include request ID in log messages. + Add the custom filter to all existing handlers + """ + logger.info("Initializing request ID logging") + root_logger = logging.getLogger() + + # 添加自定义过滤器到所有处理器 + for handler in root_logger.handlers: + # 添加请求ID过滤器 + handler.addFilter(RequestIdFilter()) + + # 更新格式化器以包含请求ID，调整格式使其更紧凑整齐 + formatter = logging.Formatter( + fmt="%(asctime)s.%(msecs)03d [%(request_id)s] %(levelname)-5s %(name)-20s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + handler.setFormatter(formatter) + + logger.info( + f"Updated {len(root_logger.handlers)} handlers with request ID formatting" + ) + + # 如果没有处理器，添加一个标准输出处理器 + if not root_logger.handlers: + handler = logging.StreamHandler() + formatter = logging.Formatter( + fmt="%(asctime)s.%(msecs)03d [%(request_id)s] %(levelname)-5s %(name)-20s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + handler.setFormatter(formatter) + handler.addFilter(RequestIdFilter()) + root_logger.addHandler(handler) + logger.info("Added new StreamHandler with request ID formatting") + + +class RequestIdFilter(logging.Filter): + """Filter that adds request ID to log messages""" + + def filter(self, record: LogRecord) -> bool: + request_id = request_id_var.get() + if request_id is not None: + # 为日志记录添加请求ID属性，使用短格式 + if len(request_id) > 8: + # 截取ID的前8个字符，确保显示整齐 + short_id = request_id[:8] + if "-" in request_id: + # 尝试保留格式，例如 test-req-1-XXX + parts = request_id.split("-") + if len(parts) >= 3: + # 如果格式是 xxx-xxx-n-randompart + short_id = f"{parts[0]}-{parts[1]}-{parts[2]}" + record.request_id = short_id + else: + record.request_id = request_id + + # 添加执行时间属性 + start_time = _request_start_time_ctx.get() + if start_time is not None: + elapsed_ms = int((time.time() - start_time) * 1000) + record.elapsed_ms = elapsed_ms + # 添加执行时间到消息中 + if not hasattr(record, "message_with_elapsed"): + record.message_with_elapsed = True + record.msg = f"{record.msg} (elapsed: {elapsed_ms}ms)" + else: + # 如果没有请求ID，使用占位符 + record.request_id = "no-req-id" + + return True + + +@contextlib.contextmanager +def request_id_context(request_id: str = None): + """Context manager that sets a request ID for the current context + + Args: + request_id: 要使用的请求ID，如果为None则自动生成 + + Example: + with request_id_context("req-123"): + # 在这个代码块中的所有日志都会包含请求ID req-123 + logging.info("Processing request") + """ + # Generate or use provided request ID + req_id = request_id or str(uuid.uuid4()) + + # Set start time and request ID + start_time = time.time() + req_token = request_id_var.set(req_id) + time_token = _request_start_time_ctx.set(start_time) + + logger.info(f"Starting new request with ID: {req_id}") + + try: + yield request_id_var.get() + finally: + # Log completion and reset context vars + elapsed_ms = int((time.time() - start_time) * 1000) + logger.info(f"Request {req_id} completed in {elapsed_ms}ms") + request_id_var.reset(req_token) + _request_start_time_ctx.reset(time_token) diff --git a/services/docreader/supervisord.conf b/services/docreader/supervisord.conf new file mode 100644 index 0000000..e0e2c63 --- /dev/null +++ b/services/docreader/supervisord.conf @@ -0,0 +1,32 @@ +[supervisord] +nodaemon=true +logfile=/var/log/supervisord.log +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=info +pidfile=/var/run/supervisord.pid +user=root + +[program:docreader] +command=/bin/sh -c "python /app/src/server/server.py 2>&1" +directory=/app +autostart=true +autorestart=true +startretries=5 +startsecs=5 +priority=10 +redirect_stderr=true +stdout_logfile=/var/log/docreader.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=10 +environment=PYTHONPATH="/app/src" + +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock diff --git "a/\344\272\247\345\223\201\346\226\207\346\241\243.md" "b/\344\272\247\345\223\201\346\226\207\346\241\243.md" deleted file mode 100644 index 629bc0e..0000000 --- "a/\344\272\247\345\223\201\346\226\207\346\241\243.md" +++ /dev/null @@ -1,664 +0,0 @@ -# C-Cube 智能客服系统产品文档 - -## 📋 目录 - -1. [产品概述](#产品概述) -2. [核心功能](#核心功能) -3. [技术架构](#技术架构) -4. [部署方案](#部署方案) -5. [使用指南](#使用指南) -6. [API接口文档](#api接口文档) -7. [安全与合规](#安全与合规) -8. [常见问题](#常见问题) - ---- - -## 产品概述 - -### 产品定位 - -C-Cube 智能客服系统是一款基于人工智能技术的企业级客服解决方案，旨在帮助企业提升客户服务效率，降低运营成本，提供7×24小时全天候智能客户服务。 - -### 产品愿景 - -通过AI技术赋能客户服务，让每一次客户互动都充满智慧，创造卓越的客户体验。 - -### 核心价值 - -- **智能化**：基于自然语言处理和机器学习技术，提供智能对话服务 -- **高效性**：自动处理80%以上的常规咨询，大幅降低人工客服压力 -- **全渠道**：支持网页、移动端、微信、企业微信等多渠道接入 -- **可扩展**：灵活的插件机制，支持业务定制化需求 -- **易集成**：提供完善的API接口，快速集成到现有业务系统 - ---- - -## 核心功能 - -### 1. 智能对话引擎 - -#### 1.1 自然语言理解（NLU） -- **意图识别**：准确识别用户咨询意图，支持300+预定义意图 -- **实体提取**：自动提取关键信息（如订单号、产品名称、时间等） -- **上下文管理**：支持多轮对话，理解上下文语境 -- **情感分析**：实时分析用户情绪，智能调整回复策略 - -#### 1.2 知识库管理 -- **智能问答**：基于知识图谱的精准问答 -- **相似问题推荐**：自动匹配相似问题，提高问答覆盖率 -- **知识更新**：支持在线编辑和批量导入 -- **版本控制**：知识库版本管理，支持回滚 - -#### 1.3 对话策略 -- **智能路由**：根据问题复杂度自动分配给AI或人工客服 -- **主动推送**：基于用户行为主动推送相关信息 -- **多语言支持**：支持中文、英文等10+语言 -- **个性化回复**：基于用户画像提供个性化服务 - -### 2. 人机协同 - -#### 2.1 智能辅助 -- **实时建议**：为人工客服提供实时回复建议 -- **快捷回复**：常用话术库，一键发送 -- **客户信息展示**：集成CRM，实时显示客户历史信息 - -#### 2.2 工单管理 -- **自动创建**：对话中自动生成工单 -- **工单流转**：支持多级审批和流转 -- **SLA监控**：实时监控服务水平协议 - -### 3. 多渠道接入 - -#### 3.1 支持渠道 -- 网页聊天窗口 -- 移动端APP -- 微信公众号/小程序 -- 企业微信 -- 钉钉 -- 电话语音（IVR集成） -- 邮件 - -#### 3.2 统一管理 -- 跨渠道会话同步 -- 统一客户身份识别 -- 一站式管理后台 - -### 4. 数据分析与报表 - -#### 4.1 实时监控 -- 会话量统计 -- 响应时间分析 -- 客服工作量统计 -- 系统性能监控 - -#### 4.2 智能分析 -- 热点问题分析 -- 客户满意度分析 -- 转化率分析 -- 趋势预测 - -#### 4.3 可视化报表 -- 自定义报表 -- 数据导出 -- 定时推送 - -### 5. 训练与优化 - -#### 5.1 机器学习 -- 持续学习：从人工客服对话中学习 -- 主动学习：识别低置信度问题，标注后强化 -- A/B测试：对比不同策略效果 - -#### 5.2 质量管理 -- 对话质检 -- 问题标注 -- 模型评估 - ---- - -## 技术架构 - -### 系统架构图 - -``` -┌─────────────────────────────────────────────────────────────┐ -│ 接入层 │ -│ Web │ APP │ 微信 │ 企业微信 │ 钉钉 │ 电话 │ 邮件 │ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ API网关层 │ -│ 负载均衡 │ 鉴权 │ 限流 │ 日志 │ 监控 │ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ 业务服务层 │ -├─────────────────────────────────────────────────────────────┤ -│ 对话管理 │ 会话管理 │ 用户管理 │ 权限管理 │ -├─────────────────────────────────────────────────────────────┤ -│ 工单系统 │ 知识库 │ 消息队列 │ 任务调度 │ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ AI服务层 │ -├─────────────────────────────────────────────────────────────┤ -│ NLU引擎 │ 对话管理 │ 知识图谱 │ 推荐引擎 │ -├─────────────────────────────────────────────────────────────┤ -│ 情感分析 │ 语音识别 │ 语音合成 │ 翻译服务 │ -└────────────────────┬────────────────────────────────────────┘ - │ -┌────────────────────▼────────────────────────────────────────┐ -│ 数据层 │ -│ MySQL │ Redis │ MongoDB │ Elasticsearch │ HDFS │ -└─────────────────────────────────────────────────────────────┘ -``` - -### 技术栈 - -#### 后端技术 -- **编程语言**：Go、Python -- **框架**：Gin（Go Web框架）、FastAPI（Python） -- **消息队列**：Kafka、RabbitMQ -- **缓存**：Redis -- **数据库**：MySQL、MongoDB、Elasticsearch -- **任务调度**：XXL-Job - -#### 前端技术 -- **框架**：React、Vue.js -- **UI组件**：Ant Design、Element UI -- **状态管理**：Redux、Vuex -- **实时通信**：WebSocket、Socket.io - -#### AI技术 -- **深度学习框架**：PyTorch、TensorFlow -- **NLP模型**：BERT、GPT、T5 -- **向量检索**：Faiss、Milvus -- **机器学习**：Scikit-learn、XGBoost - -#### DevOps -- **容器化**：Docker、Kubernetes -- **CI/CD**：Jenkins、GitLab CI -- **监控**：Prometheus、Grafana -- **日志**：ELK Stack（Elasticsearch、Logstash、Kibana） - ---- - -## 部署方案 - -### 1. 云端部署（推荐） - -#### 1.1 SaaS模式 -- 即开即用，无需部署 -- 按需付费，成本可控 -- 自动升级，持续优化 - -#### 1.2 私有云部署 -- 支持阿里云、腾讯云、AWS等主流云平台 -- 高可用架构，支持多区域部署 -- 弹性伸缩，自动应对流量高峰 - -### 2. 本地部署 - -#### 2.1 硬件要求 -- **最小配置**（500并发）： - - CPU：16核 - - 内存：64GB - - 存储：1TB SSD - - 网络：千兆网卡 - -- **推荐配置**（2000并发）： - - CPU：32核 - - 内存：128GB - - 存储：2TB SSD RAID - - 网络：万兆网卡 - -#### 2.2 软件要求 -- 操作系统：CentOS 7+、Ubuntu 18.04+ -- Docker：20.10+ -- Kubernetes：1.20+（可选） - -### 3. 混合部署 - -结合云端和本地优势： -- 核心数据本地存储 -- AI计算云端执行 -- 灵活配置，安全可控 - ---- - -## 使用指南 - -### 快速开始 - -#### 1. 系统登录 -``` -访问地址：https://your-domain.com/admin -默认账号：admin -默认密码：admin123（首次登录请修改） -``` - -#### 2. 创建机器人 - -**步骤：** -1. 进入「机器人管理」页面 -2. 点击「新建机器人」 -3. 填写基本信息： - - 机器人名称 - - 欢迎语 - - 默认回复 -4. 配置知识库 -5. 设置对话策略 -6. 保存并发布 - -#### 3. 集成到网站 - -**HTML集成代码：** -```html - -``` - -#### 4. 管理知识库 - -**添加问答：** -1. 进入「知识库管理」 -2. 点击「添加问答」 -3. 输入标准问题 -4. 添加相似问题（可选） -5. 编辑答案（支持富文本、图片、链接） -6. 设置生效时间 -7. 保存 - -**批量导入：** -1. 下载模板（Excel格式） -2. 填写问答内容 -3. 上传文件 -4. 系统自动解析并导入 - -#### 5. 客服工作台 - -**接待客户：** -1. 登录客服工作台 -2. 查看待处理会话列表 -3. 点击会话进入对话界面 -4. 使用快捷回复或自定义回复 -5. 完成后标记会话状态 - -**查看客户信息：** -- 右侧面板显示客户基本信息 -- 历史对话记录 -- 订单信息（需集成CRM） - ---- - -## API接口文档 - -### 1. 认证 - -所有API请求需要在Header中携带认证信息： - -```http -Authorization: Bearer YOUR_API_KEY -Content-Type: application/json -``` - -### 2. 发送消息 - -**接口**：`POST /api/v1/chat/send` - -**请求参数：** -```json -{ - "bot_id": "bot_123456", - "user_id": "user_789", - "message": "我想查询订单状态", - "session_id": "session_abc", - "metadata": { - "channel": "web", - "ip": "192.168.1.1" - } -} -``` - -**响应示例：** -```json -{ - "code": 0, - "message": "success", - "data": { - "reply": "请提供您的订单号，我帮您查询。", - "session_id": "session_abc", - "intent": "order_query", - "confidence": 0.95, - "need_human": false - } -} -``` - -### 3. 查询会话历史 - -**接口**：`GET /api/v1/chat/history` - -**请求参数：** -``` -session_id: 会话ID -page: 页码（默认1） -page_size: 每页数量（默认20） -``` - -**响应示例：** -```json -{ - "code": 0, - "message": "success", - "data": { - "total": 50, - "messages": [ - { - "id": "msg_001", - "sender": "user", - "content": "你好", - "timestamp": "2024-01-15T10:30:00Z" - }, - { - "id": "msg_002", - "sender": "bot", - "content": "您好！有什么可以帮您？", - "timestamp": "2024-01-15T10:30:01Z" - } - ] - } -} -``` - -### 4. 创建工单 - -**接口**：`POST /api/v1/ticket/create` - -**请求参数：** -```json -{ - "user_id": "user_789", - "title": "产品使用问题", - "content": "功能X无法正常使用", - "priority": "medium", - "category": "technical", - "attachments": ["file_url_1", "file_url_2"] -} -``` - -**响应示例：** -```json -{ - "code": 0, - "message": "success", - "data": { - "ticket_id": "TK20240115001", - "status": "open", - "created_at": "2024-01-15T10:35:00Z" - } -} -``` - -### 5. 知识库查询 - -**接口**：`POST /api/v1/knowledge/search` - -**请求参数：** -```json -{ - "query": "如何退款", - "top_k": 5, - "threshold": 0.7 -} -``` - -**响应示例：** -```json -{ - "code": 0, - "message": "success", - "data": { - "results": [ - { - "question": "如何申请退款？", - "answer": "您可以在订单详情页点击退款按钮...", - "score": 0.92, - "category": "售后服务" - } - ] - } -} -``` - -### 错误码说明 - -| 错误码 | 说明 | -|--------|------| -| 0 | 成功 | -| 1001 | 参数错误 | -| 1002 | 认证失败 | -| 1003 | 权限不足 | -| 1004 | 资源不存在 | -| 1005 | 请求频率超限 | -| 5000 | 服务器内部错误 | - ---- - -## 安全与合规 - -### 1. 数据安全 - -#### 1.1 数据加密 -- **传输加密**：全站HTTPS，TLS 1.3 -- **存储加密**：敏感数据AES-256加密 -- **密钥管理**：独立密钥管理系统（KMS） - -#### 1.2 数据隔离 -- 多租户数据物理隔离 -- 访问权限细粒度控制 -- 数据访问审计日志 - -#### 1.3 数据备份 -- 自动每日全量备份 -- 实时增量备份 -- 异地容灾备份 - -### 2. 权限管理 - -#### 2.1 角色权限 -- **超级管理员**：所有权限 -- **管理员**：业务配置、数据查看 -- **客服主管**：客服管理、报表查看 -- **客服人员**：接待客户、工单处理 -- **只读用户**：仅查看权限 - -#### 2.2 操作审计 -- 所有操作记录日志 -- 敏感操作二次确认 -- 异常行为告警 - -### 3. 合规认证 - -- ISO 27001 信息安全管理体系认证 -- 等保三级认证 -- GDPR 数据保护合规 -- SOC 2 Type II 认证 - -### 4. 隐私保护 - -- 用户数据匿名化处理 -- 支持数据删除请求 -- 隐私政策透明公开 -- 定期隐私影响评估 - ---- - -## 常见问题 - -### Q1：如何提高机器人回答准确率？ - -**A：** -1. **丰富知识库**：添加更多高质量问答 -2. **标注相似问题**：为每个问题添加多个相似表达 -3. **持续训练**：定期标注未识别问题，重新训练模型 -4. **优化意图**：根据实际对话调整意图分类 -5. **启用学习模式**：让系统从人工客服对话中学习 - -### Q2：如何设置人机协同策略？ - -**A：** -在「对话策略」中配置： -- **置信度阈值**：低于0.7转人工 -- **敏感词触发**：包含特定关键词转人工 -- **意图匹配**：特定意图（如投诉）直接转人工 -- **用户选择**：提供"转人工"按钮 -- **工作时间**：非工作时间自动留言 - -### Q3：支持多语言吗？ - -**A：** -支持，目前支持： -- 中文（简体/繁体） -- 英文 -- 日语 -- 韩语 -- 西班牙语 -- 法语 -- 德语 -- 阿拉伯语 -- 俄语 -- 泰语 - -可在机器人设置中启用多语言模式，系统自动识别用户语言。 - -### Q4：如何集成现有的CRM系统？ - -**A：** -提供三种集成方式： -1. **API集成**：通过REST API实时同步数据 -2. **Webhook**：配置Webhook接收事件通知 -3. **数据库直连**：支持主流数据库直接连接（需评估安全性） - -详细集成文档请参考《系统集成指南》。 - -### Q5：并发量大时如何保证性能？ - -**A：** -系统采用分布式架构，支持水平扩展： -1. **负载均衡**：自动分发请求到多个节点 -2. **缓存机制**：热点数据Redis缓存 -3. **消息队列**：异步处理非实时任务 -4. **数据库优化**：读写分离、分库分表 -5. **CDN加速**：静态资源全球加速 - -建议： -- 1000并发以下：单机部署 -- 1000-5000并发：集群部署（3-5节点） -- 5000+并发：Kubernetes弹性部署 - -### Q6：机器人可以学习吗？ - -**A：** -支持多种学习模式： -1. **监督学习**：从标注数据中学习 -2. **迁移学习**：从人工客服对话中学习 -3. **强化学习**：根据用户反馈优化回复策略 -4. **主动学习**：自动识别需要标注的问题 - -### Q7：如何评估机器人效果？ - -**A：** -系统提供多维度评估指标： -- **识别准确率**：意图识别准确率 -- **回答准确率**：答案匹配准确率 -- **解决率**：机器人独立解决问题的比例 -- **转人工率**：需要转人工的比例 -- **满意度**：用户满意度评分 -- **响应时间**：平均响应时间 - -建议每月进行一次全面评估，持续优化。 - -### Q8：系统支持私有化部署吗？ - -**A：** -完全支持私有化部署，提供： -- 完整源代码（企业版） -- 部署文档和工具 -- 技术支持和培训 -- 定期版本升级 - -私有化部署流程： -1. 需求评估 -2. 硬件准备 -3. 系统安装 -4. 数据迁移 -5. 测试验收 -6. 培训交付 - -### Q9：如何保证系统稳定性？ - -**A：** -多重保障措施： -1. **高可用架构**：多节点部署，无单点故障 -2. **健康检查**：实时监控服务状态 -3. **自动恢复**：故障自动切换和恢复 -4. **限流保护**：防止流量过载 -5. **降级机制**：极端情况下保证核心功能 -6. **7×24监控**：专业运维团队值守 - -SLA承诺：99.9%可用性（企业版） - -### Q10：费用是如何计算的？ - -**A：** -提供灵活的计费方式： - -**SaaS版本：** -- 基础版：¥999/月（1000会话/月） -- 标准版：¥2999/月（5000会话/月） -- 专业版：¥5999/月（20000会话/月） -- 企业版：定制报价 - -**私有化部署：** -- License费用 -- 实施服务费 -- 年度维护费 - -详细报价请联系销售团队。 - ---- - -## 联系我们 - -- **官方网站**：https://www.c-cube.ai -- **技术支持**：support@c-cube.ai -- **商务咨询**：sales@c-cube.ai -- **电话**：400-XXX-XXXX -- **地址**：北京市朝阳区XXX大厦XX层 - ---- - -## 版本历史 - -| 版本 | 日期 | 说明 | -|------|------|------| -| v1.0 | 2024-01-15 | 初始版本发布 | - ---- - -**版权所有 © 2024 C-Cube 智能客服系统**