From 585dd8f4e13129e2d8ba1f8337b3eeb130fa8bdb Mon Sep 17 00:00:00 2001 From: xgopilot Date: Sun, 26 Oct 2025 10:38:18 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20=E4=BB=8E=20URL=20=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E7=9F=A5=E8=AF=86=E6=97=B6=E8=87=AA=E5=8A=A8=E6=8F=90?= =?UTF-8?q?=E5=8F=96=20HTML=20=E6=A0=87=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 问题 当前从 URL 创建知识时,生成的 knowledge 没有标题,影响搜索效率 ## 解决方案 1. 扩展 protobuf 定义,在 ReadResponse 中添加 title 字段 2. WebParser 自动从 HTML 提取标题: - 优先使用 标签 - 其次使用 <h1> 标签 - 最后使用默认值 "Untitled Web Page" 3. Go 服务从响应中获取并更新知识标题 ## 修改文件 - services/docreader/src/proto/docreader.proto: 添加 title 字段 - services/docreader/src/parser/base_parser.py: ParseResult 添加 title 支持 - services/docreader/src/parser/web_parser.py: 保存提取的标题 - services/docreader/src/proto/docreader.pb.go: 更新 Go protobuf 定义 - services/docreader/src/server/server.py: 返回提取的标题 - internal/application/service/knowledge.go: 使用提取的标题更新知识记录 Fixes #30 🤖 Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: cn0809 <63532205+cn0809@users.noreply.github.com> --- internal/application/service/knowledge.go | 6 ++++++ services/docreader/src/parser/base_parser.py | 5 ++++- services/docreader/src/parser/web_parser.py | 3 +++ services/docreader/src/proto/docreader.pb.go | 8 ++++++++ services/docreader/src/proto/docreader.proto | 1 + services/docreader/src/server/server.py | 7 ++++--- 6 files changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 7b8c26e..8aa6a62 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -833,6 +833,12 @@ func (s *knowledgeService) processDocumentFromURL(ctx context.Context, return } + // Update knowledge title if extracted from HTML + if resp.GetTitle() != "" && knowledge.Title == "" { + knowledge.Title = resp.GetTitle() + logger.GetLogger(ctx).Infof("Extracted title from URL: %s", knowledge.Title) + } + // Process and store chunks s.processChunks(ctx, kb, knowledge, resp.Chunks) } diff --git a/services/docreader/src/parser/base_parser.py b/services/docreader/src/parser/base_parser.py index 052bc82..e3af322 100644 --- a/services/docreader/src/parser/base_parser.py +++ b/services/docreader/src/parser/base_parser.py @@ -58,6 +58,7 @@ class ParseResult: text: str # Extracted text content chunks: Optional[List[Chunk]] = None # Chunk results + title: Optional[str] = None # Document title (for URLs) class BaseParser(ABC): @@ -672,7 +673,9 @@ def parse(self, content: bytes) -> ParseResult: f"Skipping image processing for unsupported file type: {file_ext}" ) - return ParseResult(text=text, chunks=chunks) + # Include title if available (for WebParser) + title = getattr(self, 'extracted_title', None) + return ParseResult(text=text, chunks=chunks, title=title) def _split_into_units(self, text: str) -> List[str]: """ diff --git a/services/docreader/src/parser/web_parser.py b/services/docreader/src/parser/web_parser.py index 44c883a..2d53383 100644 --- a/services/docreader/src/parser/web_parser.py +++ b/services/docreader/src/parser/web_parser.py @@ -15,6 +15,7 @@ class WebParser(BaseParser): def __init__(self, title: str, **kwargs): self.title = title + self.extracted_title = None # Store extracted title from HTML self.proxy = os.environ.get("WEB_PROXY", "") super().__init__(file_name=title, **kwargs) logger.info(f"Initialized WebParser with title: {title}") @@ -110,6 +111,8 @@ def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any logger.info("No title found, using default") logger.info(f"Web page title: {title}") + # Store the extracted title for later use + self.extracted_title = title.strip() if title else "" text = "\n".join( (line.strip() for line in text.splitlines() if line.strip()) ) diff --git a/services/docreader/src/proto/docreader.pb.go b/services/docreader/src/proto/docreader.pb.go index 95199af..cc917b3 100644 --- a/services/docreader/src/proto/docreader.pb.go +++ b/services/docreader/src/proto/docreader.pb.go @@ -629,6 +629,7 @@ type ReadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Chunks []*Chunk `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"` // 文档分块 Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` // 错误信息 + Title string `protobuf:"bytes,3,opt,name=title,proto3" json:"title,omitempty"` // 文档标题 (仅从URL读取时返回) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -677,6 +678,13 @@ func (x *ReadResponse) GetError() string { return "" } +func (x *ReadResponse) GetTitle() string { + if x != nil { + return x.Title + } + return "" +} + var File_docreader_proto protoreflect.FileDescriptor const file_docreader_proto_rawDesc = "" + diff --git a/services/docreader/src/proto/docreader.proto b/services/docreader/src/proto/docreader.proto index 44ff4ac..b799c82 100644 --- a/services/docreader/src/proto/docreader.proto +++ b/services/docreader/src/proto/docreader.proto @@ -86,4 +86,5 @@ message Chunk { message ReadResponse { repeated Chunk chunks = 1; // 文档分块 string error = 2; // 错误信息 + string title = 3; // 文档标题 (仅从URL读取时返回) } \ No newline at end of file diff --git a/services/docreader/src/server/server.py b/services/docreader/src/server/server.py index 9018d3e..11a7b96 100644 --- a/services/docreader/src/server/server.py +++ b/services/docreader/src/server/server.py @@ -262,13 +262,14 @@ def ReadFromURL(self, request, context): context.set_details(error_msg) return ReadResponse(error=error_msg) - # Convert to protobuf message, including image info + # Convert to protobuf message, including image info and title logger.info( - f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks" + f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks, title: {result.title}" ) response = ReadResponse( - chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks] + chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks], + title=result.title if result.title else "" ) logger.info(f"Response size: {response.ByteSize()} bytes") return response From 943eacf5188ff6c67a0e489d5b1338079998734d Mon Sep 17 00:00:00 2001 From: xgopilot <noreply@goplus.org> Date: Sun, 26 Oct 2025 11:19:20 +0000 Subject: [PATCH 2/4] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20URL=20=E6=A0=87?= =?UTF-8?q?=E9=A2=98=E6=9C=AA=E4=BF=9D=E5=AD=98=E5=88=B0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在 processDocumentFromURL 中提取到标题后立即调用 UpdateKnowledge 保存到数据库 Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: cn0809 <63532205+cn0809@users.noreply.github.com> --- internal/application/service/knowledge.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 8aa6a62..ccce5d9 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -837,6 +837,10 @@ func (s *knowledgeService) processDocumentFromURL(ctx context.Context, if resp.GetTitle() != "" && knowledge.Title == "" { knowledge.Title = resp.GetTitle() logger.GetLogger(ctx).Infof("Extracted title from URL: %s", knowledge.Title) + // Update database immediately + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to update knowledge title") + } } // Process and store chunks From e9de12b1d35b434a42f9873f1f6b32395bdc665b Mon Sep 17 00:00:00 2001 From: xgopilot <noreply@goplus.org> Date: Sun, 26 Oct 2025 11:35:05 +0000 Subject: [PATCH 3/4] =?UTF-8?q?debug:=20=E6=B7=BB=E5=8A=A0=E8=AF=A6?= =?UTF-8?q?=E7=BB=86=E6=97=A5=E5=BF=97=E4=BB=A5=E8=B0=83=E8=AF=95=20URL=20?= =?UTF-8?q?=E6=A0=87=E9=A2=98=E6=8F=90=E5=8F=96=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 processDocumentFromURL 中添加调试日志 - 记录 resp.GetTitle() 和 knowledge.Title 的值 - 记录数据库更新是否成功 - 帮助定位标题未保存的根本原因 Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: cn0809 <63532205+cn0809@users.noreply.github.com> --- internal/application/service/knowledge.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index ccce5d9..3f48df5 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -834,13 +834,18 @@ func (s *knowledgeService) processDocumentFromURL(ctx context.Context, } // Update knowledge title if extracted from HTML + logger.GetLogger(ctx).Infof("DEBUG: resp.GetTitle()='%s', knowledge.Title='%s'", resp.GetTitle(), knowledge.Title) if resp.GetTitle() != "" && knowledge.Title == "" { knowledge.Title = resp.GetTitle() logger.GetLogger(ctx).Infof("Extracted title from URL: %s", knowledge.Title) // Update database immediately if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to update knowledge title") + } else { + logger.GetLogger(ctx).Infof("DEBUG: Successfully updated knowledge title in database") } + } else { + logger.GetLogger(ctx).Infof("DEBUG: Skipping title update - condition not met") } // Process and store chunks From ac6317ac5e4abd580e653c5d1d1ff2ea3db6a470 Mon Sep 17 00:00:00 2001 From: xgopilot <noreply@goplus.org> Date: Sun, 26 Oct 2025 11:35:48 +0000 Subject: [PATCH 4/4] =?UTF-8?q?debug:=20=E5=9C=A8=20Python=20docreader=20?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E4=B8=AD=E6=B7=BB=E5=8A=A0=E6=A0=87=E9=A2=98?= =?UTF-8?q?=E6=8F=90=E5=8F=96=E8=B0=83=E8=AF=95=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 web_parser.py 中记录 extracted_title 的值 - 在 base_parser.py 中记录 ParseResult 的 title 字段 - 在 server.py 中记录发送给 Go 的响应 title - 帮助追踪整个标题提取和传递链路 Generated with [codeagent](https://github.com/qbox/codeagent) Co-authored-by: cn0809 <63532205+cn0809@users.noreply.github.com> --- services/docreader/src/parser/base_parser.py | 1 + services/docreader/src/parser/web_parser.py | 1 + services/docreader/src/server/server.py | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/services/docreader/src/parser/base_parser.py b/services/docreader/src/parser/base_parser.py index e3af322..3af42b9 100644 --- a/services/docreader/src/parser/base_parser.py +++ b/services/docreader/src/parser/base_parser.py @@ -675,6 +675,7 @@ def parse(self, content: bytes) -> ParseResult: # Include title if available (for WebParser) title = getattr(self, 'extracted_title', None) + logger.info(f"DEBUG: Creating ParseResult with title='{title}'") return ParseResult(text=text, chunks=chunks, title=title) def _split_into_units(self, text: str) -> List[str]: diff --git a/services/docreader/src/parser/web_parser.py b/services/docreader/src/parser/web_parser.py index 2d53383..bf8ebd0 100644 --- a/services/docreader/src/parser/web_parser.py +++ b/services/docreader/src/parser/web_parser.py @@ -113,6 +113,7 @@ def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any logger.info(f"Web page title: {title}") # Store the extracted title for later use self.extracted_title = title.strip() if title else "" + logger.info(f"DEBUG: Stored extracted_title='{self.extracted_title}'") text = "\n".join( (line.strip() for line in text.splitlines() if line.strip()) ) diff --git a/services/docreader/src/server/server.py b/services/docreader/src/server/server.py index 11a7b96..34c6e16 100644 --- a/services/docreader/src/server/server.py +++ b/services/docreader/src/server/server.py @@ -267,9 +267,11 @@ def ReadFromURL(self, request, context): f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks, title: {result.title}" ) + response_title = result.title if result.title else "" + logger.info(f"DEBUG: Building ReadResponse with title='{response_title}'") response = ReadResponse( chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks], - title=result.title if result.title else "" + title=response_title ) logger.info(f"Response size: {response.ByteSize()} bytes") return response