diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 7b8c26e..3f48df5 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -833,6 +833,21 @@ func (s *knowledgeService) processDocumentFromURL(ctx context.Context, return } + // Update knowledge title if extracted from HTML + logger.GetLogger(ctx).Infof("DEBUG: resp.GetTitle()='%s', knowledge.Title='%s'", resp.GetTitle(), knowledge.Title) + if resp.GetTitle() != "" && knowledge.Title == "" { + knowledge.Title = resp.GetTitle() + logger.GetLogger(ctx).Infof("Extracted title from URL: %s", knowledge.Title) + // Update database immediately + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("Failed to update knowledge title") + } else { + logger.GetLogger(ctx).Infof("DEBUG: Successfully updated knowledge title in database") + } + } else { + logger.GetLogger(ctx).Infof("DEBUG: Skipping title update - condition not met") + } + // Process and store chunks s.processChunks(ctx, kb, knowledge, resp.Chunks) } diff --git a/services/docreader/src/parser/base_parser.py b/services/docreader/src/parser/base_parser.py index 052bc82..3af42b9 100644 --- a/services/docreader/src/parser/base_parser.py +++ b/services/docreader/src/parser/base_parser.py @@ -58,6 +58,7 @@ class ParseResult: text: str # Extracted text content chunks: Optional[List[Chunk]] = None # Chunk results + title: Optional[str] = None # Document title (for URLs) class BaseParser(ABC): @@ -672,7 +673,10 @@ def parse(self, content: bytes) -> ParseResult: f"Skipping image processing for unsupported file type: {file_ext}" ) - return ParseResult(text=text, chunks=chunks) + # Include title if available (for WebParser) + title = getattr(self, 'extracted_title', None) + logger.info(f"DEBUG: Creating ParseResult with title='{title}'") + return ParseResult(text=text, chunks=chunks, title=title) def _split_into_units(self, text: str) -> List[str]: """ diff --git a/services/docreader/src/parser/web_parser.py b/services/docreader/src/parser/web_parser.py index 44c883a..bf8ebd0 100644 --- a/services/docreader/src/parser/web_parser.py +++ b/services/docreader/src/parser/web_parser.py @@ -15,6 +15,7 @@ class WebParser(BaseParser): def __init__(self, title: str, **kwargs): self.title = title + self.extracted_title = None # Store extracted title from HTML self.proxy = os.environ.get("WEB_PROXY", "") super().__init__(file_name=title, **kwargs) logger.info(f"Initialized WebParser with title: {title}") @@ -110,6 +111,9 @@ def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any logger.info("No title found, using default") logger.info(f"Web page title: {title}") + # Store the extracted title for later use + self.extracted_title = title.strip() if title else "" + logger.info(f"DEBUG: Stored extracted_title='{self.extracted_title}'") text = "\n".join( (line.strip() for line in text.splitlines() if line.strip()) ) diff --git a/services/docreader/src/proto/docreader.pb.go b/services/docreader/src/proto/docreader.pb.go index 95199af..cc917b3 100644 --- a/services/docreader/src/proto/docreader.pb.go +++ b/services/docreader/src/proto/docreader.pb.go @@ -629,6 +629,7 @@ type ReadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Chunks []*Chunk `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"` // 文档分块 Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` // 错误信息 + Title string `protobuf:"bytes,3,opt,name=title,proto3" json:"title,omitempty"` // 文档标题 (仅从URL读取时返回) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -677,6 +678,13 @@ func (x *ReadResponse) GetError() string { return "" } +func (x *ReadResponse) GetTitle() string { + if x != nil { + return x.Title + } + return "" +} + var File_docreader_proto protoreflect.FileDescriptor const file_docreader_proto_rawDesc = "" + diff --git a/services/docreader/src/proto/docreader.proto b/services/docreader/src/proto/docreader.proto index 44ff4ac..b799c82 100644 --- a/services/docreader/src/proto/docreader.proto +++ b/services/docreader/src/proto/docreader.proto @@ -86,4 +86,5 @@ message Chunk { message ReadResponse { repeated Chunk chunks = 1; // 文档分块 string error = 2; // 错误信息 + string title = 3; // 文档标题 (仅从URL读取时返回) } \ No newline at end of file diff --git a/services/docreader/src/server/server.py b/services/docreader/src/server/server.py index 9018d3e..34c6e16 100644 --- a/services/docreader/src/server/server.py +++ b/services/docreader/src/server/server.py @@ -262,13 +262,16 @@ def ReadFromURL(self, request, context): context.set_details(error_msg) return ReadResponse(error=error_msg) - # Convert to protobuf message, including image info + # Convert to protobuf message, including image info and title logger.info( - f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks" + f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks, title: {result.title}" ) + response_title = result.title if result.title else "" + logger.info(f"DEBUG: Building ReadResponse with title='{response_title}'") response = ReadResponse( - chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks] + chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks], + title=response_title ) logger.info(f"Response size: {response.ByteSize()} bytes") return response