WeKnora deployment source with the reviewed wiki identity, summary and recovery fixes

docreader/proto/docreader.pb.go

@@ -0,0 +1,796 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.7
+// 	protoc        v5.29.3
+// source: docreader.proto
+
+package proto
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type ReadConfig struct {
+	state                 protoimpl.MessageState `protogen:"open.v1"`
+	ParserEngine          string                 `protobuf:"bytes,1,opt,name=parser_engine,json=parserEngine,proto3" json:"parser_engine,omitempty"`
+	ParserEngineOverrides map[string]string      `protobuf:"bytes,2,rep,name=parser_engine_overrides,json=parserEngineOverrides,proto3" json:"parser_engine_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields         protoimpl.UnknownFields
+	sizeCache             protoimpl.SizeCache
+}
+
+func (x *ReadConfig) Reset() {
+	*x = ReadConfig{}
+	mi := &file_docreader_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ReadConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReadConfig) ProtoMessage() {}
+
+func (x *ReadConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReadConfig.ProtoReflect.Descriptor instead.
+func (*ReadConfig) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *ReadConfig) GetParserEngine() string {
+	if x != nil {
+		return x.ParserEngine
+	}
+	return ""
+}
+
+func (x *ReadConfig) GetParserEngineOverrides() map[string]string {
+	if x != nil {
+		return x.ParserEngineOverrides
+	}
+	return nil
+}
+
+// Unified read request: set file_content for file mode, url for URL mode.
+type ReadRequest struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	FileContent   []byte                 `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"`
+	FileName      string                 `protobuf:"bytes,2,opt,name=file_name,json=fileName,proto3" json:"file_name,omitempty"`
+	FileType      string                 `protobuf:"bytes,3,opt,name=file_type,json=fileType,proto3" json:"file_type,omitempty"`
+	Url           string                 `protobuf:"bytes,4,opt,name=url,proto3" json:"url,omitempty"`
+	Title         string                 `protobuf:"bytes,5,opt,name=title,proto3" json:"title,omitempty"`
+	Config        *ReadConfig            `protobuf:"bytes,6,opt,name=config,proto3" json:"config,omitempty"`
+	RequestId     string                 `protobuf:"bytes,7,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ReadRequest) Reset() {
+	*x = ReadRequest{}
+	mi := &file_docreader_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ReadRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReadRequest) ProtoMessage() {}
+
+func (x *ReadRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReadRequest.ProtoReflect.Descriptor instead.
+func (*ReadRequest) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ReadRequest) GetFileContent() []byte {
+	if x != nil {
+		return x.FileContent
+	}
+	return nil
+}
+
+func (x *ReadRequest) GetFileName() string {
+	if x != nil {
+		return x.FileName
+	}
+	return ""
+}
+
+func (x *ReadRequest) GetFileType() string {
+	if x != nil {
+		return x.FileType
+	}
+	return ""
+}
+
+func (x *ReadRequest) GetUrl() string {
+	if x != nil {
+		return x.Url
+	}
+	return ""
+}
+
+func (x *ReadRequest) GetTitle() string {
+	if x != nil {
+		return x.Title
+	}
+	return ""
+}
+
+func (x *ReadRequest) GetConfig() *ReadConfig {
+	if x != nil {
+		return x.Config
+	}
+	return nil
+}
+
+func (x *ReadRequest) GetRequestId() string {
+	if x != nil {
+		return x.RequestId
+	}
+	return ""
+}
+
+type ImageRef struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Filename      string                 `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"`
+	OriginalRef   string                 `protobuf:"bytes,2,opt,name=original_ref,json=originalRef,proto3" json:"original_ref,omitempty"`
+	MimeType      string                 `protobuf:"bytes,3,opt,name=mime_type,json=mimeType,proto3" json:"mime_type,omitempty"`
+	StorageKey    string                 `protobuf:"bytes,4,opt,name=storage_key,json=storageKey,proto3" json:"storage_key,omitempty"` // download URL from shared storage
+	ImageData     []byte                 `protobuf:"bytes,5,opt,name=image_data,json=imageData,proto3" json:"image_data,omitempty"`    // inline bytes fallback
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ImageRef) Reset() {
+	*x = ImageRef{}
+	mi := &file_docreader_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ImageRef) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ImageRef) ProtoMessage() {}
+
+func (x *ImageRef) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ImageRef.ProtoReflect.Descriptor instead.
+func (*ImageRef) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *ImageRef) GetFilename() string {
+	if x != nil {
+		return x.Filename
+	}
+	return ""
+}
+
+func (x *ImageRef) GetOriginalRef() string {
+	if x != nil {
+		return x.OriginalRef
+	}
+	return ""
+}
+
+func (x *ImageRef) GetMimeType() string {
+	if x != nil {
+		return x.MimeType
+	}
+	return ""
+}
+
+func (x *ImageRef) GetStorageKey() string {
+	if x != nil {
+		return x.StorageKey
+	}
+	return ""
+}
+
+func (x *ImageRef) GetImageData() []byte {
+	if x != nil {
+		return x.ImageData
+	}
+	return nil
+}
+
+type ReadResponse struct {
+	state           protoimpl.MessageState `protogen:"open.v1"`
+	MarkdownContent string                 `protobuf:"bytes,1,opt,name=markdown_content,json=markdownContent,proto3" json:"markdown_content,omitempty"`
+	ImageRefs       []*ImageRef            `protobuf:"bytes,2,rep,name=image_refs,json=imageRefs,proto3" json:"image_refs,omitempty"`
+	ImageDirPath    string                 `protobuf:"bytes,3,opt,name=image_dir_path,json=imageDirPath,proto3" json:"image_dir_path,omitempty"`
+	Metadata        map[string]string      `protobuf:"bytes,4,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	Error           string                 `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"`
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
+}
+
+func (x *ReadResponse) Reset() {
+	*x = ReadResponse{}
+	mi := &file_docreader_proto_msgTypes[3]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ReadResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReadResponse) ProtoMessage() {}
+
+func (x *ReadResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[3]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReadResponse.ProtoReflect.Descriptor instead.
+func (*ReadResponse) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *ReadResponse) GetMarkdownContent() string {
+	if x != nil {
+		return x.MarkdownContent
+	}
+	return ""
+}
+
+func (x *ReadResponse) GetImageRefs() []*ImageRef {
+	if x != nil {
+		return x.ImageRefs
+	}
+	return nil
+}
+
+func (x *ReadResponse) GetImageDirPath() string {
+	if x != nil {
+		return x.ImageDirPath
+	}
+	return ""
+}
+
+func (x *ReadResponse) GetMetadata() map[string]string {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+func (x *ReadResponse) GetError() string {
+	if x != nil {
+		return x.Error
+	}
+	return ""
+}
+
+// Metadata header for a streamed read. Sent exactly once, before any image.
+type ReadStreamMeta struct {
+	state           protoimpl.MessageState `protogen:"open.v1"`
+	MarkdownContent string                 `protobuf:"bytes,1,opt,name=markdown_content,json=markdownContent,proto3" json:"markdown_content,omitempty"`
+	ImageDirPath    string                 `protobuf:"bytes,2,opt,name=image_dir_path,json=imageDirPath,proto3" json:"image_dir_path,omitempty"`
+	Metadata        map[string]string      `protobuf:"bytes,3,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	Error           string                 `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
+	ImageCount      uint32                 `protobuf:"varint,5,opt,name=image_count,json=imageCount,proto3" json:"image_count,omitempty"` // best-effort total image count (0 if unknown)
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
+}
+
+func (x *ReadStreamMeta) Reset() {
+	*x = ReadStreamMeta{}
+	mi := &file_docreader_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ReadStreamMeta) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReadStreamMeta) ProtoMessage() {}
+
+func (x *ReadStreamMeta) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReadStreamMeta.ProtoReflect.Descriptor instead.
+func (*ReadStreamMeta) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *ReadStreamMeta) GetMarkdownContent() string {
+	if x != nil {
+		return x.MarkdownContent
+	}
+	return ""
+}
+
+func (x *ReadStreamMeta) GetImageDirPath() string {
+	if x != nil {
+		return x.ImageDirPath
+	}
+	return ""
+}
+
+func (x *ReadStreamMeta) GetMetadata() map[string]string {
+	if x != nil {
+		return x.Metadata
+	}
+	return nil
+}
+
+func (x *ReadStreamMeta) GetError() string {
+	if x != nil {
+		return x.Error
+	}
+	return ""
+}
+
+func (x *ReadStreamMeta) GetImageCount() uint32 {
+	if x != nil {
+		return x.ImageCount
+	}
+	return 0
+}
+
+// One frame of a ReadStream. The first frame carries `meta`; every subsequent
+// frame carries a single `image`.
+type ReadStreamResponse struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Types that are valid to be assigned to Payload:
+	//
+	//	*ReadStreamResponse_Meta
+	//	*ReadStreamResponse_Image
+	Payload       isReadStreamResponse_Payload `protobuf_oneof:"payload"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ReadStreamResponse) Reset() {
+	*x = ReadStreamResponse{}
+	mi := &file_docreader_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ReadStreamResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ReadStreamResponse) ProtoMessage() {}
+
+func (x *ReadStreamResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ReadStreamResponse.ProtoReflect.Descriptor instead.
+func (*ReadStreamResponse) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *ReadStreamResponse) GetPayload() isReadStreamResponse_Payload {
+	if x != nil {
+		return x.Payload
+	}
+	return nil
+}
+
+func (x *ReadStreamResponse) GetMeta() *ReadStreamMeta {
+	if x != nil {
+		if x, ok := x.Payload.(*ReadStreamResponse_Meta); ok {
+			return x.Meta
+		}
+	}
+	return nil
+}
+
+func (x *ReadStreamResponse) GetImage() *ImageRef {
+	if x != nil {
+		if x, ok := x.Payload.(*ReadStreamResponse_Image); ok {
+			return x.Image
+		}
+	}
+	return nil
+}
+
+type isReadStreamResponse_Payload interface {
+	isReadStreamResponse_Payload()
+}
+
+type ReadStreamResponse_Meta struct {
+	Meta *ReadStreamMeta `protobuf:"bytes,1,opt,name=meta,proto3,oneof"`
+}
+
+type ReadStreamResponse_Image struct {
+	Image *ImageRef `protobuf:"bytes,2,opt,name=image,proto3,oneof"`
+}
+
+func (*ReadStreamResponse_Meta) isReadStreamResponse_Payload() {}
+
+func (*ReadStreamResponse_Image) isReadStreamResponse_Payload() {}
+
+type ListEnginesRequest struct {
+	state           protoimpl.MessageState `protogen:"open.v1"`
+	ConfigOverrides map[string]string      `protobuf:"bytes,1,rep,name=config_overrides,json=configOverrides,proto3" json:"config_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
+	unknownFields   protoimpl.UnknownFields
+	sizeCache       protoimpl.SizeCache
+}
+
+func (x *ListEnginesRequest) Reset() {
+	*x = ListEnginesRequest{}
+	mi := &file_docreader_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ListEnginesRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ListEnginesRequest) ProtoMessage() {}
+
+func (x *ListEnginesRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ListEnginesRequest.ProtoReflect.Descriptor instead.
+func (*ListEnginesRequest) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *ListEnginesRequest) GetConfigOverrides() map[string]string {
+	if x != nil {
+		return x.ConfigOverrides
+	}
+	return nil
+}
+
+type ParserEngineInfo struct {
+	state             protoimpl.MessageState `protogen:"open.v1"`
+	Name              string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Description       string                 `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"`
+	FileTypes         []string               `protobuf:"bytes,3,rep,name=file_types,json=fileTypes,proto3" json:"file_types,omitempty"`
+	Available         bool                   `protobuf:"varint,4,opt,name=available,proto3" json:"available,omitempty"`
+	UnavailableReason string                 `protobuf:"bytes,5,opt,name=unavailable_reason,json=unavailableReason,proto3" json:"unavailable_reason,omitempty"`
+	unknownFields     protoimpl.UnknownFields
+	sizeCache         protoimpl.SizeCache
+}
+
+func (x *ParserEngineInfo) Reset() {
+	*x = ParserEngineInfo{}
+	mi := &file_docreader_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ParserEngineInfo) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ParserEngineInfo) ProtoMessage() {}
+
+func (x *ParserEngineInfo) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[7]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ParserEngineInfo.ProtoReflect.Descriptor instead.
+func (*ParserEngineInfo) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *ParserEngineInfo) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *ParserEngineInfo) GetDescription() string {
+	if x != nil {
+		return x.Description
+	}
+	return ""
+}
+
+func (x *ParserEngineInfo) GetFileTypes() []string {
+	if x != nil {
+		return x.FileTypes
+	}
+	return nil
+}
+
+func (x *ParserEngineInfo) GetAvailable() bool {
+	if x != nil {
+		return x.Available
+	}
+	return false
+}
+
+func (x *ParserEngineInfo) GetUnavailableReason() string {
+	if x != nil {
+		return x.UnavailableReason
+	}
+	return ""
+}
+
+type ListEnginesResponse struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Engines       []*ParserEngineInfo    `protobuf:"bytes,1,rep,name=engines,proto3" json:"engines,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ListEnginesResponse) Reset() {
+	*x = ListEnginesResponse{}
+	mi := &file_docreader_proto_msgTypes[8]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ListEnginesResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ListEnginesResponse) ProtoMessage() {}
+
+func (x *ListEnginesResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_docreader_proto_msgTypes[8]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ListEnginesResponse.ProtoReflect.Descriptor instead.
+func (*ListEnginesResponse) Descriptor() ([]byte, []int) {
+	return file_docreader_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *ListEnginesResponse) GetEngines() []*ParserEngineInfo {
+	if x != nil {
+		return x.Engines
+	}
+	return nil
+}
+
+var File_docreader_proto protoreflect.FileDescriptor
+
+const file_docreader_proto_rawDesc = "" +
+	"\n" +
+	"\x0fdocreader.proto\x12\tdocreader\"\xeb\x01\n" +
+	"\n" +
+	"ReadConfig\x12#\n" +
+	"\rparser_engine\x18\x01 \x01(\tR\fparserEngine\x12h\n" +
+	"\x17parser_engine_overrides\x18\x02 \x03(\v20.docreader.ReadConfig.ParserEngineOverridesEntryR\x15parserEngineOverrides\x1aH\n" +
+	"\x1aParserEngineOverridesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01J\x04\b\x03\x10\x04\"\xe0\x01\n" +
+	"\vReadRequest\x12!\n" +
+	"\ffile_content\x18\x01 \x01(\fR\vfileContent\x12\x1b\n" +
+	"\tfile_name\x18\x02 \x01(\tR\bfileName\x12\x1b\n" +
+	"\tfile_type\x18\x03 \x01(\tR\bfileType\x12\x10\n" +
+	"\x03url\x18\x04 \x01(\tR\x03url\x12\x14\n" +
+	"\x05title\x18\x05 \x01(\tR\x05title\x12-\n" +
+	"\x06config\x18\x06 \x01(\v2\x15.docreader.ReadConfigR\x06config\x12\x1d\n" +
+	"\n" +
+	"request_id\x18\a \x01(\tR\trequestId\"\xa6\x01\n" +
+	"\bImageRef\x12\x1a\n" +
+	"\bfilename\x18\x01 \x01(\tR\bfilename\x12!\n" +
+	"\foriginal_ref\x18\x02 \x01(\tR\voriginalRef\x12\x1b\n" +
+	"\tmime_type\x18\x03 \x01(\tR\bmimeType\x12\x1f\n" +
+	"\vstorage_key\x18\x04 \x01(\tR\n" +
+	"storageKey\x12\x1d\n" +
+	"\n" +
+	"image_data\x18\x05 \x01(\fR\timageData\"\xa9\x02\n" +
+	"\fReadResponse\x12)\n" +
+	"\x10markdown_content\x18\x01 \x01(\tR\x0fmarkdownContent\x122\n" +
+	"\n" +
+	"image_refs\x18\x02 \x03(\v2\x13.docreader.ImageRefR\timageRefs\x12$\n" +
+	"\x0eimage_dir_path\x18\x03 \x01(\tR\fimageDirPath\x12A\n" +
+	"\bmetadata\x18\x04 \x03(\v2%.docreader.ReadResponse.MetadataEntryR\bmetadata\x12\x14\n" +
+	"\x05error\x18\x05 \x01(\tR\x05error\x1a;\n" +
+	"\rMetadataEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x9a\x02\n" +
+	"\x0eReadStreamMeta\x12)\n" +
+	"\x10markdown_content\x18\x01 \x01(\tR\x0fmarkdownContent\x12$\n" +
+	"\x0eimage_dir_path\x18\x02 \x01(\tR\fimageDirPath\x12C\n" +
+	"\bmetadata\x18\x03 \x03(\v2'.docreader.ReadStreamMeta.MetadataEntryR\bmetadata\x12\x14\n" +
+	"\x05error\x18\x04 \x01(\tR\x05error\x12\x1f\n" +
+	"\vimage_count\x18\x05 \x01(\rR\n" +
+	"imageCount\x1a;\n" +
+	"\rMetadataEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"}\n" +
+	"\x12ReadStreamResponse\x12/\n" +
+	"\x04meta\x18\x01 \x01(\v2\x19.docreader.ReadStreamMetaH\x00R\x04meta\x12+\n" +
+	"\x05image\x18\x02 \x01(\v2\x13.docreader.ImageRefH\x00R\x05imageB\t\n" +
+	"\apayload\"\xb7\x01\n" +
+	"\x12ListEnginesRequest\x12]\n" +
+	"\x10config_overrides\x18\x01 \x03(\v22.docreader.ListEnginesRequest.ConfigOverridesEntryR\x0fconfigOverrides\x1aB\n" +
+	"\x14ConfigOverridesEntry\x12\x10\n" +
+	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb4\x01\n" +
+	"\x10ParserEngineInfo\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x12 \n" +
+	"\vdescription\x18\x02 \x01(\tR\vdescription\x12\x1d\n" +
+	"\n" +
+	"file_types\x18\x03 \x03(\tR\tfileTypes\x12\x1c\n" +
+	"\tavailable\x18\x04 \x01(\bR\tavailable\x12-\n" +
+	"\x12unavailable_reason\x18\x05 \x01(\tR\x11unavailableReason\"L\n" +
+	"\x13ListEnginesResponse\x125\n" +
+	"\aengines\x18\x01 \x03(\v2\x1b.docreader.ParserEngineInfoR\aengines2\xdf\x01\n" +
+	"\tDocReader\x129\n" +
+	"\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n" +
+	"\n" +
+	"ReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x000\x01\x12N\n" +
+	"\vListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3"
+
+var (
+	file_docreader_proto_rawDescOnce sync.Once
+	file_docreader_proto_rawDescData []byte
+)
+
+func file_docreader_proto_rawDescGZIP() []byte {
+	file_docreader_proto_rawDescOnce.Do(func() {
+		file_docreader_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)))
+	})
+	return file_docreader_proto_rawDescData
+}
+
+var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
+var file_docreader_proto_goTypes = []any{
+	(*ReadConfig)(nil),          // 0: docreader.ReadConfig
+	(*ReadRequest)(nil),         // 1: docreader.ReadRequest
+	(*ImageRef)(nil),            // 2: docreader.ImageRef
+	(*ReadResponse)(nil),        // 3: docreader.ReadResponse
+	(*ReadStreamMeta)(nil),      // 4: docreader.ReadStreamMeta
+	(*ReadStreamResponse)(nil),  // 5: docreader.ReadStreamResponse
+	(*ListEnginesRequest)(nil),  // 6: docreader.ListEnginesRequest
+	(*ParserEngineInfo)(nil),    // 7: docreader.ParserEngineInfo
+	(*ListEnginesResponse)(nil), // 8: docreader.ListEnginesResponse
+	nil,                         // 9: docreader.ReadConfig.ParserEngineOverridesEntry
+	nil,                         // 10: docreader.ReadResponse.MetadataEntry
+	nil,                         // 11: docreader.ReadStreamMeta.MetadataEntry
+	nil,                         // 12: docreader.ListEnginesRequest.ConfigOverridesEntry
+}
+var file_docreader_proto_depIdxs = []int32{
+	9,  // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry
+	0,  // 1: docreader.ReadRequest.config:type_name -> docreader.ReadConfig
+	2,  // 2: docreader.ReadResponse.image_refs:type_name -> docreader.ImageRef
+	10, // 3: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry
+	11, // 4: docreader.ReadStreamMeta.metadata:type_name -> docreader.ReadStreamMeta.MetadataEntry
+	4,  // 5: docreader.ReadStreamResponse.meta:type_name -> docreader.ReadStreamMeta
+	2,  // 6: docreader.ReadStreamResponse.image:type_name -> docreader.ImageRef
+	12, // 7: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry
+	7,  // 8: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo
+	1,  // 9: docreader.DocReader.Read:input_type -> docreader.ReadRequest
+	1,  // 10: docreader.DocReader.ReadStream:input_type -> docreader.ReadRequest
+	6,  // 11: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest
+	3,  // 12: docreader.DocReader.Read:output_type -> docreader.ReadResponse
+	5,  // 13: docreader.DocReader.ReadStream:output_type -> docreader.ReadStreamResponse
+	8,  // 14: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse
+	12, // [12:15] is the sub-list for method output_type
+	9,  // [9:12] is the sub-list for method input_type
+	9,  // [9:9] is the sub-list for extension type_name
+	9,  // [9:9] is the sub-list for extension extendee
+	0,  // [0:9] is the sub-list for field type_name
+}
+
+func init() { file_docreader_proto_init() }
+func file_docreader_proto_init() {
+	if File_docreader_proto != nil {
+		return
+	}
+	file_docreader_proto_msgTypes[5].OneofWrappers = []any{
+		(*ReadStreamResponse_Meta)(nil),
+		(*ReadStreamResponse_Image)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   13,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_docreader_proto_goTypes,
+		DependencyIndexes: file_docreader_proto_depIdxs,
+		MessageInfos:      file_docreader_proto_msgTypes,
+	}.Build()
+	File_docreader_proto = out.File
+	file_docreader_proto_goTypes = nil
+	file_docreader_proto_depIdxs = nil
+}

docreader/proto/docreader.proto

@@ -0,0 +1,86 @@
+syntax = "proto3";
+
+package docreader;
+
+option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto";
+
+service DocReader {
+  rpc Read(ReadRequest) returns (ReadResponse) {}
+  // ReadStream is the streaming counterpart of Read. It first emits one
+  // ReadStreamResponse carrying the parse metadata (markdown / metadata /
+  // error), then emits one message per image. This keeps every gRPC message
+  // small so large scanned PDFs (hundreds of page images, far exceeding the
+  // unary message-size cap) can be returned without RESOURCE_EXHAUSTED and
+  // with bounded memory on both ends.
+  rpc ReadStream(ReadRequest) returns (stream ReadStreamResponse) {}
+  rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}
+}
+
+message ReadConfig {
+  string parser_engine = 1;
+  map<string, string> parser_engine_overrides = 2;
+  // image_storage removed: image persistence is now handled entirely by the Go App.
+  // Field number 3 is reserved for backward compatibility.
+  reserved 3;
+}
+
+// Unified read request: set file_content for file mode, url for URL mode.
+message ReadRequest {
+  bytes  file_content = 1;
+  string file_name = 2;
+  string file_type = 3;
+  string url = 4;
+  string title = 5;
+  ReadConfig config = 6;
+  string request_id = 7;
+}
+
+message ImageRef {
+  string filename = 1;
+  string original_ref = 2;
+  string mime_type = 3;
+  string storage_key = 4;    // download URL from shared storage
+  bytes  image_data = 5;     // inline bytes fallback
+}
+
+message ReadResponse {
+  string markdown_content = 1;
+  repeated ImageRef image_refs = 2;
+  string image_dir_path = 3;
+  map<string, string> metadata = 4;
+  string error = 5;
+}
+
+// Metadata header for a streamed read. Sent exactly once, before any image.
+message ReadStreamMeta {
+  string markdown_content = 1;
+  string image_dir_path = 2;
+  map<string, string> metadata = 3;
+  string error = 4;
+  uint32 image_count = 5; // best-effort total image count (0 if unknown)
+}
+
+// One frame of a ReadStream. The first frame carries `meta`; every subsequent
+// frame carries a single `image`.
+message ReadStreamResponse {
+  oneof payload {
+    ReadStreamMeta meta = 1;
+    ImageRef image = 2;
+  }
+}
+
+message ListEnginesRequest {
+  map<string, string> config_overrides = 1;
+}
+
+message ParserEngineInfo {
+  string name = 1;
+  string description = 2;
+  repeated string file_types = 3;
+  bool available = 4;
+  string unavailable_reason = 5;
+}
+
+message ListEnginesResponse {
+  repeated ParserEngineInfo engines = 1;
+}

docreader/proto/docreader_grpc.pb.go

@@ -0,0 +1,213 @@
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.5.1
+// - protoc             v5.29.3
+// source: docreader.proto
+
+package proto
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.64.0 or later.
+const _ = grpc.SupportPackageIsVersion9
+
+const (
+	DocReader_Read_FullMethodName        = "/docreader.DocReader/Read"
+	DocReader_ReadStream_FullMethodName  = "/docreader.DocReader/ReadStream"
+	DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines"
+)
+
+// DocReaderClient is the client API for DocReader service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type DocReaderClient interface {
+	Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
+	// ReadStream is the streaming counterpart of Read. It first emits one
+	// ReadStreamResponse carrying the parse metadata (markdown / metadata /
+	// error), then emits one message per image. This keeps every gRPC message
+	// small so large scanned PDFs (hundreds of page images, far exceeding the
+	// unary message-size cap) can be returned without RESOURCE_EXHAUSTED and
+	// with bounded memory on both ends.
+	ReadStream(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadStreamResponse], error)
+	ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error)
+}
+
+type docReaderClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewDocReaderClient(cc grpc.ClientConnInterface) DocReaderClient {
+	return &docReaderClient{cc}
+}
+
+func (c *docReaderClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ReadResponse)
+	err := c.cc.Invoke(ctx, DocReader_Read_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *docReaderClient) ReadStream(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ReadStreamResponse], error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	stream, err := c.cc.NewStream(ctx, &DocReader_ServiceDesc.Streams[0], DocReader_ReadStream_FullMethodName, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &grpc.GenericClientStream[ReadRequest, ReadStreamResponse]{ClientStream: stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type DocReader_ReadStreamClient = grpc.ServerStreamingClient[ReadStreamResponse]
+
+func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(ListEnginesResponse)
+	err := c.cc.Invoke(ctx, DocReader_ListEngines_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// DocReaderServer is the server API for DocReader service.
+// All implementations must embed UnimplementedDocReaderServer
+// for forward compatibility.
+type DocReaderServer interface {
+	Read(context.Context, *ReadRequest) (*ReadResponse, error)
+	// ReadStream is the streaming counterpart of Read. It first emits one
+	// ReadStreamResponse carrying the parse metadata (markdown / metadata /
+	// error), then emits one message per image. This keeps every gRPC message
+	// small so large scanned PDFs (hundreds of page images, far exceeding the
+	// unary message-size cap) can be returned without RESOURCE_EXHAUSTED and
+	// with bounded memory on both ends.
+	ReadStream(*ReadRequest, grpc.ServerStreamingServer[ReadStreamResponse]) error
+	ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error)
+	mustEmbedUnimplementedDocReaderServer()
+}
+
+// UnimplementedDocReaderServer must be embedded to have
+// forward compatible implementations.
+//
+// NOTE: this should be embedded by value instead of pointer to avoid a nil
+// pointer dereference when methods are called.
+type UnimplementedDocReaderServer struct{}
+
+func (UnimplementedDocReaderServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Read not implemented")
+}
+func (UnimplementedDocReaderServer) ReadStream(*ReadRequest, grpc.ServerStreamingServer[ReadStreamResponse]) error {
+	return status.Errorf(codes.Unimplemented, "method ReadStream not implemented")
+}
+func (UnimplementedDocReaderServer) ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ListEngines not implemented")
+}
+func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {}
+func (UnimplementedDocReaderServer) testEmbeddedByValue()                   {}
+
+// UnsafeDocReaderServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to DocReaderServer will
+// result in compilation errors.
+type UnsafeDocReaderServer interface {
+	mustEmbedUnimplementedDocReaderServer()
+}
+
+func RegisterDocReaderServer(s grpc.ServiceRegistrar, srv DocReaderServer) {
+	// If the following call pancis, it indicates UnimplementedDocReaderServer was
+	// embedded by pointer and is nil.  This will cause panics if an
+	// unimplemented method is ever invoked, so we test this at initialization
+	// time to prevent it from happening at runtime later due to I/O.
+	if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
+		t.testEmbeddedByValue()
+	}
+	s.RegisterService(&DocReader_ServiceDesc, srv)
+}
+
+func _DocReader_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ReadRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(DocReaderServer).Read(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: DocReader_Read_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(DocReaderServer).Read(ctx, req.(*ReadRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _DocReader_ReadStream_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(ReadRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(DocReaderServer).ReadStream(m, &grpc.GenericServerStream[ReadRequest, ReadStreamResponse]{ServerStream: stream})
+}
+
+// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
+type DocReader_ReadStreamServer = grpc.ServerStreamingServer[ReadStreamResponse]
+
+func _DocReader_ListEngines_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListEnginesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(DocReaderServer).ListEngines(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: DocReader_ListEngines_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(DocReaderServer).ListEngines(ctx, req.(*ListEnginesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// DocReader_ServiceDesc is the grpc.ServiceDesc for DocReader service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var DocReader_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "docreader.DocReader",
+	HandlerType: (*DocReaderServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Read",
+			Handler:    _DocReader_Read_Handler,
+		},
+		{
+			MethodName: "ListEngines",
+			Handler:    _DocReader_ListEngines_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "ReadStream",
+			Handler:       _DocReader_ReadStream_Handler,
+			ServerStreams: true,
+		},
+	},
+	Metadata: "docreader.proto",
+}

docreader/proto/docreader_pb2.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: docreader.proto
+# Protobuf Python Version: 6.31.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    6,
+    31,
+    1,
+    '',
+    'docreader.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xba\x01\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd2\x01\n\x0eReadStreamMeta\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\x16\n\x0eimage_dir_path\x18\x02 \x01(\t\x12\x39\n\x08metadata\x18\x03 \x03(\x0b\x32\'.docreader.ReadStreamMeta.MetadataEntry\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x13\n\x0bimage_count\x18\x05 \x01(\r\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\x12ReadStreamResponse\x12)\n\x04meta\x18\x01 \x01(\x0b\x32\x19.docreader.ReadStreamMetaH\x00\x12$\n\x05image\x18\x02 \x01(\x0b\x32\x13.docreader.ImageRefH\x00\x42\t\n\x07payload\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo2\xdf\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\nReadStream\x12\x16.docreader.ReadRequest\x1a\x1d.docreader.ReadStreamResponse\"\x00\x30\x01\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docreader_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  _globals['DESCRIPTOR']._loaded_options = None
+  _globals['DESCRIPTOR']._serialized_options = b'Z3github.com/Tencent/WeKnora/internal/docreader/proto'
+  _globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._loaded_options = None
+  _globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001'
+  _globals['_READRESPONSE_METADATAENTRY']._loaded_options = None
+  _globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001'
+  _globals['_READSTREAMMETA_METADATAENTRY']._loaded_options = None
+  _globals['_READSTREAMMETA_METADATAENTRY']._serialized_options = b'8\001'
+  _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None
+  _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001'
+  _globals['_READCONFIG']._serialized_start=31
+  _globals['_READCONFIG']._serialized_end=217
+  _globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=151
+  _globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=211
+  _globals['_READREQUEST']._serialized_start=220
+  _globals['_READREQUEST']._serialized_end=380
+  _globals['_IMAGEREF']._serialized_start=382
+  _globals['_IMAGEREF']._serialized_end=492
+  _globals['_READRESPONSE']._serialized_start=495
+  _globals['_READRESPONSE']._serialized_end=721
+  _globals['_READRESPONSE_METADATAENTRY']._serialized_start=674
+  _globals['_READRESPONSE_METADATAENTRY']._serialized_end=721
+  _globals['_READSTREAMMETA']._serialized_start=724
+  _globals['_READSTREAMMETA']._serialized_end=934
+  _globals['_READSTREAMMETA_METADATAENTRY']._serialized_start=674
+  _globals['_READSTREAMMETA_METADATAENTRY']._serialized_end=721
+  _globals['_READSTREAMRESPONSE']._serialized_start=936
+  _globals['_READSTREAMRESPONSE']._serialized_end=1048
+  _globals['_LISTENGINESREQUEST']._serialized_start=1051
+  _globals['_LISTENGINESREQUEST']._serialized_end=1205
+  _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=1151
+  _globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=1205
+  _globals['_PARSERENGINEINFO']._serialized_start=1207
+  _globals['_PARSERENGINEINFO']._serialized_end=1327
+  _globals['_LISTENGINESRESPONSE']._serialized_start=1329
+  _globals['_LISTENGINESRESPONSE']._serialized_end=1396
+  _globals['_DOCREADER']._serialized_start=1399
+  _globals['_DOCREADER']._serialized_end=1622
+# @@protoc_insertion_point(module_scope)

docreader/proto/docreader_pb2.pyi

@@ -0,0 +1,137 @@
+from google.protobuf.internal import containers as _containers
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from collections.abc import Iterable as _Iterable, Mapping as _Mapping
+from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class ReadConfig(_message.Message):
+    __slots__ = ("parser_engine", "parser_engine_overrides")
+    class ParserEngineOverridesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    PARSER_ENGINE_FIELD_NUMBER: _ClassVar[int]
+    PARSER_ENGINE_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
+    parser_engine: str
+    parser_engine_overrides: _containers.ScalarMap[str, str]
+    def __init__(self, parser_engine: _Optional[str] = ..., parser_engine_overrides: _Optional[_Mapping[str, str]] = ...) -> None: ...
+
+class ReadRequest(_message.Message):
+    __slots__ = ("file_content", "file_name", "file_type", "url", "title", "config", "request_id")
+    FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
+    FILE_NAME_FIELD_NUMBER: _ClassVar[int]
+    FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
+    URL_FIELD_NUMBER: _ClassVar[int]
+    TITLE_FIELD_NUMBER: _ClassVar[int]
+    CONFIG_FIELD_NUMBER: _ClassVar[int]
+    REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
+    file_content: bytes
+    file_name: str
+    file_type: str
+    url: str
+    title: str
+    config: ReadConfig
+    request_id: str
+    def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., url: _Optional[str] = ..., title: _Optional[str] = ..., config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
+
+class ImageRef(_message.Message):
+    __slots__ = ("filename", "original_ref", "mime_type", "storage_key", "image_data")
+    FILENAME_FIELD_NUMBER: _ClassVar[int]
+    ORIGINAL_REF_FIELD_NUMBER: _ClassVar[int]
+    MIME_TYPE_FIELD_NUMBER: _ClassVar[int]
+    STORAGE_KEY_FIELD_NUMBER: _ClassVar[int]
+    IMAGE_DATA_FIELD_NUMBER: _ClassVar[int]
+    filename: str
+    original_ref: str
+    mime_type: str
+    storage_key: str
+    image_data: bytes
+    def __init__(self, filename: _Optional[str] = ..., original_ref: _Optional[str] = ..., mime_type: _Optional[str] = ..., storage_key: _Optional[str] = ..., image_data: _Optional[bytes] = ...) -> None: ...
+
+class ReadResponse(_message.Message):
+    __slots__ = ("markdown_content", "image_refs", "image_dir_path", "metadata", "error")
+    class MetadataEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    MARKDOWN_CONTENT_FIELD_NUMBER: _ClassVar[int]
+    IMAGE_REFS_FIELD_NUMBER: _ClassVar[int]
+    IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int]
+    METADATA_FIELD_NUMBER: _ClassVar[int]
+    ERROR_FIELD_NUMBER: _ClassVar[int]
+    markdown_content: str
+    image_refs: _containers.RepeatedCompositeFieldContainer[ImageRef]
+    image_dir_path: str
+    metadata: _containers.ScalarMap[str, str]
+    error: str
+    def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ...) -> None: ...
+
+class ReadStreamMeta(_message.Message):
+    __slots__ = ("markdown_content", "image_dir_path", "metadata", "error", "image_count")
+    class MetadataEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    MARKDOWN_CONTENT_FIELD_NUMBER: _ClassVar[int]
+    IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int]
+    METADATA_FIELD_NUMBER: _ClassVar[int]
+    ERROR_FIELD_NUMBER: _ClassVar[int]
+    IMAGE_COUNT_FIELD_NUMBER: _ClassVar[int]
+    markdown_content: str
+    image_dir_path: str
+    metadata: _containers.ScalarMap[str, str]
+    error: str
+    image_count: int
+    def __init__(self, markdown_content: _Optional[str] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ..., image_count: _Optional[int] = ...) -> None: ...
+
+class ReadStreamResponse(_message.Message):
+    __slots__ = ("meta", "image")
+    META_FIELD_NUMBER: _ClassVar[int]
+    IMAGE_FIELD_NUMBER: _ClassVar[int]
+    meta: ReadStreamMeta
+    image: ImageRef
+    def __init__(self, meta: _Optional[_Union[ReadStreamMeta, _Mapping]] = ..., image: _Optional[_Union[ImageRef, _Mapping]] = ...) -> None: ...
+
+class ListEnginesRequest(_message.Message):
+    __slots__ = ("config_overrides",)
+    class ConfigOverridesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    CONFIG_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
+    config_overrides: _containers.ScalarMap[str, str]
+    def __init__(self, config_overrides: _Optional[_Mapping[str, str]] = ...) -> None: ...
+
+class ParserEngineInfo(_message.Message):
+    __slots__ = ("name", "description", "file_types", "available", "unavailable_reason")
+    NAME_FIELD_NUMBER: _ClassVar[int]
+    DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
+    FILE_TYPES_FIELD_NUMBER: _ClassVar[int]
+    AVAILABLE_FIELD_NUMBER: _ClassVar[int]
+    UNAVAILABLE_REASON_FIELD_NUMBER: _ClassVar[int]
+    name: str
+    description: str
+    file_types: _containers.RepeatedScalarFieldContainer[str]
+    available: bool
+    unavailable_reason: str
+    def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., file_types: _Optional[_Iterable[str]] = ..., available: bool = ..., unavailable_reason: _Optional[str] = ...) -> None: ...
+
+class ListEnginesResponse(_message.Message):
+    __slots__ = ("engines",)
+    ENGINES_FIELD_NUMBER: _ClassVar[int]
+    engines: _containers.RepeatedCompositeFieldContainer[ParserEngineInfo]
+    def __init__(self, engines: _Optional[_Iterable[_Union[ParserEngineInfo, _Mapping]]] = ...) -> None: ...

docreader/proto/docreader_pb2_grpc.py

@@ -0,0 +1,189 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+import warnings
+
+from docreader.proto import docreader_pb2 as docreader__pb2
+
+GRPC_GENERATED_VERSION = '1.80.0'
+GRPC_VERSION = grpc.__version__
+_version_not_supported = False
+
+try:
+    from grpc._utilities import first_version_is_lower
+    _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION)
+except ImportError:
+    _version_not_supported = True
+
+if _version_not_supported:
+    raise RuntimeError(
+        f'The grpc package installed is at version {GRPC_VERSION},'
+        + ' but the generated code in docreader_pb2_grpc.py depends on'
+        + f' grpcio>={GRPC_GENERATED_VERSION}.'
+        + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
+        + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
+    )
+
+
+class DocReaderStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Read = channel.unary_unary(
+                '/docreader.DocReader/Read',
+                request_serializer=docreader__pb2.ReadRequest.SerializeToString,
+                response_deserializer=docreader__pb2.ReadResponse.FromString,
+                _registered_method=True)
+        self.ReadStream = channel.unary_stream(
+                '/docreader.DocReader/ReadStream',
+                request_serializer=docreader__pb2.ReadRequest.SerializeToString,
+                response_deserializer=docreader__pb2.ReadStreamResponse.FromString,
+                _registered_method=True)
+        self.ListEngines = channel.unary_unary(
+                '/docreader.DocReader/ListEngines',
+                request_serializer=docreader__pb2.ListEnginesRequest.SerializeToString,
+                response_deserializer=docreader__pb2.ListEnginesResponse.FromString,
+                _registered_method=True)
+
+
+class DocReaderServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def Read(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def ReadStream(self, request, context):
+        """ReadStream is the streaming counterpart of Read. It first emits one
+        ReadStreamResponse carrying the parse metadata (markdown / metadata /
+        error), then emits one message per image. This keeps every gRPC message
+        small so large scanned PDFs (hundreds of page images, far exceeding the
+        unary message-size cap) can be returned without RESOURCE_EXHAUSTED and
+        with bounded memory on both ends.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def ListEngines(self, request, context):
+        """Missing associated documentation comment in .proto file."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_DocReaderServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Read': grpc.unary_unary_rpc_method_handler(
+                    servicer.Read,
+                    request_deserializer=docreader__pb2.ReadRequest.FromString,
+                    response_serializer=docreader__pb2.ReadResponse.SerializeToString,
+            ),
+            'ReadStream': grpc.unary_stream_rpc_method_handler(
+                    servicer.ReadStream,
+                    request_deserializer=docreader__pb2.ReadRequest.FromString,
+                    response_serializer=docreader__pb2.ReadStreamResponse.SerializeToString,
+            ),
+            'ListEngines': grpc.unary_unary_rpc_method_handler(
+                    servicer.ListEngines,
+                    request_deserializer=docreader__pb2.ListEnginesRequest.FromString,
+                    response_serializer=docreader__pb2.ListEnginesResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'docreader.DocReader', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+    server.add_registered_method_handlers('docreader.DocReader', rpc_method_handlers)
+
+
+ # This class is part of an EXPERIMENTAL API.
+class DocReader(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def Read(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/docreader.DocReader/Read',
+            docreader__pb2.ReadRequest.SerializeToString,
+            docreader__pb2.ReadResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
+    @staticmethod
+    def ReadStream(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_stream(
+            request,
+            target,
+            '/docreader.DocReader/ReadStream',
+            docreader__pb2.ReadRequest.SerializeToString,
+            docreader__pb2.ReadStreamResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
+    @staticmethod
+    def ListEngines(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/docreader.DocReader/ListEngines',
+            docreader__pb2.ListEnginesRequest.SerializeToString,
+            docreader__pb2.ListEnginesResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)