Skip to content

Commit 67cf96b

Browse files
birdnamoobirdnamoo
committed
chore: bump version to 0.10.4, update native readers for unpdf/unhwp 0.2.x API
- Update Undoc 0.1.13 → 0.1.16, Unhwp 0.1.16 → 0.2.1, Unpdf 0.1.6 → 0.2.1 - Migrate PdfDocumentReader: Pdf static API → UnpdfDocument instance API - Migrate HwpDocumentReader: UnhwpConverter/RenderOptions/DocumentFormat → UnhwpDocument/MarkdownOptions - Migrate MultiModalPdfDocumentReader: Pdf.ExtractImages → UnpdfDocument.GetResourceIds/GetResourceData
1 parent 5b9908a commit 67cf96b

5 files changed

Lines changed: 95 additions & 139 deletions

File tree

‎Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<Project>
33
<!-- Version Information (can be overridden by build scripts) -->
44
<PropertyGroup>
5-
<VersionPrefix>0.10.3</VersionPrefix>
5+
<VersionPrefix>0.10.4</VersionPrefix>
66
<AssemblyVersion>$(VersionPrefix)</AssemblyVersion>
77
<FileVersion>$(VersionPrefix)</FileVersion>
88
<PackageVersion Condition="'$(PackageVersion)' == ''">$(VersionPrefix)</PackageVersion>

‎Directory.Packages.props‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@
2424
<PackageVersion Include="CsvHelper" Version="33.1.0" />
2525
<PackageVersion Include="Markdig" Version="0.45.0" />
2626
<!-- Native Document Readers (Rust FFI) -->
27-
<PackageVersion Include="Undoc" Version="0.1.13" />
28-
<PackageVersion Include="Unhwp" Version="0.1.16" />
29-
<PackageVersion Include="Unpdf" Version="0.1.6" />
27+
<PackageVersion Include="Undoc" Version="0.1.16" />
28+
<PackageVersion Include="Unhwp" Version="0.2.1" />
29+
<PackageVersion Include="Unpdf" Version="0.2.1" />
3030
<!-- Test-only: Used for programmatic test file generation -->
3131
<PackageVersion Include="DocumentFormat.OpenXml" Version="3.4.1" />
3232
</ItemGroup>

‎src/FileFlux.Core/Infrastructure/Readers/HwpDocumentReader.cs‎

Lines changed: 49 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -55,23 +55,12 @@ public async Task<ReadResult> ReadAsync(string filePath, CancellationToken cance
5555
ReaderType = ReaderType
5656
};
5757

58-
// Detect format using Unhwp native library
59-
var format = UnhwpConverter.DetectFormat(filePath);
60-
result.DocumentProps["hwp_format"] = format switch
61-
{
62-
DocumentFormat.Hwp5 => "HWP5",
63-
DocumentFormat.Hwpx => "HWPX",
64-
DocumentFormat.Hwp3 => "HWP3",
65-
_ => "Unknown"
66-
};
67-
58+
result.DocumentProps["hwp_format"] = extension == ".hwpx" ? "HWPX" : "HWP5";
6859
result.DocumentProps["file_type"] = "hwp_document";
6960

70-
// Get document info using Parse
71-
using var parseResult = UnhwpConverter.Parse(filePath);
72-
result.DocumentProps["section_count"] = parseResult.SectionCount;
73-
result.DocumentProps["paragraph_count"] = parseResult.ParagraphCount;
74-
result.DocumentProps["image_count"] = parseResult.ImageCount;
61+
using var doc = UnhwpDocument.ParseFile(filePath);
62+
result.DocumentProps["section_count"] = doc.SectionCount;
63+
result.DocumentProps["resource_count"] = doc.ResourceCount;
7564

7665
// HWP documents are treated as single logical page
7766
result.Pages.Add(new PageInfo
@@ -81,7 +70,7 @@ public async Task<ReadResult> ReadAsync(string filePath, CancellationToken cance
8170
Props =
8271
{
8372
["file_type"] = "hwp_document",
84-
["format"] = format.ToString()
73+
["format"] = extension == ".hwpx" ? "HWPX" : "HWP5"
8574
}
8675
});
8776

@@ -127,14 +116,12 @@ public async Task<ReadResult> ReadAsync(Stream stream, string fileName, Cancella
127116
ReaderType = ReaderType
128117
};
129118

130-
// Detect format by extension for stream
131119
result.DocumentProps["hwp_format"] = extension == ".hwpx" ? "HWPX" : "HWP5";
132120
result.DocumentProps["file_type"] = "hwp_document";
133121

134-
using var parseResult = UnhwpConverter.ParseBytes(bytes);
135-
result.DocumentProps["section_count"] = parseResult.SectionCount;
136-
result.DocumentProps["paragraph_count"] = parseResult.ParagraphCount;
137-
result.DocumentProps["image_count"] = parseResult.ImageCount;
122+
using var doc = UnhwpDocument.ParseBytes(bytes);
123+
result.DocumentProps["section_count"] = doc.SectionCount;
124+
result.DocumentProps["resource_count"] = doc.ResourceCount;
138125

139126
result.Pages.Add(new PageInfo
140127
{
@@ -218,45 +205,38 @@ private static RawContent ExtractHwpContent(string filePath, CancellationToken c
218205
var structuralHints = new Dictionary<string, object>();
219206
var extractedImages = new List<ImageInfo>();
220207

221-
// Convert to Markdown with cleanup using Unhwp native library
222-
using var parseResult = UnhwpConverter.Parse(filePath, new RenderOptions
208+
// Parse and convert to Markdown using Unhwp native library
209+
using var doc = UnhwpDocument.ParseFile(filePath);
210+
var markdown = doc.ToMarkdown(new MarkdownOptions
223211
{
224212
IncludeFrontmatter = false,
225213
EscapeSpecialChars = false,
226-
PreserveLineBreaks = false,
227-
TableFallback = TableFallback.Markdown
214+
ParagraphSpacing = false
228215
});
229216

230-
var markdown = parseResult.Markdown;
231-
232217
// Remove null bytes
233218
markdown = TextSanitizer.RemoveNullBytes(markdown);
234219

235-
// Detect format
236-
var format = UnhwpConverter.DetectFormat(filePath);
237-
structuralHints["hwp_format"] = format switch
238-
{
239-
DocumentFormat.Hwp5 => "HWP5",
240-
DocumentFormat.Hwpx => "HWPX",
241-
DocumentFormat.Hwp3 => "HWP3",
242-
_ => "Unknown"
243-
};
244-
245-
structuralHints["section_count"] = parseResult.SectionCount;
246-
structuralHints["paragraph_count"] = parseResult.ParagraphCount;
220+
structuralHints["hwp_format"] = extension == ".hwpx" ? "HWPX" : "HWP5";
221+
structuralHints["section_count"] = doc.SectionCount;
247222

248-
// Extract embedded images
249-
foreach (var image in parseResult.Images)
223+
// Extract embedded resources (images)
224+
var resourceIds = doc.GetResourceIds();
225+
foreach (var id in resourceIds)
250226
{
251-
var imageInfo = new ImageInfo
227+
var data = doc.GetResourceData(id);
228+
if (data != null)
252229
{
253-
Id = image.Name,
254-
MimeType = GuessMimeType(image.Name),
255-
Data = image.Data,
256-
OriginalSize = image.Data.Length,
257-
SourceUrl = $"embedded:{image.Name}"
258-
};
259-
extractedImages.Add(imageInfo);
230+
var imageInfo = new ImageInfo
231+
{
232+
Id = id,
233+
MimeType = GuessMimeType(id),
234+
Data = data,
235+
OriginalSize = data.Length,
236+
SourceUrl = $"embedded:{id}"
237+
};
238+
extractedImages.Add(imageInfo);
239+
}
260240
}
261241

262242
structuralHints["file_type"] = "hwp_document";
@@ -305,38 +285,38 @@ private static RawContent ExtractHwpContentFromBytes(byte[] bytes, string fileNa
305285
var structuralHints = new Dictionary<string, object>();
306286
var extractedImages = new List<ImageInfo>();
307287

308-
// Convert to Markdown with cleanup using Unhwp native library
309-
using var parseResult = UnhwpConverter.ParseBytes(bytes, new RenderOptions
288+
// Parse and convert to Markdown using Unhwp native library
289+
using var doc = UnhwpDocument.ParseBytes(bytes);
290+
var markdown = doc.ToMarkdown(new MarkdownOptions
310291
{
311292
IncludeFrontmatter = false,
312293
EscapeSpecialChars = false,
313-
PreserveLineBreaks = false,
314-
TableFallback = TableFallback.Markdown
294+
ParagraphSpacing = false
315295
});
316296

317-
var markdown = parseResult.Markdown;
318-
319297
// Remove null bytes
320298
markdown = TextSanitizer.RemoveNullBytes(markdown);
321299

322-
// Detect format by extension
323300
structuralHints["hwp_format"] = extension == ".hwpx" ? "HWPX" : "HWP5";
301+
structuralHints["section_count"] = doc.SectionCount;
324302

325-
structuralHints["section_count"] = parseResult.SectionCount;
326-
structuralHints["paragraph_count"] = parseResult.ParagraphCount;
327-
328-
// Extract embedded images
329-
foreach (var image in parseResult.Images)
303+
// Extract embedded resources (images)
304+
var resourceIds = doc.GetResourceIds();
305+
foreach (var id in resourceIds)
330306
{
331-
var imageInfo = new ImageInfo
307+
var data = doc.GetResourceData(id);
308+
if (data != null)
332309
{
333-
Id = image.Name,
334-
MimeType = GuessMimeType(image.Name),
335-
Data = image.Data,
336-
OriginalSize = image.Data.Length,
337-
SourceUrl = $"embedded:{image.Name}"
338-
};
339-
extractedImages.Add(imageInfo);
310+
var imageInfo = new ImageInfo
311+
{
312+
Id = id,
313+
MimeType = GuessMimeType(id),
314+
Data = data,
315+
OriginalSize = data.Length,
316+
SourceUrl = $"embedded:{id}"
317+
};
318+
extractedImages.Add(imageInfo);
319+
}
340320
}
341321

342322
structuralHints["file_type"] = "hwp_document";

‎src/FileFlux.Core/Infrastructure/Readers/PdfDocumentReader.cs‎

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -55,26 +55,18 @@ public async Task<ReadResult> ReadAsync(string filePath, CancellationToken cance
5555
};
5656

5757
// Get document info using Unpdf
58-
var docInfo = Pdf.GetInfo(filePath);
59-
60-
if (!string.IsNullOrWhiteSpace(docInfo.Title))
61-
result.DocumentProps["title"] = docInfo.Title;
62-
if (!string.IsNullOrWhiteSpace(docInfo.Author))
63-
result.DocumentProps["author"] = docInfo.Author;
64-
if (!string.IsNullOrWhiteSpace(docInfo.Subject))
65-
result.DocumentProps["subject"] = docInfo.Subject;
66-
if (!string.IsNullOrWhiteSpace(docInfo.Creator))
67-
result.DocumentProps["creator"] = docInfo.Creator;
68-
if (!string.IsNullOrWhiteSpace(docInfo.Producer))
69-
result.DocumentProps["producer"] = docInfo.Producer;
70-
if (!string.IsNullOrWhiteSpace(docInfo.PdfVersion))
71-
result.DocumentProps["pdf_version"] = docInfo.PdfVersion;
72-
73-
result.DocumentProps["page_count"] = docInfo.PageCount;
74-
result.DocumentProps["encrypted"] = docInfo.Encrypted;
58+
using var doc = UnpdfDocument.ParseFile(filePath);
59+
60+
if (!string.IsNullOrWhiteSpace(doc.Title))
61+
result.DocumentProps["title"] = doc.Title;
62+
if (!string.IsNullOrWhiteSpace(doc.Author))
63+
result.DocumentProps["author"] = doc.Author;
64+
65+
var pageCount = doc.SectionCount;
66+
result.DocumentProps["page_count"] = pageCount;
7567

7668
// Add page info
77-
for (int i = 1; i <= docInfo.PageCount; i++)
69+
for (int i = 1; i <= pageCount; i++)
7870
{
7971
result.Pages.Add(new PageInfo
8072
{
@@ -220,21 +212,19 @@ private static RawContent ExtractPdfContent(string filePath, ExtractOptions? opt
220212

221213
cancellationToken.ThrowIfCancellationRequested();
222214

223-
// Convert to Markdown using Unpdf native library
224-
var markdown = Pdf.ToMarkdown(filePath);
215+
// Parse and convert to Markdown using Unpdf native library
216+
using var doc = UnpdfDocument.ParseFile(filePath);
217+
var markdown = doc.ToMarkdown();
225218

226219
// Remove null bytes
227220
markdown = TextSanitizer.RemoveNullBytes(markdown);
228221

229-
// Get document metadata
230-
var docInfo = Pdf.GetInfo(filePath);
231-
232-
if (!string.IsNullOrWhiteSpace(docInfo.Title))
233-
structuralHints["document_title"] = docInfo.Title;
234-
if (!string.IsNullOrWhiteSpace(docInfo.Author))
235-
structuralHints["author"] = docInfo.Author;
222+
if (!string.IsNullOrWhiteSpace(doc.Title))
223+
structuralHints["document_title"] = doc.Title;
224+
if (!string.IsNullOrWhiteSpace(doc.Author))
225+
structuralHints["author"] = doc.Author;
236226

237-
structuralHints["page_count"] = docInfo.PageCount;
227+
structuralHints["page_count"] = doc.SectionCount;
238228

239229
// Update structural hints
240230
structuralHints["file_type"] = "pdf_document";

‎src/FileFlux/Infrastructure/Readers/MultiModalPdfDocumentReader.cs‎

Lines changed: 24 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -97,58 +97,55 @@ private async Task<RawContent> ExtractWithImageProcessing(
9797
// Prepare document context (for relevance evaluation)
9898
var documentContext = PrepareDocumentContext(baseContent, filePath);
9999

100-
// Create temp directory for image extraction
101-
var tempDir = Path.Combine(Path.GetTempPath(), $"fileflux_pdf_images_{Guid.NewGuid():N}");
102-
103100
try
104101
{
105-
Directory.CreateDirectory(tempDir);
106-
107-
// Extract images using Unpdf
108-
var extractedImages = Pdf.ExtractImages(filePath, tempDir);
102+
// Extract images using Unpdf native library
103+
using var doc = UnpdfDocument.ParseFile(filePath);
104+
var resourceIds = doc.GetResourceIds();
109105

110106
var imageCount = 0;
111107
var includedImageCount = 0;
112108
var excludedImageCount = 0;
113109

114-
// Process extracted images
115-
var imagesToProcess = new List<(ExtractedImage Image, byte[] Bytes)>();
110+
// Process extracted image resources
111+
var imagesToProcess = new List<byte[]>();
116112

117-
foreach (var image in extractedImages)
113+
foreach (var id in resourceIds)
118114
{
119115
cancellationToken.ThrowIfCancellationRequested();
120116

121-
// Filter decorative images by size
122-
var width = image.Width ?? 0;
123-
var height = image.Height ?? 0;
124-
125-
if (ImageProcessingConstants.IsDecorativeImage(width, height))
126-
{
117+
var imageBytes = doc.GetResourceData(id);
118+
if (imageBytes == null || imageBytes.Length <= 100)
127119
continue;
128-
}
129120

130-
// Read image bytes from file
131-
if (File.Exists(image.Path))
121+
// Filter decorative images by size using resource metadata
122+
int width = 0, height = 0;
123+
using var resourceInfo = doc.GetResourceInfo(id);
124+
if (resourceInfo != null)
132125
{
133-
var imageBytes = await File.ReadAllBytesAsync(image.Path, cancellationToken);
134-
if (imageBytes.Length > 100)
135-
{
136-
imagesToProcess.Add((image, imageBytes));
137-
}
126+
if (resourceInfo.RootElement.TryGetProperty("width", out var w))
127+
width = w.GetInt32();
128+
if (resourceInfo.RootElement.TryGetProperty("height", out var h))
129+
height = h.GetInt32();
138130
}
131+
132+
if (ImageProcessingConstants.IsDecorativeImage(width, height))
133+
continue;
134+
135+
imagesToProcess.Add(imageBytes);
139136
}
140137

141138
// Process images through IImageToTextService
142139
var imageTextResults = new List<ImageToTextResult>();
143-
foreach (var (image, bytes) in imagesToProcess)
140+
foreach (var bytes in imagesToProcess)
144141
{
145142
cancellationToken.ThrowIfCancellationRequested();
146143

147144
try
148145
{
149146
var options = new ImageToTextOptions
150147
{
151-
ImageTypeHint = "document", // PDF images are typically document/chart
148+
ImageTypeHint = "document",
152149
Quality = "medium",
153150
ExtractStructure = true
154151
};
@@ -251,18 +248,7 @@ private async Task<RawContent> ExtractWithImageProcessing(
251248
}
252249
finally
253250
{
254-
// Cleanup temp directory
255-
try
256-
{
257-
if (Directory.Exists(tempDir))
258-
{
259-
Directory.Delete(tempDir, recursive: true);
260-
}
261-
}
262-
catch
263-
{
264-
// Ignore cleanup errors
265-
}
251+
// No temp files to clean up (resources extracted directly from native library)
266252
}
267253

268254
return new RawContent

0 commit comments

Comments
 (0)