Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #322 from FlowiseAI/bugfix/Weaviate-PDF 

Bugfix/pdf loader add legacy option
This commit is contained in:
Henry Heng 2023-06-16 15:28:07 +01:00 committed by GitHub
parent 07fb71d716 95d3dd3faf
commit 1d2edecd46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -6,6 +6,8 @@
FROM node:18-alpine
RUN apk add --update libc6-compat python3 make g++
# needed for pdfjs-dist
RUN apk add --no-cache build-base cairo-dev pango-dev
WORKDIR /usr/src/packages

2
docker/Dockerfile
View File

@ -4,6 +4,8 @@ USER root
RUN apk add --no-cache git
RUN apk add --no-cache python3 py3-pip make g++
# needed for pdfjs-dist
RUN apk add --no-cache build-base cairo-dev pango-dev
# You can install a specific version like: flowise@1.0.0
RUN npm install -g flowise

20
packages/components/nodes/documentloaders/Pdf/Pdf.ts
View File

@ -49,6 +49,13 @@ class Pdf_DocumentLoaders implements INode {
],
default: 'perPage'
},
{
label: 'Use Legacy Build',
name: 'legacyBuild',
type: 'boolean',
optional: true,
additionalParams: true
},
{
label: 'Metadata',
name: 'metadata',
@ -64,6 +71,7 @@ class Pdf_DocumentLoaders implements INode {
const pdfFileBase64 = nodeData.inputs?.pdfFile as string
const usage = nodeData.inputs?.usage as string
const metadata = nodeData.inputs?.metadata
const legacyBuild = nodeData.inputs?.legacyBuild as boolean
let alldocs = []
let files: string[] = []
@ -81,8 +89,9 @@ class Pdf_DocumentLoaders implements INode {
if (usage === 'perFile') {
const loader = new PDFLoader(new Blob([bf]), {
splitPages: false,
// @ts-ignore
pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
pdfjs: () =>
// @ts-ignore
legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
})
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
@ -92,8 +101,11 @@ class Pdf_DocumentLoaders implements INode {
alldocs.push(...docs)
}
} else {
// @ts-ignore
const loader = new PDFLoader(new Blob([bf]), { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') })
const loader = new PDFLoader(new Blob([bf]), {
pdfjs: () =>
// @ts-ignore
legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
})
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
alldocs.push(...docs)

3
packages/components/package.json
View File

@ -39,9 +39,10 @@
"moment": "^2.29.3",
"node-fetch": "^2.6.11",
"pdf-parse": "^1.1.1",
"pdfjs-dist": "^3.7.107",
"playwright": "^1.35.0",
"srt-parser-2": "^1.2.3",
"puppeteer": "^20.7.1",
"srt-parser-2": "^1.2.3",
"weaviate-ts-client": "^1.1.0",
"ws": "^8.9.0"
},