Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add multiple loaders and documents

This commit is contained in:
Henry 2023-04-30 22:57:12 +01:00
parent e333db8245
commit 40b655e66b
7 changed files with 96 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
packages/components/nodes/documentloaders/Folder/Folder.ts Normal file
View File

@ -0,0 +1,67 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { TextLoader } from 'langchain/document_loaders/fs/text'
import { DirectoryLoader } from 'langchain/document_loaders/fs/directory'
import { JSONLoader } from 'langchain/document_loaders/fs/json'
import { CSVLoader } from 'langchain/document_loaders/fs/csv'
import { PDFLoader } from 'langchain/document_loaders/fs/pdf'
import { DocxLoader } from 'langchain/document_loaders/fs/docx'
class Folder_DocumentLoaders implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
constructor() {
this.label = 'Folder with Files'
this.name = 'folderFiles'
this.type = 'Document'
this.icon = 'folder.svg'
this.category = 'Document Loaders'
this.description = `Load data from folder with multiple files`
this.baseClasses = [this.type]
this.inputs = [
{
label: 'Folder Path',
name: 'folderPath',
type: 'string',
placeholder: ''
},
{
label: 'Text Splitter',
name: 'textSplitter',
type: 'TextSplitter',
optional: true
}
]
}
async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const folderPath = nodeData.inputs?.folderPath as string
const loader = new DirectoryLoader(folderPath, {
'.json': (path) => new JSONLoader(path),
'.txt': (path) => new TextLoader(path),
'.csv': (path) => new CSVLoader(path),
'.docx': (path) => new DocxLoader(path),
// @ts-ignore
'.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') })
})
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
} else {
const docs = await loader.load()
return docs
}
}
}
module.exports = { nodeClass: Folder_DocumentLoaders }

4
packages/components/nodes/documentloaders/Folder/folder.svg Normal file
View File

@ -0,0 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-folder" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M5 4h4l3 3h7a2 2 0 0 1 2 2v8a2 2 0 0 1 -2 2h-14a2 2 0 0 1 -2 -2v-11a2 2 0 0 1 2 -2"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 397 B

8
packages/components/nodes/vectorstores/Chroma_Upsert/Chroma_Upsert.ts
View File

@ -27,7 +27,8 @@ class ChromaUpsert_VectorStores implements INode {
{
label: 'Document',
name: 'document',
type: 'Document'
type: 'Document',
list: true
},
{
label: 'Embeddings',
@ -60,9 +61,10 @@ class ChromaUpsert_VectorStores implements INode {
const embeddings = nodeData.inputs?.embeddings as Embeddings
const output = nodeData.outputs?.output as string
const flattenDocs = docs.flat()
const finalDocs = []
for (let i = 0; i < docs.length; i += 1) {
finalDocs.push(new Document(docs[i]))
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
}
const vectorStore = await Chroma.fromDocuments(finalDocs, embeddings, {

8
packages/components/nodes/vectorstores/InMemory/InMemoryVectorStore.ts
View File

@ -27,7 +27,8 @@ class InMemoryVectorStore_VectorStores implements INode {
{
label: 'Document',
name: 'document',
type: 'Document'
type: 'Document',
list: true
},
{
label: 'Embeddings',
@ -54,9 +55,10 @@ class InMemoryVectorStore_VectorStores implements INode {
const embeddings = nodeData.inputs?.embeddings as Embeddings
const output = nodeData.outputs?.output as string
const flattenDocs = docs.flat()
const finalDocs = []
for (let i = 0; i < docs.length; i += 1) {
finalDocs.push(new Document(docs[i]))
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
}
const vectorStore = await MemoryVectorStore.fromDocuments(finalDocs, embeddings)

8
packages/components/nodes/vectorstores/Pinecone_Upsert/Pinecone_Upsert.ts
View File

@ -28,7 +28,8 @@ class PineconeUpsert_VectorStores implements INode {
{
label: 'Document',
name: 'document',
type: 'Document'
type: 'Document',
list: true
},
{
label: 'Embeddings',
@ -84,9 +85,10 @@ class PineconeUpsert_VectorStores implements INode {
const pineconeIndex = client.Index(index)
const flattenDocs = docs.flat()
const finalDocs = []
for (let i = 0; i < docs.length; i += 1) {
finalDocs.push(new Document(docs[i]))
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
}
const obj: PineconeLibArgs = {

8
packages/components/nodes/vectorstores/Supabase_Upsert/Supabase_Upsert.ts
View File

@ -28,7 +28,8 @@ class SupabaseUpsert_VectorStores implements INode {
{
label: 'Document',
name: 'document',
type: 'Document'
type: 'Document',
list: true
},
{
label: 'Embeddings',
@ -81,9 +82,10 @@ class SupabaseUpsert_VectorStores implements INode {
const client = createClient(supabaseProjUrl, supabaseApiKey)
const flattenDocs = docs.flat()
const finalDocs = []
for (let i = 0; i < docs.length; i += 1) {
finalDocs.push(new Document(docs[i]))
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
}
const vectorStore = await SupabaseVectorStore.fromDocuments(finalDocs, embeddings, {

8
packages/components/nodes/vectorstores/Weaviate_Upsert/Weaviate_Upsert.ts
View File

@ -28,7 +28,8 @@ class WeaviateUpsert_VectorStores implements INode {
{
label: 'Document',
name: 'document',
type: 'Document'
type: 'Document',
list: true
},
{
label: 'Embeddings',
@ -121,9 +122,10 @@ class WeaviateUpsert_VectorStores implements INode {
const client: WeaviateClient = weaviate.client(clientConfig)
const flattenDocs = docs.flat()
const finalDocs = []
for (let i = 0; i < docs.length; i += 1) {
finalDocs.push(new Document(docs[i]))
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
}
const obj: WeaviateLibArgs = {