diff --git a/packages/components/nodes/documentloaders/Csv/Csv.png b/packages/components/nodes/documentloaders/Csv/Csv.png new file mode 100644 index 000000000..41b84e16a Binary files /dev/null and b/packages/components/nodes/documentloaders/Csv/Csv.png differ diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts new file mode 100644 index 000000000..db1a3bac6 --- /dev/null +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -0,0 +1,59 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { CSVLoader } from "langchain/document_loaders/fs/csv"; + +class Csv_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Csv File' + this.name = 'csvFile' + this.type = 'Document' + this.icon = 'Csv.png' + this.category = 'Document Loaders' + this.description = `Load data from CSV files` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Csv File', + name: 'csvFile', + type: 'file', + fileType: '.csv' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const csvFileBase64 = nodeData.inputs?.csvFile as string + const splitDataURI = csvFileBase64.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + + const blob = new Blob([bf]) + const loader = new CSVLoader(blob) + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + return docs + } else { + const docs = await loader.load() + return docs + } + } +} + +module.exports = { nodeClass: Csv_DocumentLoaders } diff --git a/packages/components/package.json b/packages/components/package.json index d2d680376..d75cda9d5 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -21,6 +21,7 @@ "@pinecone-database/pinecone": "^0.0.12", "axios": "^0.27.2", "chromadb": "^1.3.1", + "d3-dsv": "2", "dotenv": "^16.0.0", "express": "^4.17.3", "form-data": "^4.0.0",