diff --git a/package.json b/package.json index 59f213b53..27c5358ef 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,6 @@ "*.{js,jsx,ts,tsx,json,md}": "eslint --fix" }, "devDependencies": { - "turbo": "1.7.4", "@babel/preset-env": "^7.19.4", "@babel/preset-typescript": "7.18.6", "@types/express": "^4.17.13", @@ -48,6 +47,7 @@ "pretty-quick": "^3.1.3", "rimraf": "^3.0.2", "run-script-os": "^1.1.6", + "turbo": "1.7.4", "typescript": "^4.8.4" }, "engines": { diff --git a/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts new file mode 100644 index 000000000..1b37a4008 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts @@ -0,0 +1,141 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { getBaseClasses } from '../../../src/utils' +import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' + +class SingleStoreExisting_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'SingleStore Load Existing Table' + this.name = 'singlestoreExisting' + this.type = 'SingleStore' + this.icon = 'singlestore.svg' + this.category = 'Vector Stores' + this.description = 'Load existing document from SingleStore' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'User', + name: 'user', + type: 'string' + }, + { + label: 'Password', + name: 'password', + type: 'password' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string', + placeholder: 'embeddings', + additionalParams: true, + optional: true + }, + { + label: 'Content Column Name', + name: 'contentColumnName', + type: 'string', + placeholder: 'content', + additionalParams: true, + optional: true + }, + { + label: 'Vector Column Name', + name: 'vectorColumnName', + type: 'string', + placeholder: 'vector', + additionalParams: true, + optional: true + }, + { + label: 'Metadata Column Name', + name: 'metadataColumnName', + type: 'string', + placeholder: 'metadata', + additionalParams: true, + optional: true + }, + { + label: 'Top K', + name: 'topK', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'SingleStore Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'SingleStore Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const singleStoreConnectionConfig = { + connectionOptions: { + host: nodeData.inputs?.host as string, + port: 3306, + user: nodeData.inputs?.user as string, + password: nodeData.inputs?.password as string, + database: nodeData.inputs?.database as string + }, + ...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}), + ...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}), + ...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}), + ...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {}) + } as SingleStoreVectorStoreConfig + + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + let vectorStore: SingleStoreVectorStore + + vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: SingleStoreExisting_VectorStores } diff --git a/packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg b/packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg new file mode 100644 index 000000000..bd8dc8177 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg @@ -0,0 +1,20 @@ + + + SingleStore + + + + + + + + + + + + + + + + + diff --git a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts new file mode 100644 index 000000000..eb00381d2 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts @@ -0,0 +1,157 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { getBaseClasses } from '../../../src/utils' +import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' +import { flatten } from 'lodash' + +class SingleStoreUpsert_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'SingleStore Upsert Document' + this.name = 'singlestoreUpsert' + this.type = 'SingleStore' + this.icon = 'singlestore.svg' + this.category = 'Vector Stores' + this.description = 'Upsert documents to SingleStore' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document', + list: true + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'User', + name: 'user', + type: 'string' + }, + { + label: 'Password', + name: 'password', + type: 'password' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string', + placeholder: 'embeddings', + additionalParams: true, + optional: true + }, + { + label: 'Content Column Name', + name: 'contentColumnName', + type: 'string', + placeholder: 'content', + additionalParams: true, + optional: true + }, + { + label: 'Vector Column Name', + name: 'vectorColumnName', + type: 'string', + placeholder: 'vector', + additionalParams: true, + optional: true + }, + { + label: 'Metadata Column Name', + name: 'metadataColumnName', + type: 'string', + placeholder: 'metadata', + additionalParams: true, + optional: true + }, + { + label: 'Top K', + name: 'topK', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'SingleStore Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'SingleStore Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const singleStoreConnectionConfig = { + connectionOptions: { + host: nodeData.inputs?.host as string, + port: 3306, + user: nodeData.inputs?.user as string, + password: nodeData.inputs?.password as string, + database: nodeData.inputs?.database as string + }, + ...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}), + ...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}), + ...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}), + ...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {}) + } as SingleStoreVectorStoreConfig + + const docs = nodeData.inputs?.document as Document[] + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + let vectorStore: SingleStoreVectorStore + + vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig) + vectorStore.addDocuments.bind(vectorStore)(finalDocs) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: SingleStoreUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg b/packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg new file mode 100644 index 000000000..bd8dc8177 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg @@ -0,0 +1,20 @@ + + + SingleStore + + + + + + + + + + + + + + + + + diff --git a/packages/components/package.json b/packages/components/package.json index 37d630c33..14bd20700 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -52,7 +52,8 @@ "srt-parser-2": "^1.2.3", "vm2": "^3.9.19", "weaviate-ts-client": "^1.1.0", - "ws": "^8.9.0" + "ws": "^8.9.0", + "mysql2": "^3.5.1" }, "devDependencies": { "@types/gulp": "4.0.9",