From 880754358db6a32030f1ab820140c7bdebc6709f Mon Sep 17 00:00:00 2001 From: apeng-singlestore <127370261+apeng-singlestore@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:54:48 -0700 Subject: [PATCH 1/5] Add singlestore upsert and existing --- package.json | 5 +- .../Singlestore_Existing.ts | 138 ++++++++++++++++ .../Singlestore_Existing/singlestore.svg | 20 +++ .../Singlestore_Upsert/Singlestore_Upsert.ts | 154 ++++++++++++++++++ .../Singlestore_Upsert/singlestore.svg | 20 +++ 5 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts create mode 100644 packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg create mode 100644 packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts create mode 100644 packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg diff --git a/package.json b/package.json index 61ea436b4..8f9004baa 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,6 @@ "*.{js,jsx,ts,tsx,json,md}": "eslint --fix" }, "devDependencies": { - "turbo": "1.7.4", "@babel/preset-env": "^7.19.4", "@babel/preset-typescript": "7.18.6", "@types/express": "^4.17.13", @@ -48,9 +47,13 @@ "pretty-quick": "^3.1.3", "rimraf": "^3.0.2", "run-script-os": "^1.1.6", + "turbo": "1.7.4", "typescript": "^4.8.4" }, "engines": { "node": ">=18.15.0" + }, + "dependencies": { + "mysql2": "^3.5.1" } } diff --git a/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts new file mode 100644 index 000000000..61ae84f60 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts @@ -0,0 +1,138 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams, INodeOptionsValue } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { getBaseClasses } from '../../../src/utils' +import { ConnectionOptions, SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' +import { flatten } from 'lodash' +import { integer } from '@opensearch-project/opensearch/api/types' + +class SingleStoreExisting_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'SingleStore Load Existing Table' + this.name = 'singlestoreExisting' + this.type = 'SingleStore' + this.icon = 'singlestore.svg' + this.category = 'Vector Stores' + this.description = 'Load existing document from SingleStore' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'Port', + name: 'port', + type: 'string' + }, + { + label: 'User', + name: 'user', + type: 'string' + }, + { + label: 'Password', + name: 'password', + type: 'password' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string' + }, + { + label: 'Content Column Name', + name: 'contentColumnName', + type: 'string' + }, + { + label: 'Vector Column Name', + name: 'vectorColumnName', + type: 'string' + }, + { + label: 'Metadata Column Name', + name: 'metadataColumnName', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'SingleStore Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'SingleStore Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const singleStoreConnectionConfig = { + connectionOptions: { + host: nodeData.inputs?.host as string, + port: nodeData.inputs?.port as integer, + user: nodeData.inputs?.user as string, + password: nodeData.inputs?.password as string, + database: nodeData.inputs?.database as string + }, + tableName: nodeData.inputs?.tableName as string, + contentColumnName: nodeData.inputs?.contentColumnName as string, + vectorColumnName: nodeData.inputs?.vectorColumnName as string, + metadataColumnName: nodeData.inputs?.metadataColumnName as string + } as SingleStoreVectorStoreConfig + + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + let vectorStore: SingleStoreVectorStore + + vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig) + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: SingleStoreExisting_VectorStores } diff --git a/packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg b/packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg new file mode 100644 index 000000000..bd8dc8177 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Existing/singlestore.svg @@ -0,0 +1,20 @@ + + + SingleStore + + + + + + + + + + + + + + + + + diff --git a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts new file mode 100644 index 000000000..b5f873a58 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts @@ -0,0 +1,154 @@ +import { INode, INodeData, INodeOutputsValue, INodeParams, INodeOptionsValue } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { getBaseClasses } from '../../../src/utils' +import { ConnectionOptions, SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' +import { flatten } from 'lodash' +import { integer } from '@opensearch-project/opensearch/api/types' +import { MemoryVectorStore } from 'langchain/vectorstores/memory' + +class SingleStoreUpsert_VectorStores implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'SingleStore Upsert Document' + this.name = 'singlestoreUpsert' + this.type = 'SingleStore' + this.icon = 'singlestore.svg' + this.category = 'Vector Stores' + this.description = 'Upsert documents to SingleStore' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document', + list: true + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'Port', + name: 'port', + type: 'string' + }, + { + label: 'User', + name: 'user', + type: 'string' + }, + { + label: 'Password', + name: 'password', + type: 'password' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string' + }, + { + label: 'Content Column Name', + name: 'contentColumnName', + type: 'string' + }, + { + label: 'Vector Column Name', + name: 'vectorColumnName', + type: 'string' + }, + { + label: 'Metadata Column Name', + name: 'metadataColumnName', + type: 'string' + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'SingleStore Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'SingleStore Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)] + } + ] + } + + async init(nodeData: INodeData): Promise { + const singleStoreConnectionConfig = { + connectionOptions: { + host: nodeData.inputs?.host as string, + port: nodeData.inputs?.port as integer, + user: nodeData.inputs?.user as string, + password: nodeData.inputs?.password as string, + database: nodeData.inputs?.database as string + }, + tableName: nodeData.inputs?.tableName as string, + contentColumnName: nodeData.inputs?.contentColumnName as string, + vectorColumnName: nodeData.inputs?.vectorColumnName as string, + metadataColumnName: nodeData.inputs?.metadataColumnName as string + } as SingleStoreVectorStoreConfig + + const docs = nodeData.inputs?.document as Document[] + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseInt(topK, 10) : 4 + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + let vectorStore: SingleStoreVectorStore | MemoryVectorStore + + vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig) + vectorStore.addDocuments.bind(vectorStore)(finalDocs) + + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: SingleStoreUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg b/packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg new file mode 100644 index 000000000..bd8dc8177 --- /dev/null +++ b/packages/components/nodes/vectorstores/Singlestore_Upsert/singlestore.svg @@ -0,0 +1,20 @@ + + + SingleStore + + + + + + + + + + + + + + + + + From 5ce082e34f8a9499cdf195dbc15ce0339a1d935d Mon Sep 17 00:00:00 2001 From: apeng-singlestore <127370261+apeng-singlestore@users.noreply.github.com> Date: Fri, 14 Jul 2023 15:24:24 -0700 Subject: [PATCH 2/5] remove redundant imports and move some options to additional --- .../Singlestore_Existing.ts | 43 +++++++++--------- .../Singlestore_Upsert/Singlestore_Upsert.ts | 44 ++++++++++--------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts index 61ae84f60..3b880b24c 100644 --- a/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts +++ b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts @@ -1,10 +1,7 @@ -import { INode, INodeData, INodeOutputsValue, INodeParams, INodeOptionsValue } from '../../../src/Interface' +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { Embeddings } from 'langchain/embeddings/base' -import { Document } from 'langchain/document' import { getBaseClasses } from '../../../src/utils' -import { ConnectionOptions, SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' -import { flatten } from 'lodash' -import { integer } from '@opensearch-project/opensearch/api/types' +import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' class SingleStoreExisting_VectorStores implements INode { label: string @@ -36,11 +33,6 @@ class SingleStoreExisting_VectorStores implements INode { name: 'host', type: 'string' }, - { - label: 'Port', - name: 'port', - type: 'string' - }, { label: 'User', name: 'user', @@ -59,27 +51,38 @@ class SingleStoreExisting_VectorStores implements INode { { label: 'Table Name', name: 'tableName', - type: 'string' + type: 'string', + placeholder: 'embeddings', + additionalParams: true, + optional: true }, { label: 'Content Column Name', name: 'contentColumnName', - type: 'string' + type: 'string', + placeholder: 'content', + additionalParams: true, + optional: true }, { label: 'Vector Column Name', name: 'vectorColumnName', - type: 'string' + type: 'string', + placeholder: 'vector', + additionalParams: true, + optional: true }, { label: 'Metadata Column Name', name: 'metadataColumnName', - type: 'string' + type: 'string', + placeholder: 'metadata', + additionalParams: true, + optional: true }, { label: 'Top K', name: 'topK', - description: 'Number of top results to fetch. Default to 4', placeholder: '4', type: 'number', additionalParams: true, @@ -104,15 +107,15 @@ class SingleStoreExisting_VectorStores implements INode { const singleStoreConnectionConfig = { connectionOptions: { host: nodeData.inputs?.host as string, - port: nodeData.inputs?.port as integer, + port: 3306, user: nodeData.inputs?.user as string, password: nodeData.inputs?.password as string, database: nodeData.inputs?.database as string }, - tableName: nodeData.inputs?.tableName as string, - contentColumnName: nodeData.inputs?.contentColumnName as string, - vectorColumnName: nodeData.inputs?.vectorColumnName as string, - metadataColumnName: nodeData.inputs?.metadataColumnName as string + ...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}), + ...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}), + ...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}), + ...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {}) } as SingleStoreVectorStoreConfig const embeddings = nodeData.inputs?.embeddings as Embeddings diff --git a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts index b5f873a58..dc59edb6d 100644 --- a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts +++ b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts @@ -2,9 +2,8 @@ import { INode, INodeData, INodeOutputsValue, INodeParams, INodeOptionsValue } f import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' import { getBaseClasses } from '../../../src/utils' -import { ConnectionOptions, SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' +import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' import { flatten } from 'lodash' -import { integer } from '@opensearch-project/opensearch/api/types' import { MemoryVectorStore } from 'langchain/vectorstores/memory' class SingleStoreUpsert_VectorStores implements INode { @@ -43,11 +42,6 @@ class SingleStoreUpsert_VectorStores implements INode { name: 'host', type: 'string' }, - { - label: 'Port', - name: 'port', - type: 'string' - }, { label: 'User', name: 'user', @@ -66,27 +60,38 @@ class SingleStoreUpsert_VectorStores implements INode { { label: 'Table Name', name: 'tableName', - type: 'string' + type: 'string', + placeholder: 'embeddings', + additionalParams: true, + optional: true }, { label: 'Content Column Name', name: 'contentColumnName', - type: 'string' + type: 'string', + placeholder: 'content', + additionalParams: true, + optional: true }, { label: 'Vector Column Name', name: 'vectorColumnName', - type: 'string' + type: 'string', + placeholder: 'vector', + additionalParams: true, + optional: true }, { label: 'Metadata Column Name', name: 'metadataColumnName', - type: 'string' + type: 'string', + placeholder: 'metadata', + additionalParams: true, + optional: true }, { label: 'Top K', name: 'topK', - description: 'Number of top results to fetch. Default to 4', placeholder: '4', type: 'number', additionalParams: true, @@ -111,15 +116,15 @@ class SingleStoreUpsert_VectorStores implements INode { const singleStoreConnectionConfig = { connectionOptions: { host: nodeData.inputs?.host as string, - port: nodeData.inputs?.port as integer, + port: 3306, user: nodeData.inputs?.user as string, password: nodeData.inputs?.password as string, database: nodeData.inputs?.database as string }, - tableName: nodeData.inputs?.tableName as string, - contentColumnName: nodeData.inputs?.contentColumnName as string, - vectorColumnName: nodeData.inputs?.vectorColumnName as string, - metadataColumnName: nodeData.inputs?.metadataColumnName as string + ...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}), + ...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}), + ...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}), + ...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {}) } as SingleStoreVectorStoreConfig const docs = nodeData.inputs?.document as Document[] @@ -134,11 +139,10 @@ class SingleStoreUpsert_VectorStores implements INode { finalDocs.push(new Document(flattenDocs[i])) } - let vectorStore: SingleStoreVectorStore | MemoryVectorStore - + let vectorStore: SingleStoreVectorStore + vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig) vectorStore.addDocuments.bind(vectorStore)(finalDocs) - if (output === 'retriever') { const retriever = vectorStore.asRetriever(k) From 72489fb078cd77549d7eb6779091b01b02094d1a Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 17 Jul 2023 23:18:36 +0100 Subject: [PATCH 3/5] Fix lint Singlestore_Existing.ts --- .../vectorstores/Singlestore_Existing/Singlestore_Existing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts index 3b880b24c..1b37a4008 100644 --- a/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts +++ b/packages/components/nodes/vectorstores/Singlestore_Existing/Singlestore_Existing.ts @@ -124,7 +124,7 @@ class SingleStoreExisting_VectorStores implements INode { const k = topK ? parseInt(topK, 10) : 4 let vectorStore: SingleStoreVectorStore - + vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig) if (output === 'retriever') { From 587ac62296fd240dde00a54fb7c8591c31f1adfb Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 17 Jul 2023 23:19:38 +0100 Subject: [PATCH 4/5] Fix lint Singlestore_Upsert.ts --- .../vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts index dc59edb6d..eb00381d2 100644 --- a/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts +++ b/packages/components/nodes/vectorstores/Singlestore_Upsert/Singlestore_Upsert.ts @@ -1,10 +1,9 @@ -import { INode, INodeData, INodeOutputsValue, INodeParams, INodeOptionsValue } from '../../../src/Interface' +import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' import { getBaseClasses } from '../../../src/utils' import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore' import { flatten } from 'lodash' -import { MemoryVectorStore } from 'langchain/vectorstores/memory' class SingleStoreUpsert_VectorStores implements INode { label: string From 258496e0c76d6d82bd9e2911fb34982d21bccd08 Mon Sep 17 00:00:00 2001 From: apeng-singlestore <127370261+apeng-singlestore@users.noreply.github.com> Date: Tue, 18 Jul 2023 21:36:23 -0700 Subject: [PATCH 5/5] moved package to packages/components/package.json --- package.json | 3 --- packages/components/package.json | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 8f9004baa..d22d5a7cd 100644 --- a/package.json +++ b/package.json @@ -52,8 +52,5 @@ }, "engines": { "node": ">=18.15.0" - }, - "dependencies": { - "mysql2": "^3.5.1" } } diff --git a/packages/components/package.json b/packages/components/package.json index 32a846109..060519cfd 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -50,7 +50,8 @@ "srt-parser-2": "^1.2.3", "vm2": "^3.9.19", "weaviate-ts-client": "^1.1.0", - "ws": "^8.9.0" + "ws": "^8.9.0", + "mysql2": "^3.5.1" }, "devDependencies": { "@types/gulp": "4.0.9",