Upstash Vector Integration (#2284)

* add: upstash integration

* update upstash vector credential labels

* add filter to retriever/vectorstore as arg

* require embeddings

* lint-fix

packages/components/credentials/UpstashVectorApi.credential.ts

@@ -0,0 +1,29 @@
+import { INodeParams, INodeCredential } from '../src/Interface'
+
+class UpstashVectorApi implements INodeCredential {
+    label: string
+    name: string
+    version: number
+    description: string
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'Upstash Vector API'
+        this.name = 'upstashVectorApi'
+        this.version = 1.0
+        this.inputs = [
+            {
+                label: 'Upstash Vector REST URL',
+                name: 'UPSTASH_VECTOR_REST_URL',
+                type: 'string'
+            },
+            {
+                label: 'Upstash Vector REST Token',
+                name: 'UPSTASH_VECTOR_REST_TOKEN',
+                type: 'password'
+            }
+        ]
+    }
+}
+
+module.exports = { credClass: UpstashVectorApi }

packages/components/nodes/vectorstores/Upstash/Upstash.ts

@@ -0,0 +1,179 @@
+import { flatten } from 'lodash'
+import { IndexingResult, INode, INodeOutputsValue, INodeParams, INodeData, ICommonObject } from '../../../src/Interface'
+import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
+import { Embeddings } from '@langchain/core/embeddings'
+import { Document } from '@langchain/core/documents'
+import { UpstashVectorStore } from '@langchain/community/vectorstores/upstash'
+import { Index as UpstashIndex } from '@upstash/vector'
+import { index } from '../../../src/indexing'
+import { resolveVectorStoreOrRetriever } from '../VectorStoreUtils'
+
+type UpstashVectorStoreParams = {
+    index: UpstashIndex
+    filter?: string
+}
+class Upstash_VectorStores implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    badge: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+    credential: INodeParams
+    outputs: INodeOutputsValue[]
+
+    constructor() {
+        this.label = 'Upstash Vector'
+        this.name = 'upstash'
+        this.version = 1.0
+        this.type = 'Upstash'
+        this.icon = 'upstash.svg'
+        this.category = 'Vector Stores'
+        this.description =
+            'Upsert data as embedding or string and perform similarity search with Upstash, the leading serverless data platform'
+        this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
+        this.badge = 'NEW'
+        this.credential = {
+            label: 'Connect Credential',
+            name: 'credential',
+            type: 'credential',
+            description: 'Necessary credentials for the HTTP connection',
+            credentialNames: ['upstashVectorApi']
+        }
+        this.inputs = [
+            {
+                label: 'Document',
+                name: 'document',
+                type: 'Document',
+                list: true,
+                optional: true
+            },
+            {
+                label: 'Embeddings',
+                name: 'embeddings',
+                type: 'Embeddings'
+            },
+            {
+                label: 'Record Manager',
+                name: 'recordManager',
+                type: 'RecordManager',
+                description: 'Keep track of the record to prevent duplication',
+                optional: true
+            },
+            {
+                label: 'Upstash Metadata Filter',
+                name: 'upstashMetadataFilter',
+                type: 'string',
+                optional: true,
+                additionalParams: true
+            },
+            {
+                label: 'Top K',
+                name: 'topK',
+                description: 'Number of top results to fetch. Default to 4',
+                placeholder: '4',
+                type: 'number',
+                additionalParams: true,
+                optional: true
+            }
+        ]
+        this.outputs = [
+            {
+                label: 'Upstash Retriever',
+                name: 'retriever',
+                baseClasses: this.baseClasses
+            },
+            {
+                label: 'Upstash Vector Store',
+                name: 'vectorStore',
+                baseClasses: [this.type, ...getBaseClasses(UpstashVectorStore)]
+            }
+        ]
+    }
+
+    //@ts-ignore
+    vectorStoreMethods = {
+        async upsert(nodeData: INodeData, options: ICommonObject): Promise<Partial<IndexingResult>> {
+            const docs = nodeData.inputs?.document as Document[]
+            const embeddings = nodeData.inputs?.embeddings as Embeddings
+            const recordManager = nodeData.inputs?.recordManager
+
+            const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+            const UPSTASH_VECTOR_REST_URL = getCredentialParam('UPSTASH_VECTOR_REST_URL', credentialData, nodeData)
+            const UPSTASH_VECTOR_REST_TOKEN = getCredentialParam('UPSTASH_VECTOR_REST_TOKEN', credentialData, nodeData)
+
+            const upstashIndex = new UpstashIndex({
+                url: UPSTASH_VECTOR_REST_URL,
+                token: UPSTASH_VECTOR_REST_TOKEN
+            })
+
+            const flattenDocs = docs && docs.length ? flatten(docs) : []
+            const finalDocs = []
+            for (let i = 0; i < flattenDocs.length; i += 1) {
+                if (flattenDocs[i] && flattenDocs[i].pageContent) {
+                    finalDocs.push(new Document(flattenDocs[i]))
+                }
+            }
+
+            const obj = {
+                index: upstashIndex
+            }
+
+            try {
+                if (recordManager) {
+                    const vectorStore = await UpstashVectorStore.fromExistingIndex(embeddings, obj)
+                    await recordManager.createSchema()
+                    const res = await index({
+                        docsSource: finalDocs,
+                        recordManager,
+                        vectorStore,
+                        options: {
+                            cleanup: recordManager?.cleanup,
+                            sourceIdKey: recordManager?.sourceIdKey ?? 'source',
+                            vectorStoreName: UPSTASH_VECTOR_REST_URL
+                        }
+                    })
+
+                    return res
+                } else {
+                    await UpstashVectorStore.fromDocuments(finalDocs, embeddings, obj)
+                    return { numAdded: finalDocs.length, addedDocs: finalDocs }
+                }
+            } catch (e) {
+                throw new Error(e)
+            }
+        }
+    }
+
+    async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
+        const upstashMetadataFilter = nodeData.inputs?.upstashMetadataFilter
+        const embeddings = nodeData.inputs?.embeddings as Embeddings
+
+        const credentialData = await getCredentialData(nodeData.credential ?? '', options)
+        const UPSTASH_VECTOR_REST_URL = getCredentialParam('UPSTASH_VECTOR_REST_URL', credentialData, nodeData)
+        const UPSTASH_VECTOR_REST_TOKEN = getCredentialParam('UPSTASH_VECTOR_REST_TOKEN', credentialData, nodeData)
+
+        const upstashIndex = new UpstashIndex({
+            url: UPSTASH_VECTOR_REST_URL,
+            token: UPSTASH_VECTOR_REST_TOKEN
+        })
+
+        const obj: UpstashVectorStoreParams = {
+            index: upstashIndex
+        }
+
+        if (upstashMetadataFilter) {
+            obj.filter = upstashMetadataFilter
+        }
+
+        const vectorStore = await UpstashVectorStore.fromExistingIndex(embeddings, obj)
+
+        return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter)
+    }
+}
+
+module.exports = { nodeClass: Upstash_VectorStores }

packages/components/nodes/vectorstores/Upstash/upstash.svg

@@ -0,0 +1,15 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="118" height="118" fill="none">
+    <g clip-path="url(#upstash_icon_dark_bg)">
+        <path fill="#00E9A3" d="M15.105 103.244c19.416 19.526 50.895 19.526 70.311 0 19.416-19.526 19.416-51.185 0-70.711l-8.789 8.839c14.562 14.645 14.562 38.388 0 53.033-14.562 14.644-38.171 14.644-52.733 0l-8.789 8.839Z"/>
+        <path fill="#00E9A3" d="M32.683 85.566c9.708 9.763 25.447 9.763 35.155 0 9.708-9.763 9.708-25.592 0-35.355L59.05 59.05c4.854 4.881 4.854 12.796 0 17.677a12.38 12.38 0 0 1-17.578 0l-8.79 8.839Z"/>
+        <path fill="#00E9A3" d="M102.994 14.855c-19.416-19.526-50.895-19.526-70.311 0-19.416 19.527-19.416 51.185 0 70.711l8.788-8.839c-14.561-14.645-14.561-38.388 0-53.033 14.562-14.644 38.172-14.644 52.734 0l8.789-8.839Z"/>
+        <path fill="#00E9A3" d="M85.416 32.533c-9.708-9.763-25.448-9.763-35.156 0-9.708 9.763-9.708 25.592 0 35.355l8.79-8.839c-4.855-4.881-4.855-12.795 0-17.677a12.38 12.38 0 0 1 17.577 0l8.789-8.839Z"/>
+        <path fill="#fff" fill-opacity=".8" d="M102.994 14.855c-19.416-19.526-50.896-19.526-70.312 0-19.416 19.527-19.416 51.185 0 70.711l8.79-8.839c-14.563-14.645-14.563-38.388 0-53.033 14.561-14.644 38.17-14.644 52.732 0l8.79-8.839Z"/>
+        <path fill="#fff" fill-opacity=".8" d="M85.416 32.533c-9.708-9.763-25.448-9.763-35.156 0-9.708 9.763-9.708 25.592 0 35.355l8.79-8.839c-4.855-4.881-4.855-12.795 0-17.677a12.38 12.38 0 0 1 17.577 0l8.789-8.839Z"/>
+    </g>
+    <defs>
+        <clipPath id="upstash_icon_dark_bg">
+            <path fill="#fff" d="M15 0h88v118H15z"/>
+        </clipPath>
+    </defs>
+</svg>

packages/components/package.json

@@ -103,6 +103,7 @@
         "socket.io": "^4.6.1",
         "srt-parser-2": "^1.2.3",
         "typeorm": "^0.3.6",
+        "@upstash/vector": "1.0.7",
         "vm2": "^3.9.19",
         "weaviate-ts-client": "^1.1.0",
         "winston": "^3.9.0",