Merge pull request #540 from apeng-singlestore/feature/singlestore
Add singlestore upsert and existing
This commit is contained in:
Henry Heng
GitHub
commit
491c4be8af
|
|
@ -28,7 +28,6 @@
|
|||
"*.{js,jsx,ts,tsx,json,md}": "eslint --fix"
|
||||
},
|
||||
"devDependencies": {
|
||||
"turbo": "1.7.4",
|
||||
"@babel/preset-env": "^7.19.4",
|
||||
"@babel/preset-typescript": "7.18.6",
|
||||
"@types/express": "^4.17.13",
|
||||
|
|
@ -48,6 +47,7 @@
|
|||
"pretty-quick": "^3.1.3",
|
||||
"rimraf": "^3.0.2",
|
||||
"run-script-os": "^1.1.6",
|
||||
"turbo": "1.7.4",
|
||||
"typescript": "^4.8.4"
|
||||
},
|
||||
"engines": {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,141 @@
|
|||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
|
||||
|
||||
class SingleStoreExisting_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'SingleStore Load Existing Table'
|
||||
this.name = 'singlestoreExisting'
|
||||
this.type = 'SingleStore'
|
||||
this.icon = 'singlestore.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Load existing document from SingleStore'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'User',
|
||||
name: 'user',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Password',
|
||||
name: 'password',
|
||||
type: 'password'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'embeddings',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Content Column Name',
|
||||
name: 'contentColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'content',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Vector Column Name',
|
||||
name: 'vectorColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'vector',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Column Name',
|
||||
name: 'metadataColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'metadata',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'SingleStore Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'SingleStore Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const singleStoreConnectionConfig = {
|
||||
connectionOptions: {
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: 3306,
|
||||
user: nodeData.inputs?.user as string,
|
||||
password: nodeData.inputs?.password as string,
|
||||
database: nodeData.inputs?.database as string
|
||||
},
|
||||
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
|
||||
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
|
||||
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
|
||||
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
|
||||
} as SingleStoreVectorStoreConfig
|
||||
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
let vectorStore: SingleStoreVectorStore
|
||||
|
||||
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: SingleStoreExisting_VectorStores }
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg width="256px" height="256px" viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid">
|
||||
<title>SingleStore</title>
|
||||
<defs>
|
||||
<linearGradient x1="67.3449258%" y1="-26.0044686%" x2="-18.5227789%" y2="22.9877555%" id="singleStoreLinearGradient-1">
|
||||
<stop stop-color="#FF7BFF" offset="0%"></stop>
|
||||
<stop stop-color="#AA00FF" offset="35.0158%"></stop>
|
||||
<stop stop-color="#8800CC" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
<linearGradient x1="36.2591509%" y1="-19.3628763%" x2="111.72205%" y2="44.9975357%" id="singleStoreLinearGradient-2">
|
||||
<stop stop-color="#FF7BFF" offset="3.54358%"></stop>
|
||||
<stop stop-color="#8800CC" offset="57.6537%"></stop>
|
||||
<stop stop-color="#311B92" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<g>
|
||||
<path d="M133.793438,0 C161.220114,7.62806846 186.208847,26.8506986 196.569923,50.3452712 C212.416431,88.4856134 208.759637,136.695058 191.389506,165.376569 C176.761849,188.8709 154.211058,201.381085 128.308006,201.075829 C88.0823106,200.770814 55.4752171,168.732936 55.1704441,128.456768 C55.1704441,88.1803574 86.8634599,54.9221798 128.308006,54.9221798 C135.012288,54.9221798 144.679052,55.851955 155.649674,60.4286222 C155.649674,60.4286222 147.762766,55.757287 127.50695,52.6192355 C69.3015772,44.9912153 0.621898574,89.095884 16.4683968,190.701711 C38.409617,229.757339 80.4639504,256.303989 128.308006,255.997284 C198.703093,255.692994 256.299161,198.024717 255.996071,127.236226 C255.996071,59.498847 200.836263,1.83073691 133.793438,0 Z" fill="url(#singleStoreLinearGradient-1)"></path>
|
||||
<path d="M181.635561,54.0037552 C171.884031,33.5605356 151.771183,17.3889203 127.087223,10.9813448 C121.601791,9.45574074 115.811828,8.84547014 109.412318,8.540359 C99.9653199,8.540359 90.8230945,9.76087603 81.376096,12.2018618 C57.9109865,19.2196838 41.455174,32.950265 31.7034025,43.6293966 C19.20906,57.9701518 10.9810571,72.9211776 6.1052197,87.8722034 C6.1052197,88.1774594 5.8004708,88.4824739 5.8004708,89.0927445 C5.4957219,90.3132857 4.27677462,93.9746678 4.27677462,94.8901944 C3.97202572,95.500465 3.97202572,96.4157502 3.66730098,97.0260207 C3.36255208,98.2465619 3.05780318,99.4668616 2.75307844,100.687403 C2.75307844,100.992659 2.75307844,101.297673 2.44832954,101.602688 C-5.47492441,140.963571 7.68750379,176.286091 15.6107577,189.406305 C17.5925312,192.688049 19.2199033,195.425935 20.8508738,197.938019 C2.87119611,100.298588 54.558966,53.6984992 113.373885,52.477958 C144.152582,51.8679289 174.931278,64.3778723 189.863709,89.3980006 C188.949389,75.6672745 188.035312,68.0392543 181.635561,54.0037552 Z" fill="url(#singleStoreLinearGradient-2)"></path>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
|
|
@ -0,0 +1,157 @@
|
|||
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
|
||||
import { Embeddings } from 'langchain/embeddings/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import { getBaseClasses } from '../../../src/utils'
|
||||
import { SingleStoreVectorStore, SingleStoreVectorStoreConfig } from 'langchain/vectorstores/singlestore'
|
||||
import { flatten } from 'lodash'
|
||||
|
||||
class SingleStoreUpsert_VectorStores implements INode {
|
||||
label: string
|
||||
name: string
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'SingleStore Upsert Document'
|
||||
this.name = 'singlestoreUpsert'
|
||||
this.type = 'SingleStore'
|
||||
this.icon = 'singlestore.svg'
|
||||
this.category = 'Vector Stores'
|
||||
this.description = 'Upsert documents to SingleStore'
|
||||
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
type: 'Document',
|
||||
list: true
|
||||
},
|
||||
{
|
||||
label: 'Embeddings',
|
||||
name: 'embeddings',
|
||||
type: 'Embeddings'
|
||||
},
|
||||
{
|
||||
label: 'Host',
|
||||
name: 'host',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'User',
|
||||
name: 'user',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Password',
|
||||
name: 'password',
|
||||
type: 'password'
|
||||
},
|
||||
{
|
||||
label: 'Database',
|
||||
name: 'database',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Table Name',
|
||||
name: 'tableName',
|
||||
type: 'string',
|
||||
placeholder: 'embeddings',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Content Column Name',
|
||||
name: 'contentColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'content',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Vector Column Name',
|
||||
name: 'vectorColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'vector',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Metadata Column Name',
|
||||
name: 'metadataColumnName',
|
||||
type: 'string',
|
||||
placeholder: 'metadata',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Top K',
|
||||
name: 'topK',
|
||||
placeholder: '4',
|
||||
type: 'number',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'SingleStore Retriever',
|
||||
name: 'retriever',
|
||||
baseClasses: this.baseClasses
|
||||
},
|
||||
{
|
||||
label: 'SingleStore Vector Store',
|
||||
name: 'vectorStore',
|
||||
baseClasses: [this.type, ...getBaseClasses(SingleStoreVectorStore)]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const singleStoreConnectionConfig = {
|
||||
connectionOptions: {
|
||||
host: nodeData.inputs?.host as string,
|
||||
port: 3306,
|
||||
user: nodeData.inputs?.user as string,
|
||||
password: nodeData.inputs?.password as string,
|
||||
database: nodeData.inputs?.database as string
|
||||
},
|
||||
...(nodeData.inputs?.tableName ? { tableName: nodeData.inputs.tableName as string } : {}),
|
||||
...(nodeData.inputs?.contentColumnName ? { contentColumnName: nodeData.inputs.contentColumnName as string } : {}),
|
||||
...(nodeData.inputs?.vectorColumnName ? { vectorColumnName: nodeData.inputs.vectorColumnName as string } : {}),
|
||||
...(nodeData.inputs?.metadataColumnName ? { metadataColumnName: nodeData.inputs.metadataColumnName as string } : {})
|
||||
} as SingleStoreVectorStoreConfig
|
||||
|
||||
const docs = nodeData.inputs?.document as Document[]
|
||||
const embeddings = nodeData.inputs?.embeddings as Embeddings
|
||||
const output = nodeData.outputs?.output as string
|
||||
const topK = nodeData.inputs?.topK as string
|
||||
const k = topK ? parseInt(topK, 10) : 4
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
const finalDocs = []
|
||||
for (let i = 0; i < flattenDocs.length; i += 1) {
|
||||
finalDocs.push(new Document(flattenDocs[i]))
|
||||
}
|
||||
|
||||
let vectorStore: SingleStoreVectorStore
|
||||
|
||||
vectorStore = new SingleStoreVectorStore(embeddings, singleStoreConnectionConfig)
|
||||
vectorStore.addDocuments.bind(vectorStore)(finalDocs)
|
||||
|
||||
if (output === 'retriever') {
|
||||
const retriever = vectorStore.asRetriever(k)
|
||||
return retriever
|
||||
} else if (output === 'vectorStore') {
|
||||
;(vectorStore as any).k = k
|
||||
return vectorStore
|
||||
}
|
||||
return vectorStore
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: SingleStoreUpsert_VectorStores }
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg width="256px" height="256px" viewBox="0 0 256 256" version="1.1" xmlns="http://www.w3.org/2000/svg" preserveAspectRatio="xMidYMid">
|
||||
<title>SingleStore</title>
|
||||
<defs>
|
||||
<linearGradient x1="67.3449258%" y1="-26.0044686%" x2="-18.5227789%" y2="22.9877555%" id="singleStoreLinearGradient-1">
|
||||
<stop stop-color="#FF7BFF" offset="0%"></stop>
|
||||
<stop stop-color="#AA00FF" offset="35.0158%"></stop>
|
||||
<stop stop-color="#8800CC" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
<linearGradient x1="36.2591509%" y1="-19.3628763%" x2="111.72205%" y2="44.9975357%" id="singleStoreLinearGradient-2">
|
||||
<stop stop-color="#FF7BFF" offset="3.54358%"></stop>
|
||||
<stop stop-color="#8800CC" offset="57.6537%"></stop>
|
||||
<stop stop-color="#311B92" offset="100%"></stop>
|
||||
</linearGradient>
|
||||
</defs>
|
||||
<g>
|
||||
<path d="M133.793438,0 C161.220114,7.62806846 186.208847,26.8506986 196.569923,50.3452712 C212.416431,88.4856134 208.759637,136.695058 191.389506,165.376569 C176.761849,188.8709 154.211058,201.381085 128.308006,201.075829 C88.0823106,200.770814 55.4752171,168.732936 55.1704441,128.456768 C55.1704441,88.1803574 86.8634599,54.9221798 128.308006,54.9221798 C135.012288,54.9221798 144.679052,55.851955 155.649674,60.4286222 C155.649674,60.4286222 147.762766,55.757287 127.50695,52.6192355 C69.3015772,44.9912153 0.621898574,89.095884 16.4683968,190.701711 C38.409617,229.757339 80.4639504,256.303989 128.308006,255.997284 C198.703093,255.692994 256.299161,198.024717 255.996071,127.236226 C255.996071,59.498847 200.836263,1.83073691 133.793438,0 Z" fill="url(#singleStoreLinearGradient-1)"></path>
|
||||
<path d="M181.635561,54.0037552 C171.884031,33.5605356 151.771183,17.3889203 127.087223,10.9813448 C121.601791,9.45574074 115.811828,8.84547014 109.412318,8.540359 C99.9653199,8.540359 90.8230945,9.76087603 81.376096,12.2018618 C57.9109865,19.2196838 41.455174,32.950265 31.7034025,43.6293966 C19.20906,57.9701518 10.9810571,72.9211776 6.1052197,87.8722034 C6.1052197,88.1774594 5.8004708,88.4824739 5.8004708,89.0927445 C5.4957219,90.3132857 4.27677462,93.9746678 4.27677462,94.8901944 C3.97202572,95.500465 3.97202572,96.4157502 3.66730098,97.0260207 C3.36255208,98.2465619 3.05780318,99.4668616 2.75307844,100.687403 C2.75307844,100.992659 2.75307844,101.297673 2.44832954,101.602688 C-5.47492441,140.963571 7.68750379,176.286091 15.6107577,189.406305 C17.5925312,192.688049 19.2199033,195.425935 20.8508738,197.938019 C2.87119611,100.298588 54.558966,53.6984992 113.373885,52.477958 C144.152582,51.8679289 174.931278,64.3778723 189.863709,89.3980006 C188.949389,75.6672745 188.035312,68.0392543 181.635561,54.0037552 Z" fill="url(#singleStoreLinearGradient-2)"></path>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
|
|
@ -52,7 +52,8 @@
|
|||
"srt-parser-2": "^1.2.3",
|
||||
"vm2": "^3.9.19",
|
||||
"weaviate-ts-client": "^1.1.0",
|
||||
"ws": "^8.9.0"
|
||||
"ws": "^8.9.0",
|
||||
"mysql2": "^3.5.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/gulp": "4.0.9",
|
||||
|
|
|
|||
Loading…
Reference in New Issue