diff --git a/packages/components/credentials/PostgresApi.credential.ts b/packages/components/credentials/PostgresApi.credential.ts new file mode 100644 index 000000000..262149cb8 --- /dev/null +++ b/packages/components/credentials/PostgresApi.credential.ts @@ -0,0 +1,31 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class PostgresApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Postgres API' + this.name = 'PostgresApi' + this.version = 1.0 + this.inputs = [ + { + label: 'User', + name: 'user', + type: 'string', + placeholder: '' + }, + { + label: 'Password', + name: 'password', + type: 'password', + placeholder: '' + } + ] + } +} + +module.exports = { credClass: PostgresApi } diff --git a/packages/components/nodes/vectorstores/Postgres_Existing/Postgres_Exisiting.ts b/packages/components/nodes/vectorstores/Postgres_Existing/Postgres_Exisiting.ts new file mode 100644 index 000000000..bbd89e21e --- /dev/null +++ b/packages/components/nodes/vectorstores/Postgres_Existing/Postgres_Exisiting.ts @@ -0,0 +1,172 @@ +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { DataSourceOptions } from 'typeorm' +import { TypeORMVectorStore, TypeORMVectorStoreDocument } from 'langchain/vectorstores/typeorm' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { Pool } from 'pg' + +class Postgres_Existing_VectorStores implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Postgres Load Existing Index' + this.name = 'postgresExistingIndex' + this.version = 1.0 + this.type = 'Postgres' + this.icon = 'postgres.svg' + this.category = 'Vector Stores' + this.description = 'Load existing index from Postgres using pgvector (i.e: Document has been upserted)' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['PostgresApi'] + } + this.inputs = [ + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Port', + name: 'port', + type: 'number', + placeholder: '6432', + optional: true + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string', + placeholder: 'embeddings', + additionalParams: true, + optional: true + }, + { + label: 'Top K', + name: 'topK', + description: 'Number of top results to fetch. Default to 4', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Postgres Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Postgres Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(TypeORMVectorStore)] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const user = getCredentialParam('user', credentialData, nodeData) + const password = getCredentialParam('password', credentialData, nodeData) + const tableName = nodeData.inputs?.tableName as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseFloat(topK) : 4 + + const postgresConnectionOptions = { + type: 'postgres', + host: nodeData.inputs?.host as string, + port: nodeData.inputs?.port as number, + username: user, + password: password, + database: nodeData.inputs?.database as string + } + + const args = { + postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions, + tableName: tableName + } + + const vectorStore = await TypeORMVectorStore.fromDataSource(embeddings, args) + + // Rewrite the method to use pg pool connection instead of the default connection + /* Otherwise a connection error is displayed when the chain tries to execute the function + [chain/start] [1:chain:ConversationalRetrievalQAChain] Entering Chain run with input: { "question": "what the document is about", "chat_history": [] } + [retriever/start] [1:chain:ConversationalRetrievalQAChain > 2:retriever:VectorStoreRetriever] Entering Retriever run with input: { "query": "what the document is about" } + [ERROR]: uncaughtException: Illegal invocation TypeError: Illegal invocation at Socket.ref (node:net:1524:18) at Connection.ref (.../node_modules/pg/lib/connection.js:183:17) at Client.ref (.../node_modules/pg/lib/client.js:591:21) at BoundPool._pulseQueue (/node_modules/pg-pool/index.js:148:28) at .../node_modules/pg-pool/index.js:184:37 at process.processTicksAndRejections (node:internal/process/task_queues:77:11) + */ + vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => { + const embeddingString = `[${query.join(',')}]` + const _filter = filter ?? '{}' + + const queryString = ` + SELECT *, embedding <=> $1 as "_distance" + FROM ${tableName} + WHERE metadata @> $2 + ORDER BY "_distance" ASC + LIMIT $3;` + + const poolOptions = { + host: postgresConnectionOptions.host, + port: postgresConnectionOptions.port, + user: postgresConnectionOptions.username, + password: postgresConnectionOptions.password, + database: postgresConnectionOptions.database + } + const pool = new Pool(poolOptions) + const conn = await pool.connect() + + const documents = await conn.query(queryString, [embeddingString, _filter, k]) + + conn.release() + + const results = [] as [TypeORMVectorStoreDocument, number][] + for (const doc of documents.rows) { + if (doc._distance != null && doc.pageContent != null) { + const document = new Document(doc) as TypeORMVectorStoreDocument + document.id = doc.id + results.push([document, doc._distance]) + } + } + + return results + } + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: Postgres_Existing_VectorStores } diff --git a/packages/components/nodes/vectorstores/Postgres_Existing/postgres.svg b/packages/components/nodes/vectorstores/Postgres_Existing/postgres.svg new file mode 100644 index 000000000..f631e7a84 --- /dev/null +++ b/packages/components/nodes/vectorstores/Postgres_Existing/postgres.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts b/packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts new file mode 100644 index 000000000..30a6e3eff --- /dev/null +++ b/packages/components/nodes/vectorstores/Postgres_Upsert/Postgres_Upsert.ts @@ -0,0 +1,211 @@ +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Embeddings } from 'langchain/embeddings/base' +import { Document } from 'langchain/document' +import { DataSourceOptions } from 'typeorm' +import { TypeORMVectorStore, TypeORMVectorStoreDocument } from 'langchain/vectorstores/typeorm' +import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { flatten } from 'lodash' +import { Pool } from 'pg' + +class PostgresUpsert_VectorStores implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + outputs: INodeOutputsValue[] + + constructor() { + this.label = 'Postgres Upsert Document' + this.name = 'postgresUpsert' + this.version = 1.0 + this.type = 'Postgres' + this.icon = 'postgres.svg' + this.category = 'Vector Stores' + this.description = 'Upsert documents to Postgres using pgvector' + this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + description: 'Needed when using Postgres cloud hosted', + optional: true, + credentialNames: ['PostgresApi'] + } + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document', + list: true + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Host', + name: 'host', + type: 'string' + }, + { + label: 'Database', + name: 'database', + type: 'string' + }, + { + label: 'Port', + name: 'port', + type: 'number', + placeholder: '6432', + optional: true + }, + { + label: 'Table Name', + name: 'tableName', + type: 'string', + placeholder: 'embeddings', + additionalParams: true, + optional: true + }, + { + label: 'Content Column Name', + name: 'contentColumnName', + type: 'string', + placeholder: 'content', + additionalParams: true, + optional: true + }, + { + label: 'Vector Column Name', + name: 'vectorColumnName', + type: 'string', + placeholder: 'vector', + additionalParams: true, + optional: true + }, + { + label: 'Metadata Column Name', + name: 'metadataColumnName', + type: 'string', + placeholder: 'metadata', + additionalParams: true, + optional: true + }, + { + label: 'Top K', + name: 'topK', + placeholder: '4', + type: 'number', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Postgres Retriever', + name: 'retriever', + baseClasses: this.baseClasses + }, + { + label: 'Postgres Vector Store', + name: 'vectorStore', + baseClasses: [this.type, ...getBaseClasses(TypeORMVectorStore)] + } + ] + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const user = getCredentialParam('user', credentialData, nodeData) + const password = getCredentialParam('password', credentialData, nodeData) + const tableName = nodeData.inputs?.tableName as string + const docs = nodeData.inputs?.document as Document[] + const embeddings = nodeData.inputs?.embeddings as Embeddings + const output = nodeData.outputs?.output as string + const topK = nodeData.inputs?.topK as string + const k = topK ? parseFloat(topK) : 4 + + const postgresConnectionOptions = { + type: 'postgres', + host: nodeData.inputs?.host as string, + port: nodeData.inputs?.port as number, + username: user, + password: password, + database: nodeData.inputs?.database as string + } + + const args = { + postgresConnectionOptions: postgresConnectionOptions as DataSourceOptions, + tableName: tableName + } + + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + finalDocs.push(new Document(flattenDocs[i])) + } + + const vectorStore = await TypeORMVectorStore.fromDocuments(finalDocs, embeddings, args) + + // Rewrite the method to use pg pool connection instead of the default connection + /* Otherwise a connection error is displayed when the chain tries to execute the function + [chain/start] [1:chain:ConversationalRetrievalQAChain] Entering Chain run with input: { "question": "what the document is about", "chat_history": [] } + [retriever/start] [1:chain:ConversationalRetrievalQAChain > 2:retriever:VectorStoreRetriever] Entering Retriever run with input: { "query": "what the document is about" } + [ERROR]: uncaughtException: Illegal invocation TypeError: Illegal invocation at Socket.ref (node:net:1524:18) at Connection.ref (.../node_modules/pg/lib/connection.js:183:17) at Client.ref (.../node_modules/pg/lib/client.js:591:21) at BoundPool._pulseQueue (/node_modules/pg-pool/index.js:148:28) at .../node_modules/pg-pool/index.js:184:37 at process.processTicksAndRejections (node:internal/process/task_queues:77:11) + */ + vectorStore.similaritySearchVectorWithScore = async (query: number[], k: number, filter?: any) => { + const embeddingString = `[${query.join(',')}]` + const _filter = filter ?? '{}' + + const queryString = ` + SELECT *, embedding <=> $1 as "_distance" + FROM ${tableName} + WHERE metadata @> $2 + ORDER BY "_distance" ASC + LIMIT $3;` + + const poolOptions = { + host: postgresConnectionOptions.host, + port: postgresConnectionOptions.port, + user: postgresConnectionOptions.username, + password: postgresConnectionOptions.password, + database: postgresConnectionOptions.database + } + const pool = new Pool(poolOptions) + const conn = await pool.connect() + + const documents = await conn.query(queryString, [embeddingString, _filter, k]) + + conn.release() + + const results = [] as [TypeORMVectorStoreDocument, number][] + for (const doc of documents.rows) { + if (doc._distance != null && doc.pageContent != null) { + const document = new Document(doc) as TypeORMVectorStoreDocument + document.id = doc.id + results.push([document, doc._distance]) + } + } + + return results + } + + if (output === 'retriever') { + const retriever = vectorStore.asRetriever(k) + return retriever + } else if (output === 'vectorStore') { + ;(vectorStore as any).k = k + return vectorStore + } + return vectorStore + } +} + +module.exports = { nodeClass: PostgresUpsert_VectorStores } diff --git a/packages/components/nodes/vectorstores/Postgres_Upsert/postgres.svg b/packages/components/nodes/vectorstores/Postgres_Upsert/postgres.svg new file mode 100644 index 000000000..f631e7a84 --- /dev/null +++ b/packages/components/nodes/vectorstores/Postgres_Upsert/postgres.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/components/package.json b/packages/components/package.json index bad9fb749..c48cba73f 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -50,6 +50,7 @@ "notion-to-md": "^3.1.1", "pdf-parse": "^1.1.1", "pdfjs-dist": "^3.7.107", + "pg": "^8.11.2", "playwright": "^1.35.0", "puppeteer": "^20.7.1", "pyodide": ">=0.21.0-alpha.2", @@ -63,6 +64,7 @@ "devDependencies": { "@types/gulp": "4.0.9", "@types/node-fetch": "2.6.2", + "@types/pg": "^8.10.2", "@types/ws": "^8.5.3", "gulp": "^4.0.2", "typescript": "^4.8.4" diff --git a/packages/server/.env.example b/packages/server/.env.example deleted file mode 100644 index bedbf6381..000000000 --- a/packages/server/.env.example +++ /dev/null @@ -1,26 +0,0 @@ -PORT=3000 -PASSPHRASE=MYPASSPHRASE # Passphrase used to create encryption key -# DATABASE_PATH=/your_database_path/.flowise -# APIKEY_PATH=/your_api_key_path/.flowise -# SECRETKEY_PATH=/your_api_key_path/.flowise -# LOG_PATH=/your_log_path/.flowise/logs - -# DATABASE_TYPE=postgres -# DATABASE_PORT="" -# DATABASE_HOST="" -# DATABASE_NAME="flowise" -# DATABASE_USER="" -# DATABASE_PASSWORD="" -# OVERRIDE_DATABASE=true - -# FLOWISE_USERNAME=user -# FLOWISE_PASSWORD=1234 -# DEBUG=true -# LOG_LEVEL=debug (error | warn | info | verbose | debug) -# TOOL_FUNCTION_BUILTIN_DEP=crypto,fs -# TOOL_FUNCTION_EXTERNAL_DEP=moment,lodash - -# LANGCHAIN_TRACING_V2=true -# LANGCHAIN_ENDPOINT=https://api.smith.langchain.com -# LANGCHAIN_API_KEY=your_api_key -# LANGCHAIN_PROJECT=your_project diff --git a/packages/ui/.env.example b/packages/ui/.env.example deleted file mode 100644 index 25241b73a..000000000 --- a/packages/ui/.env.example +++ /dev/null @@ -1 +0,0 @@ -PORT=8080