diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 8be10f742..4f4f18411 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -31,12 +31,21 @@ class Cheerio_DocumentLoaders implements INode { name: 'textSplitter', type: 'TextSplitter', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata + let url = nodeData.inputs?.url as string var urlPattern = new RegExp( @@ -50,14 +59,31 @@ class Cheerio_DocumentLoaders implements INode { ) // validate fragment locator const loader = new CheerioWebBaseLoader(urlPattern.test(url.trim()) ? url.trim() : '') + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts index 5a4ca76e8..bcaec79f3 100644 --- a/packages/components/nodes/documentloaders/Csv/Csv.ts +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -41,6 +41,13 @@ class Csv_DocumentLoaders implements INode { description: 'Extracting a single column', placeholder: 'Enter column name', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -49,17 +56,35 @@ class Csv_DocumentLoaders implements INode { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const csvFileBase64 = nodeData.inputs?.csvFile as string const columnName = nodeData.inputs?.columnName as string + const metadata = nodeData.inputs?.metadata const blob = new Blob(getBlob(csvFileBase64)) const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim()) + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Docx/Docx.ts b/packages/components/nodes/documentloaders/Docx/Docx.ts index 87a9cc44d..36dd04651 100644 --- a/packages/components/nodes/documentloaders/Docx/Docx.ts +++ b/packages/components/nodes/documentloaders/Docx/Docx.ts @@ -33,6 +33,13 @@ class Docx_DocumentLoaders implements INode { name: 'textSplitter', type: 'TextSplitter', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -40,17 +47,35 @@ class Docx_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const docxFileBase64 = nodeData.inputs?.docxFile as string + const metadata = nodeData.inputs?.metadata const blob = new Blob(getBlob(docxFileBase64)) const loader = new DocxLoader(blob) + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index 8767303fe..2290133e4 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -37,6 +37,13 @@ class Folder_DocumentLoaders implements INode { name: 'textSplitter', type: 'TextSplitter', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -44,6 +51,7 @@ class Folder_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const folderPath = nodeData.inputs?.folderPath as string + const metadata = nodeData.inputs?.metadata const loader = new DirectoryLoader(folderPath, { '.json': (path) => new JSONLoader(path), @@ -53,14 +61,31 @@ class Folder_DocumentLoaders implements INode { // @ts-ignore '.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) }) + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Github/Github.ts b/packages/components/nodes/documentloaders/Github/Github.ts index 2340bea74..552790abf 100644 --- a/packages/components/nodes/documentloaders/Github/Github.ts +++ b/packages/components/nodes/documentloaders/Github/Github.ts @@ -45,6 +45,13 @@ class Github_DocumentLoaders implements INode { name: 'textSplitter', type: 'TextSplitter', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -54,6 +61,7 @@ class Github_DocumentLoaders implements INode { const branch = nodeData.inputs?.branch as string const accessToken = nodeData.inputs?.accessToken as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata const options: GithubRepoLoaderParams = { branch, @@ -64,14 +72,31 @@ class Github_DocumentLoaders implements INode { if (accessToken) options.accessToken = accessToken const loader = new GithubRepoLoader(repoLink, options) + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Json/Json.ts b/packages/components/nodes/documentloaders/Json/Json.ts index 3ecdda6bd..46f7704d6 100644 --- a/packages/components/nodes/documentloaders/Json/Json.ts +++ b/packages/components/nodes/documentloaders/Json/Json.ts @@ -41,6 +41,13 @@ class Json_DocumentLoaders implements INode { description: 'Extracting multiple pointers', placeholder: 'Enter pointers name', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -49,6 +56,7 @@ class Json_DocumentLoaders implements INode { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const jsonFileBase64 = nodeData.inputs?.jsonFile as string const pointersName = nodeData.inputs?.pointersName as string + const metadata = nodeData.inputs?.metadata let pointers: string[] = [] if (pointersName) { @@ -58,14 +66,31 @@ class Json_DocumentLoaders implements INode { const blob = new Blob(getBlob(jsonFileBase64)) const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined) + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Notion/Notion.ts b/packages/components/nodes/documentloaders/Notion/Notion.ts index 457c98f82..f5bfcb2ad 100644 --- a/packages/components/nodes/documentloaders/Notion/Notion.ts +++ b/packages/components/nodes/documentloaders/Notion/Notion.ts @@ -33,6 +33,13 @@ class Notion_DocumentLoaders implements INode { name: 'textSplitter', type: 'TextSplitter', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -40,16 +47,34 @@ class Notion_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const notionFolder = nodeData.inputs?.notionFolder as string + const metadata = nodeData.inputs?.metadata const loader = new NotionLoader(notionFolder) + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Pdf/Pdf.ts b/packages/components/nodes/documentloaders/Pdf/Pdf.ts index 59c0fab7a..c27f78091 100644 --- a/packages/components/nodes/documentloaders/Pdf/Pdf.ts +++ b/packages/components/nodes/documentloaders/Pdf/Pdf.ts @@ -49,6 +49,13 @@ class Pdf_DocumentLoaders implements INode { } ], default: 'perPage' + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -57,30 +64,45 @@ class Pdf_DocumentLoaders implements INode { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const pdfFileBase64 = nodeData.inputs?.pdfFile as string const usage = nodeData.inputs?.usage as string + const metadata = nodeData.inputs?.metadata const blob = new Blob(getBlob(pdfFileBase64)) - + let docs = [] if (usage === 'perFile') { // @ts-ignore const loader = new PDFLoader(blob, { splitPages: false, pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } } else { // @ts-ignore const loader = new PDFLoader(blob, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }) if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs } } diff --git a/packages/components/nodes/documentloaders/Text/Text.ts b/packages/components/nodes/documentloaders/Text/Text.ts index 73c48ad32..466c45200 100644 --- a/packages/components/nodes/documentloaders/Text/Text.ts +++ b/packages/components/nodes/documentloaders/Text/Text.ts @@ -33,6 +33,13 @@ class Text_DocumentLoaders implements INode { name: 'textSplitter', type: 'TextSplitter', optional: true + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true } ] } @@ -40,17 +47,34 @@ class Text_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const txtFileBase64 = nodeData.inputs?.txtFile as string + const metadata = nodeData.inputs?.metadata const blob = new Blob(getBlob(txtFileBase64)) const loader = new TextLoader(blob) + let docs = [] if (textSplitter) { - const docs = await loader.loadAndSplit(textSplitter) - return docs + docs = await loader.loadAndSplit(textSplitter) } else { - const docs = await loader.load() - return docs + docs = await loader.load() } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + return docs } } diff --git a/packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts b/packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts index 866fcfed7..04706ed07 100644 --- a/packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts +++ b/packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts @@ -50,6 +50,13 @@ class Pinecone_Existing_VectorStores implements INode { type: 'string', placeholder: 'my-first-namespace', optional: true + }, + { + label: 'Pinecone Metadata Filter', + name: 'pineconeMetadataFilter', + type: 'json', + optional: true, + additionalParams: true } ] this.outputs = [ @@ -71,6 +78,8 @@ class Pinecone_Existing_VectorStores implements INode { const pineconeEnv = nodeData.inputs?.pineconeEnv as string const index = nodeData.inputs?.pineconeIndex as string const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string + const pineconeMetadataFilter = nodeData.inputs?.pineconeMetadataFilter + const embeddings = nodeData.inputs?.embeddings as Embeddings const output = nodeData.outputs?.output as string @@ -87,6 +96,10 @@ class Pinecone_Existing_VectorStores implements INode { } if (pineconeNamespace) obj.namespace = pineconeNamespace + if (pineconeMetadataFilter) { + const metadatafilter = typeof pineconeMetadataFilter === 'object' ? pineconeMetadataFilter : JSON.parse(pineconeMetadataFilter) + obj.filter = metadatafilter + } const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj) diff --git a/packages/components/nodes/vectorstores/Supabase_Existing/Supabase_Exisiting.ts b/packages/components/nodes/vectorstores/Supabase_Existing/Supabase_Exisiting.ts index 26433bc05..f97b18873 100644 --- a/packages/components/nodes/vectorstores/Supabase_Existing/Supabase_Exisiting.ts +++ b/packages/components/nodes/vectorstores/Supabase_Existing/Supabase_Exisiting.ts @@ -1,7 +1,7 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { Embeddings } from 'langchain/embeddings/base' import { getBaseClasses } from '../../../src/utils' -import { SupabaseVectorStore } from 'langchain/vectorstores/supabase' +import { SupabaseLibArgs, SupabaseVectorStore } from 'langchain/vectorstores/supabase' import { createClient } from '@supabase/supabase-js' class Supabase_Existing_VectorStores implements INode { @@ -48,6 +48,13 @@ class Supabase_Existing_VectorStores implements INode { label: 'Query Name', name: 'queryName', type: 'string' + }, + { + label: 'Supabase Metadata Filter', + name: 'supabaseMetadataFilter', + type: 'json', + optional: true, + additionalParams: true } ] this.outputs = [ @@ -70,15 +77,23 @@ class Supabase_Existing_VectorStores implements INode { const tableName = nodeData.inputs?.tableName as string const queryName = nodeData.inputs?.queryName as string const embeddings = nodeData.inputs?.embeddings as Embeddings + const supabaseMetadataFilter = nodeData.inputs?.supabaseMetadataFilter const output = nodeData.outputs?.output as string const client = createClient(supabaseProjUrl, supabaseApiKey) - const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, { + const obj: SupabaseLibArgs = { client, - tableName: tableName, - queryName: queryName - }) + tableName, + queryName + } + + if (supabaseMetadataFilter) { + const metadatafilter = typeof supabaseMetadataFilter === 'object' ? supabaseMetadataFilter : JSON.parse(supabaseMetadataFilter) + obj.filter = metadatafilter + } + + const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj) if (output === 'retriever') { const retriever = vectorStore.asRetriever() diff --git a/packages/server/marketplaces/AutoGPT.json b/packages/server/marketplaces/AutoGPT.json index e7bc8914c..1ec202776 100644 --- a/packages/server/marketplaces/AutoGPT.json +++ b/packages/server/marketplaces/AutoGPT.json @@ -481,6 +481,14 @@ "placeholder": "my-first-namespace", "optional": true, "id": "pineconeExistingIndex_1-input-pineconeNamespace-string" + }, + { + "label": "Pinecone Metadata Filter", + "name": "pineconeMetadataFilter", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "pineconeExistingIndex_1-input-pineconeMetadataFilter-json" } ], "inputAnchors": [ diff --git a/packages/server/marketplaces/BabyAGI.json b/packages/server/marketplaces/BabyAGI.json index aab2e0965..6cb40519d 100644 --- a/packages/server/marketplaces/BabyAGI.json +++ b/packages/server/marketplaces/BabyAGI.json @@ -117,6 +117,14 @@ "placeholder": "my-first-namespace", "optional": true, "id": "pineconeExistingIndex_1-input-pineconeNamespace-string" + }, + { + "label": "Pinecone Metadata Filter", + "name": "pineconeMetadataFilter", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "pineconeExistingIndex_1-input-pineconeMetadataFilter-json" } ], "inputAnchors": [ diff --git a/packages/server/marketplaces/Conversational Retrieval QA Chain.json b/packages/server/marketplaces/Conversational Retrieval QA Chain.json index 8fdea0b58..2aaba4603 100644 --- a/packages/server/marketplaces/Conversational Retrieval QA Chain.json +++ b/packages/server/marketplaces/Conversational Retrieval QA Chain.json @@ -82,6 +82,14 @@ "type": "file", "fileType": ".txt", "id": "textFile_1-input-txtFile-file" + }, + { + "label": "Metadata", + "name": "metadata", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "textFile_1-input-metadata-json" } ], "inputAnchors": [ diff --git a/packages/server/marketplaces/Github Repo QnA.json b/packages/server/marketplaces/Github Repo QnA.json index c0bdda002..2307038a0 100644 --- a/packages/server/marketplaces/Github Repo QnA.json +++ b/packages/server/marketplaces/Github Repo QnA.json @@ -150,6 +150,14 @@ "placeholder": "", "optional": true, "id": "github_1-input-accessToken-password" + }, + { + "label": "Metadata", + "name": "metadata", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "github_1-input-metadata-json" } ], "inputAnchors": [ diff --git a/packages/server/marketplaces/Metadata Filter Load.json b/packages/server/marketplaces/Metadata Filter Load.json new file mode 100644 index 000000000..99506413b --- /dev/null +++ b/packages/server/marketplaces/Metadata Filter Load.json @@ -0,0 +1,414 @@ +{ + "description": "Load existing index with metadata filters and feed into conversational retrieval QA chain", + "nodes": [ + { + "width": 300, + "height": 523, + "id": "openAI_1", + "position": { + "x": 1195.6182217299724, + "y": -12.958591115085468 + }, + "type": "customNode", + "data": { + "id": "openAI_1", + "label": "OpenAI", + "name": "openAI", + "type": "OpenAI", + "baseClasses": ["OpenAI", "BaseLLM", "BaseLanguageModel", "BaseLangChain"], + "category": "LLMs", + "description": "Wrapper around OpenAI large language models", + "inputParams": [ + { + "label": "OpenAI Api Key", + "name": "openAIApiKey", + "type": "password", + "id": "openAI_1-input-openAIApiKey-password" + }, + { + "label": "Model Name", + "name": "modelName", + "type": "options", + "options": [ + { + "label": "text-davinci-003", + "name": "text-davinci-003" + }, + { + "label": "text-davinci-002", + "name": "text-davinci-002" + }, + { + "label": "text-curie-001", + "name": "text-curie-001" + }, + { + "label": "text-babbage-001", + "name": "text-babbage-001" + } + ], + "default": "text-davinci-003", + "optional": true, + "id": "openAI_1-input-modelName-options" + }, + { + "label": "Temperature", + "name": "temperature", + "type": "number", + "default": 0.7, + "optional": true, + "id": "openAI_1-input-temperature-number" + }, + { + "label": "Max Tokens", + "name": "maxTokens", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-maxTokens-number" + }, + { + "label": "Top Probability", + "name": "topP", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-topP-number" + }, + { + "label": "Best Of", + "name": "bestOf", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-bestOf-number" + }, + { + "label": "Frequency Penalty", + "name": "frequencyPenalty", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-frequencyPenalty-number" + }, + { + "label": "Presence Penalty", + "name": "presencePenalty", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-presencePenalty-number" + }, + { + "label": "Batch Size", + "name": "batchSize", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-batchSize-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-timeout-number" + } + ], + "inputAnchors": [], + "inputs": { + "modelName": "text-davinci-003", + "temperature": "0", + "maxTokens": "", + "topP": "", + "bestOf": "", + "frequencyPenalty": "", + "presencePenalty": "", + "batchSize": "", + "timeout": "" + }, + "outputAnchors": [ + { + "id": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain", + "name": "openAI", + "label": "OpenAI", + "type": "OpenAI | BaseLLM | BaseLanguageModel | BaseLangChain" + } + ], + "outputs": {}, + "selected": false + }, + "positionAbsolute": { + "x": 1195.6182217299724, + "y": -12.958591115085468 + }, + "selected": false, + "dragging": false + }, + { + "width": 300, + "height": 329, + "id": "openAIEmbeddings_1", + "position": { + "x": 777.5098693425334, + "y": 308.4221448953297 + }, + "type": "customNode", + "data": { + "id": "openAIEmbeddings_1", + "label": "OpenAI Embeddings", + "name": "openAIEmbeddings", + "type": "OpenAIEmbeddings", + "baseClasses": ["OpenAIEmbeddings", "Embeddings"], + "category": "Embeddings", + "description": "OpenAI API to generate embeddings for a given text", + "inputParams": [ + { + "label": "OpenAI Api Key", + "name": "openAIApiKey", + "type": "password", + "id": "openAIEmbeddings_1-input-openAIApiKey-password" + }, + { + "label": "Strip New Lines", + "name": "stripNewLines", + "type": "boolean", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_1-input-stripNewLines-boolean" + }, + { + "label": "Batch Size", + "name": "batchSize", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_1-input-batchSize-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_1-input-timeout-number" + } + ], + "inputAnchors": [], + "inputs": { + "stripNewLines": "", + "batchSize": "", + "timeout": "" + }, + "outputAnchors": [ + { + "id": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "name": "openAIEmbeddings", + "label": "OpenAIEmbeddings", + "type": "OpenAIEmbeddings | Embeddings" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 777.5098693425334, + "y": 308.4221448953297 + }, + "dragging": false + }, + { + "width": 300, + "height": 279, + "id": "conversationalRetrievalQAChain_1", + "position": { + "x": 1635.097372263702, + "y": 412.82495588415594 + }, + "type": "customNode", + "data": { + "id": "conversationalRetrievalQAChain_1", + "label": "Conversational Retrieval QA Chain", + "name": "conversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain", + "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "BaseLangChain"], + "category": "Chains", + "description": "Document QA - built on RetrievalQAChain to provide a chat history component", + "inputParams": [], + "inputAnchors": [ + { + "label": "LLM", + "name": "llm", + "type": "BaseLLM", + "id": "conversationalRetrievalQAChain_1-input-llm-BaseLLM" + }, + { + "label": "Vector Store Retriever", + "name": "vectorStoreRetriever", + "type": "BaseRetriever", + "id": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever" + } + ], + "inputs": { + "llm": "{{openAI_1.data.instance}}", + "vectorStoreRetriever": "{{pineconeExistingIndex_0.data.instance}}" + }, + "outputAnchors": [ + { + "id": "conversationalRetrievalQAChain_1-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|BaseLangChain", + "name": "conversationalRetrievalQAChain", + "label": "ConversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain | BaseChain | BaseLangChain" + } + ], + "outputs": {}, + "selected": false + }, + "positionAbsolute": { + "x": 1635.097372263702, + "y": 412.82495588415594 + }, + "selected": false, + "dragging": false + }, + { + "width": 300, + "height": 702, + "id": "pineconeExistingIndex_0", + "position": { + "x": 1187.519066203033, + "y": 542.6635399602128 + }, + "type": "customNode", + "data": { + "id": "pineconeExistingIndex_0", + "label": "Pinecone Load Existing Index", + "name": "pineconeExistingIndex", + "type": "Pinecone", + "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"], + "category": "Vector Stores", + "description": "Load existing index from Pinecone (i.e: Document has been upserted)", + "inputParams": [ + { + "label": "Pinecone Api Key", + "name": "pineconeApiKey", + "type": "password", + "id": "pineconeExistingIndex_0-input-pineconeApiKey-password" + }, + { + "label": "Pinecone Environment", + "name": "pineconeEnv", + "type": "string", + "id": "pineconeExistingIndex_0-input-pineconeEnv-string" + }, + { + "label": "Pinecone Index", + "name": "pineconeIndex", + "type": "string", + "id": "pineconeExistingIndex_0-input-pineconeIndex-string" + }, + { + "label": "Pinecone Namespace", + "name": "pineconeNamespace", + "type": "string", + "placeholder": "my-first-namespace", + "optional": true, + "id": "pineconeExistingIndex_0-input-pineconeNamespace-string" + }, + { + "label": "Pinecone Metadata Filter", + "name": "pineconeMetadataFilter", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json" + } + ], + "inputAnchors": [ + { + "label": "Embeddings", + "name": "embeddings", + "type": "Embeddings", + "id": "pineconeExistingIndex_0-input-embeddings-Embeddings" + } + ], + "inputs": { + "embeddings": "{{openAIEmbeddings_1.data.instance}}", + "pineconeEnv": "northamerica-northeast1-gcp", + "pineconeIndex": "myindex", + "pineconeNamespace": "my-namespace", + "pineconeMetadataFilter": "{\"id\":\"doc1\"}" + }, + "outputAnchors": [ + { + "name": "output", + "label": "Output", + "type": "options", + "options": [ + { + "id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", + "name": "retriever", + "label": "Pinecone Retriever", + "type": "Pinecone | VectorStoreRetriever | BaseRetriever" + }, + { + "id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore", + "name": "vectorStore", + "label": "Pinecone Vector Store", + "type": "Pinecone | VectorStore" + } + ], + "default": "retriever" + } + ], + "outputs": { + "output": "retriever" + }, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 1187.519066203033, + "y": 542.6635399602128 + }, + "dragging": false + } + ], + "edges": [ + { + "source": "openAI_1", + "sourceHandle": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain", + "target": "conversationalRetrievalQAChain_1", + "targetHandle": "conversationalRetrievalQAChain_1-input-llm-BaseLLM", + "type": "buttonedge", + "id": "openAI_1-openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-llm-BaseLLM", + "data": { + "label": "" + } + }, + { + "source": "openAIEmbeddings_1", + "sourceHandle": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "target": "pineconeExistingIndex_0", + "targetHandle": "pineconeExistingIndex_0-input-embeddings-Embeddings", + "type": "buttonedge", + "id": "openAIEmbeddings_1-openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-pineconeExistingIndex_0-pineconeExistingIndex_0-input-embeddings-Embeddings", + "data": { + "label": "" + } + }, + { + "source": "pineconeExistingIndex_0", + "sourceHandle": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", + "target": "conversationalRetrievalQAChain_1", + "targetHandle": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever", + "type": "buttonedge", + "id": "pineconeExistingIndex_0-pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever", + "data": { + "label": "" + } + } + ] +} diff --git a/packages/server/marketplaces/Metadata Filter Upsert.json b/packages/server/marketplaces/Metadata Filter Upsert.json new file mode 100644 index 000000000..6913105fd --- /dev/null +++ b/packages/server/marketplaces/Metadata Filter Upsert.json @@ -0,0 +1,662 @@ +{ + "description": "Upsert multiple files with metadata filters and feed into conversational retrieval QA chain", + "nodes": [ + { + "width": 300, + "height": 376, + "id": "recursiveCharacterTextSplitter_1", + "position": { + "x": 347.5233039646277, + "y": 129.29305204134062 + }, + "type": "customNode", + "data": { + "id": "recursiveCharacterTextSplitter_1", + "label": "Recursive Character Text Splitter", + "name": "recursiveCharacterTextSplitter", + "type": "RecursiveCharacterTextSplitter", + "baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"], + "category": "Text Splitters", + "description": "Split documents recursively by different characters - starting with \"\n\n\", then \"\n\", then \" \"", + "inputParams": [ + { + "label": "Chunk Size", + "name": "chunkSize", + "type": "number", + "default": 1000, + "optional": true, + "id": "recursiveCharacterTextSplitter_1-input-chunkSize-number" + }, + { + "label": "Chunk Overlap", + "name": "chunkOverlap", + "type": "number", + "optional": true, + "id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number" + } + ], + "inputAnchors": [], + "inputs": { + "chunkSize": 1000, + "chunkOverlap": "" + }, + "outputAnchors": [ + { + "id": "recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter", + "name": "recursiveCharacterTextSplitter", + "label": "RecursiveCharacterTextSplitter", + "type": "RecursiveCharacterTextSplitter | TextSplitter" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 347.5233039646277, + "y": 129.29305204134062 + }, + "dragging": false + }, + { + "width": 300, + "height": 523, + "id": "openAI_1", + "position": { + "x": 1159.184721109528, + "y": -38.76565405456694 + }, + "type": "customNode", + "data": { + "id": "openAI_1", + "label": "OpenAI", + "name": "openAI", + "type": "OpenAI", + "baseClasses": ["OpenAI", "BaseLLM", "BaseLanguageModel", "BaseLangChain"], + "category": "LLMs", + "description": "Wrapper around OpenAI large language models", + "inputParams": [ + { + "label": "OpenAI Api Key", + "name": "openAIApiKey", + "type": "password", + "id": "openAI_1-input-openAIApiKey-password" + }, + { + "label": "Model Name", + "name": "modelName", + "type": "options", + "options": [ + { + "label": "text-davinci-003", + "name": "text-davinci-003" + }, + { + "label": "text-davinci-002", + "name": "text-davinci-002" + }, + { + "label": "text-curie-001", + "name": "text-curie-001" + }, + { + "label": "text-babbage-001", + "name": "text-babbage-001" + } + ], + "default": "text-davinci-003", + "optional": true, + "id": "openAI_1-input-modelName-options" + }, + { + "label": "Temperature", + "name": "temperature", + "type": "number", + "default": 0.7, + "optional": true, + "id": "openAI_1-input-temperature-number" + }, + { + "label": "Max Tokens", + "name": "maxTokens", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-maxTokens-number" + }, + { + "label": "Top Probability", + "name": "topP", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-topP-number" + }, + { + "label": "Best Of", + "name": "bestOf", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-bestOf-number" + }, + { + "label": "Frequency Penalty", + "name": "frequencyPenalty", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-frequencyPenalty-number" + }, + { + "label": "Presence Penalty", + "name": "presencePenalty", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-presencePenalty-number" + }, + { + "label": "Batch Size", + "name": "batchSize", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-batchSize-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAI_1-input-timeout-number" + } + ], + "inputAnchors": [], + "inputs": { + "modelName": "text-davinci-003", + "temperature": "0", + "maxTokens": "", + "topP": "", + "bestOf": "", + "frequencyPenalty": "", + "presencePenalty": "", + "batchSize": "", + "timeout": "" + }, + "outputAnchors": [ + { + "id": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain", + "name": "openAI", + "label": "OpenAI", + "type": "OpenAI | BaseLLM | BaseLanguageModel | BaseLangChain" + } + ], + "outputs": {}, + "selected": false + }, + "positionAbsolute": { + "x": 1159.184721109528, + "y": -38.76565405456694 + }, + "selected": false, + "dragging": false + }, + { + "width": 300, + "height": 329, + "id": "openAIEmbeddings_1", + "position": { + "x": 749.4044250705479, + "y": 858.4858399327618 + }, + "type": "customNode", + "data": { + "id": "openAIEmbeddings_1", + "label": "OpenAI Embeddings", + "name": "openAIEmbeddings", + "type": "OpenAIEmbeddings", + "baseClasses": ["OpenAIEmbeddings", "Embeddings"], + "category": "Embeddings", + "description": "OpenAI API to generate embeddings for a given text", + "inputParams": [ + { + "label": "OpenAI Api Key", + "name": "openAIApiKey", + "type": "password", + "id": "openAIEmbeddings_1-input-openAIApiKey-password" + }, + { + "label": "Strip New Lines", + "name": "stripNewLines", + "type": "boolean", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_1-input-stripNewLines-boolean" + }, + { + "label": "Batch Size", + "name": "batchSize", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_1-input-batchSize-number" + }, + { + "label": "Timeout", + "name": "timeout", + "type": "number", + "optional": true, + "additionalParams": true, + "id": "openAIEmbeddings_1-input-timeout-number" + } + ], + "inputAnchors": [], + "inputs": { + "stripNewLines": "", + "batchSize": "", + "timeout": "" + }, + "outputAnchors": [ + { + "id": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "name": "openAIEmbeddings", + "label": "OpenAIEmbeddings", + "type": "OpenAIEmbeddings | Embeddings" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 749.4044250705479, + "y": 858.4858399327618 + }, + "dragging": false + }, + { + "width": 300, + "height": 279, + "id": "conversationalRetrievalQAChain_1", + "position": { + "x": 1551.758177447552, + "y": 367.847388647512 + }, + "type": "customNode", + "data": { + "id": "conversationalRetrievalQAChain_1", + "label": "Conversational Retrieval QA Chain", + "name": "conversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain", + "baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "BaseLangChain"], + "category": "Chains", + "description": "Document QA - built on RetrievalQAChain to provide a chat history component", + "inputParams": [], + "inputAnchors": [ + { + "label": "LLM", + "name": "llm", + "type": "BaseLLM", + "id": "conversationalRetrievalQAChain_1-input-llm-BaseLLM" + }, + { + "label": "Vector Store Retriever", + "name": "vectorStoreRetriever", + "type": "BaseRetriever", + "id": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever" + } + ], + "inputs": { + "llm": "{{openAI_1.data.instance}}", + "vectorStoreRetriever": "{{pineconeUpsert_0.data.instance}}" + }, + "outputAnchors": [ + { + "id": "conversationalRetrievalQAChain_1-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|BaseLangChain", + "name": "conversationalRetrievalQAChain", + "label": "ConversationalRetrievalQAChain", + "type": "ConversationalRetrievalQAChain | BaseChain | BaseLangChain" + } + ], + "outputs": {}, + "selected": false + }, + "positionAbsolute": { + "x": 1551.758177447552, + "y": 367.847388647512 + }, + "selected": false, + "dragging": false + }, + { + "width": 300, + "height": 410, + "id": "textFile_0", + "position": { + "x": 756.5586098635717, + "y": -121.81747478707992 + }, + "type": "customNode", + "data": { + "id": "textFile_0", + "label": "Text File", + "name": "textFile", + "type": "Document", + "baseClasses": ["Document"], + "category": "Document Loaders", + "description": "Load data from text files", + "inputParams": [ + { + "label": "Txt File", + "name": "txtFile", + "type": "file", + "fileType": ".txt", + "id": "textFile_0-input-txtFile-file" + }, + { + "label": "Metadata", + "name": "metadata", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "textFile_0-input-metadata-json" + } + ], + "inputAnchors": [ + { + "label": "Text Splitter", + "name": "textSplitter", + "type": "TextSplitter", + "optional": true, + "id": "textFile_0-input-textSplitter-TextSplitter" + } + ], + "inputs": { + "textSplitter": "{{recursiveCharacterTextSplitter_1.data.instance}}", + "metadata": "{\"id\":\"doc1\"}" + }, + "outputAnchors": [ + { + "id": "textFile_0-output-textFile-Document", + "name": "textFile", + "label": "Document", + "type": "Document" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 756.5586098635717, + "y": -121.81747478707992 + }, + "dragging": false + }, + { + "width": 300, + "height": 505, + "id": "pdfFile_0", + "position": { + "x": 752.0044222860163, + "y": 318.11704520478617 + }, + "type": "customNode", + "data": { + "id": "pdfFile_0", + "label": "Pdf File", + "name": "pdfFile", + "type": "Document", + "baseClasses": ["Document"], + "category": "Document Loaders", + "description": "Load data from PDF files", + "inputParams": [ + { + "label": "Pdf File", + "name": "pdfFile", + "type": "file", + "fileType": ".pdf", + "id": "pdfFile_0-input-pdfFile-file" + }, + { + "label": "Usage", + "name": "usage", + "type": "options", + "options": [ + { + "label": "One document per page", + "name": "perPage" + }, + { + "label": "One document per file", + "name": "perFile" + } + ], + "default": "perPage", + "id": "pdfFile_0-input-usage-options" + }, + { + "label": "Metadata", + "name": "metadata", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "pdfFile_0-input-metadata-json" + } + ], + "inputAnchors": [ + { + "label": "Text Splitter", + "name": "textSplitter", + "type": "TextSplitter", + "optional": true, + "id": "pdfFile_0-input-textSplitter-TextSplitter" + } + ], + "inputs": { + "textSplitter": "{{recursiveCharacterTextSplitter_1.data.instance}}", + "usage": "perPage", + "metadata": "{\"id\":\"doc2\"}" + }, + "outputAnchors": [ + { + "id": "pdfFile_0-output-pdfFile-Document", + "name": "pdfFile", + "label": "Document", + "type": "Document" + } + ], + "outputs": {}, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 752.0044222860163, + "y": 318.11704520478617 + }, + "dragging": false + }, + { + "width": 300, + "height": 701, + "id": "pineconeUpsert_0", + "position": { + "x": 1161.8813042660154, + "y": 537.0216614326227 + }, + "type": "customNode", + "data": { + "id": "pineconeUpsert_0", + "label": "Pinecone Upsert Document", + "name": "pineconeUpsert", + "type": "Pinecone", + "baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"], + "category": "Vector Stores", + "description": "Upsert documents to Pinecone", + "inputParams": [ + { + "label": "Pinecone Api Key", + "name": "pineconeApiKey", + "type": "password", + "id": "pineconeUpsert_0-input-pineconeApiKey-password" + }, + { + "label": "Pinecone Environment", + "name": "pineconeEnv", + "type": "string", + "id": "pineconeUpsert_0-input-pineconeEnv-string" + }, + { + "label": "Pinecone Index", + "name": "pineconeIndex", + "type": "string", + "id": "pineconeUpsert_0-input-pineconeIndex-string" + }, + { + "label": "Pinecone Namespace", + "name": "pineconeNamespace", + "type": "string", + "placeholder": "my-first-namespace", + "optional": true, + "id": "pineconeUpsert_0-input-pineconeNamespace-string" + } + ], + "inputAnchors": [ + { + "label": "Document", + "name": "document", + "type": "Document", + "list": true, + "id": "pineconeUpsert_0-input-document-Document" + }, + { + "label": "Embeddings", + "name": "embeddings", + "type": "Embeddings", + "id": "pineconeUpsert_0-input-embeddings-Embeddings" + } + ], + "inputs": { + "document": ["{{pdfFile_0.data.instance}}", "{{textFile_0.data.instance}}"], + "embeddings": "{{openAIEmbeddings_1.data.instance}}", + "pineconeEnv": "northamerica-northeast1-gcp", + "pineconeIndex": "myindex", + "pineconeNamespace": "my-namespace" + }, + "outputAnchors": [ + { + "name": "output", + "label": "Output", + "type": "options", + "options": [ + { + "id": "pineconeUpsert_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", + "name": "retriever", + "label": "Pinecone Retriever", + "type": "Pinecone | VectorStoreRetriever | BaseRetriever" + }, + { + "id": "pineconeUpsert_0-output-vectorStore-Pinecone|VectorStore", + "name": "vectorStore", + "label": "Pinecone Vector Store", + "type": "Pinecone | VectorStore" + } + ], + "default": "retriever" + } + ], + "outputs": { + "output": "retriever" + }, + "selected": false + }, + "selected": false, + "positionAbsolute": { + "x": 1161.8813042660154, + "y": 537.0216614326227 + }, + "dragging": false + } + ], + "edges": [ + { + "source": "openAI_1", + "sourceHandle": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain", + "target": "conversationalRetrievalQAChain_1", + "targetHandle": "conversationalRetrievalQAChain_1-input-llm-BaseLLM", + "type": "buttonedge", + "id": "openAI_1-openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-llm-BaseLLM", + "data": { + "label": "" + } + }, + { + "source": "recursiveCharacterTextSplitter_1", + "sourceHandle": "recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter", + "target": "textFile_0", + "targetHandle": "textFile_0-input-textSplitter-TextSplitter", + "type": "buttonedge", + "id": "recursiveCharacterTextSplitter_1-recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter-textFile_0-textFile_0-input-textSplitter-TextSplitter", + "data": { + "label": "" + } + }, + { + "source": "recursiveCharacterTextSplitter_1", + "sourceHandle": "recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter", + "target": "pdfFile_0", + "targetHandle": "pdfFile_0-input-textSplitter-TextSplitter", + "type": "buttonedge", + "id": "recursiveCharacterTextSplitter_1-recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter-pdfFile_0-pdfFile_0-input-textSplitter-TextSplitter", + "data": { + "label": "" + } + }, + { + "source": "openAIEmbeddings_1", + "sourceHandle": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings", + "target": "pineconeUpsert_0", + "targetHandle": "pineconeUpsert_0-input-embeddings-Embeddings", + "type": "buttonedge", + "id": "openAIEmbeddings_1-openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-pineconeUpsert_0-pineconeUpsert_0-input-embeddings-Embeddings", + "data": { + "label": "" + } + }, + { + "source": "pdfFile_0", + "sourceHandle": "pdfFile_0-output-pdfFile-Document", + "target": "pineconeUpsert_0", + "targetHandle": "pineconeUpsert_0-input-document-Document", + "type": "buttonedge", + "id": "pdfFile_0-pdfFile_0-output-pdfFile-Document-pineconeUpsert_0-pineconeUpsert_0-input-document-Document", + "data": { + "label": "" + } + }, + { + "source": "textFile_0", + "sourceHandle": "textFile_0-output-textFile-Document", + "target": "pineconeUpsert_0", + "targetHandle": "pineconeUpsert_0-input-document-Document", + "type": "buttonedge", + "id": "textFile_0-textFile_0-output-textFile-Document-pineconeUpsert_0-pineconeUpsert_0-input-document-Document", + "data": { + "label": "" + } + }, + { + "source": "pineconeUpsert_0", + "sourceHandle": "pineconeUpsert_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever", + "target": "conversationalRetrievalQAChain_1", + "targetHandle": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever", + "type": "buttonedge", + "id": "pineconeUpsert_0-pineconeUpsert_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever", + "data": { + "label": "" + } + } + ] +} diff --git a/packages/server/marketplaces/Multiple VectorDB.json b/packages/server/marketplaces/Multiple VectorDB.json index d2a6b454c..64abdd9c2 100644 --- a/packages/server/marketplaces/Multiple VectorDB.json +++ b/packages/server/marketplaces/Multiple VectorDB.json @@ -821,6 +821,14 @@ "placeholder": "my-first-namespace", "optional": true, "id": "pineconeExistingIndex_1-input-pineconeNamespace-string" + }, + { + "label": "Pinecone Metadata Filter", + "name": "pineconeMetadataFilter", + "type": "json", + "optional": true, + "additionalParams": true, + "id": "pineconeExistingIndex_1-input-pineconeMetadataFilter-json" } ], "inputAnchors": [ diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index ab6a3c84f..5f5a8d14c 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -404,8 +404,10 @@ export const isSameOverrideConfig = ( existingOverrideConfig?: ICommonObject, newOverrideConfig?: ICommonObject ): boolean => { - // Skip check if its internal call - if (isInternal) return true + if (isInternal) { + if (existingOverrideConfig && Object.keys(existingOverrideConfig).length) return false + return true + } // If existing and new overrideconfig are the same if ( existingOverrideConfig && diff --git a/packages/ui/src/ui-component/json/JsonEditor.js b/packages/ui/src/ui-component/json/JsonEditor.js new file mode 100644 index 000000000..06442df27 --- /dev/null +++ b/packages/ui/src/ui-component/json/JsonEditor.js @@ -0,0 +1,64 @@ +import { useState } from 'react' +import PropTypes from 'prop-types' +import { FormControl } from '@mui/material' +import ReactJson from 'react-json-view' + +export const JsonEditorInput = ({ value, onChange, disabled = false, isDarkMode = false }) => { + const [myValue, setMyValue] = useState(value ? JSON.parse(value) : {}) + + const onClipboardCopy = (e) => { + const src = e.src + if (Array.isArray(src) || typeof src === 'object') { + navigator.clipboard.writeText(JSON.stringify(src, null, ' ')) + } else { + navigator.clipboard.writeText(src) + } + } + + return ( + <> + + {disabled && ( + onClipboardCopy(e)} + quotesOnKeys={false} + displayDataTypes={false} + /> + )} + {!disabled && ( + onClipboardCopy(e)} + onEdit={(edit) => { + setMyValue(edit.updated_src) + onChange(JSON.stringify(edit.updated_src)) + }} + onAdd={() => { + //console.log(add) + }} + onDelete={(deleteobj) => { + setMyValue(deleteobj.updated_src) + onChange(JSON.stringify(deleteobj.updated_src)) + }} + /> + )} + + + ) +} + +JsonEditorInput.propTypes = { + value: PropTypes.string, + onChange: PropTypes.func, + disabled: PropTypes.bool, + isDarkMode: PropTypes.bool +} diff --git a/packages/ui/src/views/canvas/NodeInputHandler.js b/packages/ui/src/views/canvas/NodeInputHandler.js index dfa9943e9..99b0aed40 100644 --- a/packages/ui/src/views/canvas/NodeInputHandler.js +++ b/packages/ui/src/views/canvas/NodeInputHandler.js @@ -1,6 +1,7 @@ import PropTypes from 'prop-types' import { Handle, Position, useUpdateNodeInternals } from 'reactflow' import { useEffect, useRef, useState, useContext } from 'react' +import { useSelector } from 'react-redux' // material-ui import { useTheme, styled } from '@mui/material/styles' @@ -15,6 +16,7 @@ import { File } from 'ui-component/file/File' import { SwitchInput } from 'ui-component/switch/Switch' import { flowContext } from 'store/context/ReactFlowContext' import { isValidConnection, getAvailableNodesForVariable } from 'utils/genericHelper' +import { JsonEditorInput } from 'ui-component/json/JsonEditor' const CustomWidthTooltip = styled(({ className, ...props }) => )({ [`& .${tooltipClasses.tooltip}`]: { @@ -26,6 +28,7 @@ const CustomWidthTooltip = styled(({ className, ...props }) => { const theme = useTheme() + const customization = useSelector((state) => state.customization) const ref = useRef(null) const { reactFlowInstance } = useContext(flowContext) const updateNodeInternals = useUpdateNodeInternals() @@ -166,6 +169,14 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA onDialogConfirm={(newValue, inputParamName) => onExpandDialogSave(newValue, inputParamName)} /> )} + {inputParam.type === 'json' && ( + (data.inputs[inputParam.name] = newValue)} + value={data.inputs[inputParam.name] ?? inputParam.default ?? ''} + isDarkMode={customization.isDarkMode} + /> + )} {inputParam.type === 'options' && (