Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #94 from FlowiseAI/feature/MetadataFilter 

Feature/Add metadata filter
This commit is contained in:
Henry Heng 2023-05-12 16:51:16 +01:00 committed by GitHub
parent 7313cdd9c6 f3201aedcb
commit ad5845fa7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1491 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
View File

@ -31,12 +31,21 @@ class Cheerio_DocumentLoaders implements INode {
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata
let url = nodeData.inputs?.url as string
var urlPattern = new RegExp(
@ -50,14 +59,31 @@ class Cheerio_DocumentLoaders implements INode {
) // validate fragment locator
const loader = new CheerioWebBaseLoader(urlPattern.test(url.trim()) ? url.trim() : '')
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

33
packages/components/nodes/documentloaders/Csv/Csv.ts
View File

@ -41,6 +41,13 @@ class Csv_DocumentLoaders implements INode {
description: 'Extracting a single column',
placeholder: 'Enter column name',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -49,17 +56,35 @@ class Csv_DocumentLoaders implements INode {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const csvFileBase64 = nodeData.inputs?.csvFile as string
const columnName = nodeData.inputs?.columnName as string
const metadata = nodeData.inputs?.metadata
const blob = new Blob(getBlob(csvFileBase64))
const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim())
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

33
packages/components/nodes/documentloaders/Docx/Docx.ts
View File

@ -33,6 +33,13 @@ class Docx_DocumentLoaders implements INode {
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -40,17 +47,35 @@ class Docx_DocumentLoaders implements INode {
async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const docxFileBase64 = nodeData.inputs?.docxFile as string
const metadata = nodeData.inputs?.metadata
const blob = new Blob(getBlob(docxFileBase64))
const loader = new DocxLoader(blob)
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

33
packages/components/nodes/documentloaders/Folder/Folder.ts
View File

@ -37,6 +37,13 @@ class Folder_DocumentLoaders implements INode {
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -44,6 +51,7 @@ class Folder_DocumentLoaders implements INode {
async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const folderPath = nodeData.inputs?.folderPath as string
const metadata = nodeData.inputs?.metadata
const loader = new DirectoryLoader(folderPath, {
'.json': (path) => new JSONLoader(path),
@ -53,14 +61,31 @@ class Folder_DocumentLoaders implements INode {
// @ts-ignore
'.pdf': (path) => new PDFLoader(path, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') })
})
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

33
packages/components/nodes/documentloaders/Github/Github.ts
View File

@ -45,6 +45,13 @@ class Github_DocumentLoaders implements INode {
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -54,6 +61,7 @@ class Github_DocumentLoaders implements INode {
const branch = nodeData.inputs?.branch as string
const accessToken = nodeData.inputs?.accessToken as string
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata
const options: GithubRepoLoaderParams = {
branch,
@ -64,14 +72,31 @@ class Github_DocumentLoaders implements INode {
if (accessToken) options.accessToken = accessToken
const loader = new GithubRepoLoader(repoLink, options)
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

33
packages/components/nodes/documentloaders/Json/Json.ts
View File

@ -41,6 +41,13 @@ class Json_DocumentLoaders implements INode {
description: 'Extracting multiple pointers',
placeholder: 'Enter pointers name',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -49,6 +56,7 @@ class Json_DocumentLoaders implements INode {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const jsonFileBase64 = nodeData.inputs?.jsonFile as string
const pointersName = nodeData.inputs?.pointersName as string
const metadata = nodeData.inputs?.metadata
let pointers: string[] = []
if (pointersName) {
@ -58,14 +66,31 @@ class Json_DocumentLoaders implements INode {
const blob = new Blob(getBlob(jsonFileBase64))
const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined)
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

33
packages/components/nodes/documentloaders/Notion/Notion.ts
View File

@ -33,6 +33,13 @@ class Notion_DocumentLoaders implements INode {
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -40,16 +47,34 @@ class Notion_DocumentLoaders implements INode {
async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const notionFolder = nodeData.inputs?.notionFolder as string
const metadata = nodeData.inputs?.metadata
const loader = new NotionLoader(notionFolder)
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

40
packages/components/nodes/documentloaders/Pdf/Pdf.ts
View File

@ -49,6 +49,13 @@ class Pdf_DocumentLoaders implements INode {
}
],
default: 'perPage'
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -57,30 +64,45 @@ class Pdf_DocumentLoaders implements INode {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const pdfFileBase64 = nodeData.inputs?.pdfFile as string
const usage = nodeData.inputs?.usage as string
const metadata = nodeData.inputs?.metadata
const blob = new Blob(getBlob(pdfFileBase64))
let docs = []
if (usage === 'perFile') {
// @ts-ignore
const loader = new PDFLoader(blob, { splitPages: false, pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') })
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
} else {
// @ts-ignore
const loader = new PDFLoader(blob, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') })
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

32
packages/components/nodes/documentloaders/Text/Text.ts
View File

@ -33,6 +33,13 @@ class Text_DocumentLoaders implements INode {
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
]
}
@ -40,17 +47,34 @@ class Text_DocumentLoaders implements INode {
async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const txtFileBase64 = nodeData.inputs?.txtFile as string
const metadata = nodeData.inputs?.metadata
const blob = new Blob(getBlob(txtFileBase64))
const loader = new TextLoader(blob)
let docs = []
if (textSplitter) {
const docs = await loader.loadAndSplit(textSplitter)
return docs
docs = await loader.loadAndSplit(textSplitter)
} else {
const docs = await loader.load()
return docs
docs = await loader.load()
}
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of docs) {
const newdoc = {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
}
finaldocs.push(newdoc)
}
return finaldocs
}
return docs
}
}

13
packages/components/nodes/vectorstores/Pinecone_Existing/Pinecone_Existing.ts
View File

@ -50,6 +50,13 @@ class Pinecone_Existing_VectorStores implements INode {
type: 'string',
placeholder: 'my-first-namespace',
optional: true
},
{
label: 'Pinecone Metadata Filter',
name: 'pineconeMetadataFilter',
type: 'json',
optional: true,
additionalParams: true
}
]
this.outputs = [
@ -71,6 +78,8 @@ class Pinecone_Existing_VectorStores implements INode {
const pineconeEnv = nodeData.inputs?.pineconeEnv as string
const index = nodeData.inputs?.pineconeIndex as string
const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string
const pineconeMetadataFilter = nodeData.inputs?.pineconeMetadataFilter
const embeddings = nodeData.inputs?.embeddings as Embeddings
const output = nodeData.outputs?.output as string
@ -87,6 +96,10 @@ class Pinecone_Existing_VectorStores implements INode {
}
if (pineconeNamespace) obj.namespace = pineconeNamespace
if (pineconeMetadataFilter) {
const metadatafilter = typeof pineconeMetadataFilter === 'object' ? pineconeMetadataFilter : JSON.parse(pineconeMetadataFilter)
obj.filter = metadatafilter
}
const vectorStore = await PineconeStore.fromExistingIndex(embeddings, obj)

25
packages/components/nodes/vectorstores/Supabase_Existing/Supabase_Exisiting.ts
View File

@ -1,7 +1,7 @@
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { Embeddings } from 'langchain/embeddings/base'
import { getBaseClasses } from '../../../src/utils'
import { SupabaseVectorStore } from 'langchain/vectorstores/supabase'
import { SupabaseLibArgs, SupabaseVectorStore } from 'langchain/vectorstores/supabase'
import { createClient } from '@supabase/supabase-js'
class Supabase_Existing_VectorStores implements INode {
@ -48,6 +48,13 @@ class Supabase_Existing_VectorStores implements INode {
label: 'Query Name',
name: 'queryName',
type: 'string'
},
{
label: 'Supabase Metadata Filter',
name: 'supabaseMetadataFilter',
type: 'json',
optional: true,
additionalParams: true
}
]
this.outputs = [
@ -70,15 +77,23 @@ class Supabase_Existing_VectorStores implements INode {
const tableName = nodeData.inputs?.tableName as string
const queryName = nodeData.inputs?.queryName as string
const embeddings = nodeData.inputs?.embeddings as Embeddings
const supabaseMetadataFilter = nodeData.inputs?.supabaseMetadataFilter
const output = nodeData.outputs?.output as string
const client = createClient(supabaseProjUrl, supabaseApiKey)
const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, {
const obj: SupabaseLibArgs = {
client,
tableName: tableName,
queryName: queryName
})
tableName,
queryName
}
if (supabaseMetadataFilter) {
const metadatafilter = typeof supabaseMetadataFilter === 'object' ? supabaseMetadataFilter : JSON.parse(supabaseMetadataFilter)
obj.filter = metadatafilter
}
const vectorStore = await SupabaseVectorStore.fromExistingIndex(embeddings, obj)
if (output === 'retriever') {
const retriever = vectorStore.asRetriever()

8
packages/server/marketplaces/AutoGPT.json
View File

@ -481,6 +481,14 @@
"placeholder": "my-first-namespace",
"optional": true,
"id": "pineconeExistingIndex_1-input-pineconeNamespace-string"
},
{
"label": "Pinecone Metadata Filter",
"name": "pineconeMetadataFilter",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "pineconeExistingIndex_1-input-pineconeMetadataFilter-json"
}
],
"inputAnchors": [

8
packages/server/marketplaces/BabyAGI.json
View File

@ -117,6 +117,14 @@
"placeholder": "my-first-namespace",
"optional": true,
"id": "pineconeExistingIndex_1-input-pineconeNamespace-string"
},
{
"label": "Pinecone Metadata Filter",
"name": "pineconeMetadataFilter",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "pineconeExistingIndex_1-input-pineconeMetadataFilter-json"
}
],
"inputAnchors": [

8
packages/server/marketplaces/Conversational Retrieval QA Chain.json
View File

@ -82,6 +82,14 @@
"type": "file",
"fileType": ".txt",
"id": "textFile_1-input-txtFile-file"
},
{
"label": "Metadata",
"name": "metadata",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "textFile_1-input-metadata-json"
}
],
"inputAnchors": [

8
packages/server/marketplaces/Github Repo QnA.json
View File

@ -150,6 +150,14 @@
"placeholder": "<GITHUB_ACCESS_TOKEN>",
"optional": true,
"id": "github_1-input-accessToken-password"
},
{
"label": "Metadata",
"name": "metadata",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "github_1-input-metadata-json"
}
],
"inputAnchors": [

414
packages/server/marketplaces/Metadata Filter Load.json Normal file
View File

@ -0,0 +1,414 @@
{
"description": "Load existing index with metadata filters and feed into conversational retrieval QA chain",
"nodes": [
{
"width": 300,
"height": 523,
"id": "openAI_1",
"position": {
"x": 1195.6182217299724,
"y": -12.958591115085468
},
"type": "customNode",
"data": {
"id": "openAI_1",
"label": "OpenAI",
"name": "openAI",
"type": "OpenAI",
"baseClasses": ["OpenAI", "BaseLLM", "BaseLanguageModel", "BaseLangChain"],
"category": "LLMs",
"description": "Wrapper around OpenAI large language models",
"inputParams": [
{
"label": "OpenAI Api Key",
"name": "openAIApiKey",
"type": "password",
"id": "openAI_1-input-openAIApiKey-password"
},
{
"label": "Model Name",
"name": "modelName",
"type": "options",
"options": [
{
"label": "text-davinci-003",
"name": "text-davinci-003"
},
{
"label": "text-davinci-002",
"name": "text-davinci-002"
},
{
"label": "text-curie-001",
"name": "text-curie-001"
},
{
"label": "text-babbage-001",
"name": "text-babbage-001"
}
],
"default": "text-davinci-003",
"optional": true,
"id": "openAI_1-input-modelName-options"
},
{
"label": "Temperature",
"name": "temperature",
"type": "number",
"default": 0.7,
"optional": true,
"id": "openAI_1-input-temperature-number"
},
{
"label": "Max Tokens",
"name": "maxTokens",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-maxTokens-number"
},
{
"label": "Top Probability",
"name": "topP",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-topP-number"
},
{
"label": "Best Of",
"name": "bestOf",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-bestOf-number"
},
{
"label": "Frequency Penalty",
"name": "frequencyPenalty",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-frequencyPenalty-number"
},
{
"label": "Presence Penalty",
"name": "presencePenalty",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-presencePenalty-number"
},
{
"label": "Batch Size",
"name": "batchSize",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-batchSize-number"
},
{
"label": "Timeout",
"name": "timeout",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-timeout-number"
}
],
"inputAnchors": [],
"inputs": {
"modelName": "text-davinci-003",
"temperature": "0",
"maxTokens": "",
"topP": "",
"bestOf": "",
"frequencyPenalty": "",
"presencePenalty": "",
"batchSize": "",
"timeout": ""
},
"outputAnchors": [
{
"id": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain",
"name": "openAI",
"label": "OpenAI",
"type": "OpenAI | BaseLLM | BaseLanguageModel | BaseLangChain"
}
],
"outputs": {},
"selected": false
},
"positionAbsolute": {
"x": 1195.6182217299724,
"y": -12.958591115085468
},
"selected": false,
"dragging": false
},
{
"width": 300,
"height": 329,
"id": "openAIEmbeddings_1",
"position": {
"x": 777.5098693425334,
"y": 308.4221448953297
},
"type": "customNode",
"data": {
"id": "openAIEmbeddings_1",
"label": "OpenAI Embeddings",
"name": "openAIEmbeddings",
"type": "OpenAIEmbeddings",
"baseClasses": ["OpenAIEmbeddings", "Embeddings"],
"category": "Embeddings",
"description": "OpenAI API to generate embeddings for a given text",
"inputParams": [
{
"label": "OpenAI Api Key",
"name": "openAIApiKey",
"type": "password",
"id": "openAIEmbeddings_1-input-openAIApiKey-password"
},
{
"label": "Strip New Lines",
"name": "stripNewLines",
"type": "boolean",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_1-input-stripNewLines-boolean"
},
{
"label": "Batch Size",
"name": "batchSize",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_1-input-batchSize-number"
},
{
"label": "Timeout",
"name": "timeout",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_1-input-timeout-number"
}
],
"inputAnchors": [],
"inputs": {
"stripNewLines": "",
"batchSize": "",
"timeout": ""
},
"outputAnchors": [
{
"id": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
"name": "openAIEmbeddings",
"label": "OpenAIEmbeddings",
"type": "OpenAIEmbeddings | Embeddings"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 777.5098693425334,
"y": 308.4221448953297
},
"dragging": false
},
{
"width": 300,
"height": 279,
"id": "conversationalRetrievalQAChain_1",
"position": {
"x": 1635.097372263702,
"y": 412.82495588415594
},
"type": "customNode",
"data": {
"id": "conversationalRetrievalQAChain_1",
"label": "Conversational Retrieval QA Chain",
"name": "conversationalRetrievalQAChain",
"type": "ConversationalRetrievalQAChain",
"baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "BaseLangChain"],
"category": "Chains",
"description": "Document QA - built on RetrievalQAChain to provide a chat history component",
"inputParams": [],
"inputAnchors": [
{
"label": "LLM",
"name": "llm",
"type": "BaseLLM",
"id": "conversationalRetrievalQAChain_1-input-llm-BaseLLM"
},
{
"label": "Vector Store Retriever",
"name": "vectorStoreRetriever",
"type": "BaseRetriever",
"id": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever"
}
],
"inputs": {
"llm": "{{openAI_1.data.instance}}",
"vectorStoreRetriever": "{{pineconeExistingIndex_0.data.instance}}"
},
"outputAnchors": [
{
"id": "conversationalRetrievalQAChain_1-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|BaseLangChain",
"name": "conversationalRetrievalQAChain",
"label": "ConversationalRetrievalQAChain",
"type": "ConversationalRetrievalQAChain | BaseChain | BaseLangChain"
}
],
"outputs": {},
"selected": false
},
"positionAbsolute": {
"x": 1635.097372263702,
"y": 412.82495588415594
},
"selected": false,
"dragging": false
},
{
"width": 300,
"height": 702,
"id": "pineconeExistingIndex_0",
"position": {
"x": 1187.519066203033,
"y": 542.6635399602128
},
"type": "customNode",
"data": {
"id": "pineconeExistingIndex_0",
"label": "Pinecone Load Existing Index",
"name": "pineconeExistingIndex",
"type": "Pinecone",
"baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"],
"category": "Vector Stores",
"description": "Load existing index from Pinecone (i.e: Document has been upserted)",
"inputParams": [
{
"label": "Pinecone Api Key",
"name": "pineconeApiKey",
"type": "password",
"id": "pineconeExistingIndex_0-input-pineconeApiKey-password"
},
{
"label": "Pinecone Environment",
"name": "pineconeEnv",
"type": "string",
"id": "pineconeExistingIndex_0-input-pineconeEnv-string"
},
{
"label": "Pinecone Index",
"name": "pineconeIndex",
"type": "string",
"id": "pineconeExistingIndex_0-input-pineconeIndex-string"
},
{
"label": "Pinecone Namespace",
"name": "pineconeNamespace",
"type": "string",
"placeholder": "my-first-namespace",
"optional": true,
"id": "pineconeExistingIndex_0-input-pineconeNamespace-string"
},
{
"label": "Pinecone Metadata Filter",
"name": "pineconeMetadataFilter",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "pineconeExistingIndex_0-input-pineconeMetadataFilter-json"
}
],
"inputAnchors": [
{
"label": "Embeddings",
"name": "embeddings",
"type": "Embeddings",
"id": "pineconeExistingIndex_0-input-embeddings-Embeddings"
}
],
"inputs": {
"embeddings": "{{openAIEmbeddings_1.data.instance}}",
"pineconeEnv": "northamerica-northeast1-gcp",
"pineconeIndex": "myindex",
"pineconeNamespace": "my-namespace",
"pineconeMetadataFilter": "{\"id\":\"doc1\"}"
},
"outputAnchors": [
{
"name": "output",
"label": "Output",
"type": "options",
"options": [
{
"id": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
"name": "retriever",
"label": "Pinecone Retriever",
"type": "Pinecone | VectorStoreRetriever | BaseRetriever"
},
{
"id": "pineconeExistingIndex_0-output-vectorStore-Pinecone|VectorStore",
"name": "vectorStore",
"label": "Pinecone Vector Store",
"type": "Pinecone | VectorStore"
}
],
"default": "retriever"
}
],
"outputs": {
"output": "retriever"
},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 1187.519066203033,
"y": 542.6635399602128
},
"dragging": false
}
],
"edges": [
{
"source": "openAI_1",
"sourceHandle": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain",
"target": "conversationalRetrievalQAChain_1",
"targetHandle": "conversationalRetrievalQAChain_1-input-llm-BaseLLM",
"type": "buttonedge",
"id": "openAI_1-openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-llm-BaseLLM",
"data": {
"label": ""
}
},
{
"source": "openAIEmbeddings_1",
"sourceHandle": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
"target": "pineconeExistingIndex_0",
"targetHandle": "pineconeExistingIndex_0-input-embeddings-Embeddings",
"type": "buttonedge",
"id": "openAIEmbeddings_1-openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-pineconeExistingIndex_0-pineconeExistingIndex_0-input-embeddings-Embeddings",
"data": {
"label": ""
}
},
{
"source": "pineconeExistingIndex_0",
"sourceHandle": "pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
"target": "conversationalRetrievalQAChain_1",
"targetHandle": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever",
"type": "buttonedge",
"id": "pineconeExistingIndex_0-pineconeExistingIndex_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever",
"data": {
"label": ""
}
}
]
}

662
packages/server/marketplaces/Metadata Filter Upsert.json Normal file
View File

@ -0,0 +1,662 @@
{
"description": "Upsert multiple files with metadata filters and feed into conversational retrieval QA chain",
"nodes": [
{
"width": 300,
"height": 376,
"id": "recursiveCharacterTextSplitter_1",
"position": {
"x": 347.5233039646277,
"y": 129.29305204134062
},
"type": "customNode",
"data": {
"id": "recursiveCharacterTextSplitter_1",
"label": "Recursive Character Text Splitter",
"name": "recursiveCharacterTextSplitter",
"type": "RecursiveCharacterTextSplitter",
"baseClasses": ["RecursiveCharacterTextSplitter", "TextSplitter"],
"category": "Text Splitters",
"description": "Split documents recursively by different characters - starting with \"\n\n\", then \"\n\", then \" \"",
"inputParams": [
{
"label": "Chunk Size",
"name": "chunkSize",
"type": "number",
"default": 1000,
"optional": true,
"id": "recursiveCharacterTextSplitter_1-input-chunkSize-number"
},
{
"label": "Chunk Overlap",
"name": "chunkOverlap",
"type": "number",
"optional": true,
"id": "recursiveCharacterTextSplitter_1-input-chunkOverlap-number"
}
],
"inputAnchors": [],
"inputs": {
"chunkSize": 1000,
"chunkOverlap": ""
},
"outputAnchors": [
{
"id": "recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter",
"name": "recursiveCharacterTextSplitter",
"label": "RecursiveCharacterTextSplitter",
"type": "RecursiveCharacterTextSplitter | TextSplitter"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 347.5233039646277,
"y": 129.29305204134062
},
"dragging": false
},
{
"width": 300,
"height": 523,
"id": "openAI_1",
"position": {
"x": 1159.184721109528,
"y": -38.76565405456694
},
"type": "customNode",
"data": {
"id": "openAI_1",
"label": "OpenAI",
"name": "openAI",
"type": "OpenAI",
"baseClasses": ["OpenAI", "BaseLLM", "BaseLanguageModel", "BaseLangChain"],
"category": "LLMs",
"description": "Wrapper around OpenAI large language models",
"inputParams": [
{
"label": "OpenAI Api Key",
"name": "openAIApiKey",
"type": "password",
"id": "openAI_1-input-openAIApiKey-password"
},
{
"label": "Model Name",
"name": "modelName",
"type": "options",
"options": [
{
"label": "text-davinci-003",
"name": "text-davinci-003"
},
{
"label": "text-davinci-002",
"name": "text-davinci-002"
},
{
"label": "text-curie-001",
"name": "text-curie-001"
},
{
"label": "text-babbage-001",
"name": "text-babbage-001"
}
],
"default": "text-davinci-003",
"optional": true,
"id": "openAI_1-input-modelName-options"
},
{
"label": "Temperature",
"name": "temperature",
"type": "number",
"default": 0.7,
"optional": true,
"id": "openAI_1-input-temperature-number"
},
{
"label": "Max Tokens",
"name": "maxTokens",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-maxTokens-number"
},
{
"label": "Top Probability",
"name": "topP",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-topP-number"
},
{
"label": "Best Of",
"name": "bestOf",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-bestOf-number"
},
{
"label": "Frequency Penalty",
"name": "frequencyPenalty",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-frequencyPenalty-number"
},
{
"label": "Presence Penalty",
"name": "presencePenalty",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-presencePenalty-number"
},
{
"label": "Batch Size",
"name": "batchSize",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-batchSize-number"
},
{
"label": "Timeout",
"name": "timeout",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAI_1-input-timeout-number"
}
],
"inputAnchors": [],
"inputs": {
"modelName": "text-davinci-003",
"temperature": "0",
"maxTokens": "",
"topP": "",
"bestOf": "",
"frequencyPenalty": "",
"presencePenalty": "",
"batchSize": "",
"timeout": ""
},
"outputAnchors": [
{
"id": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain",
"name": "openAI",
"label": "OpenAI",
"type": "OpenAI | BaseLLM | BaseLanguageModel | BaseLangChain"
}
],
"outputs": {},
"selected": false
},
"positionAbsolute": {
"x": 1159.184721109528,
"y": -38.76565405456694
},
"selected": false,
"dragging": false
},
{
"width": 300,
"height": 329,
"id": "openAIEmbeddings_1",
"position": {
"x": 749.4044250705479,
"y": 858.4858399327618
},
"type": "customNode",
"data": {
"id": "openAIEmbeddings_1",
"label": "OpenAI Embeddings",
"name": "openAIEmbeddings",
"type": "OpenAIEmbeddings",
"baseClasses": ["OpenAIEmbeddings", "Embeddings"],
"category": "Embeddings",
"description": "OpenAI API to generate embeddings for a given text",
"inputParams": [
{
"label": "OpenAI Api Key",
"name": "openAIApiKey",
"type": "password",
"id": "openAIEmbeddings_1-input-openAIApiKey-password"
},
{
"label": "Strip New Lines",
"name": "stripNewLines",
"type": "boolean",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_1-input-stripNewLines-boolean"
},
{
"label": "Batch Size",
"name": "batchSize",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_1-input-batchSize-number"
},
{
"label": "Timeout",
"name": "timeout",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_1-input-timeout-number"
}
],
"inputAnchors": [],
"inputs": {
"stripNewLines": "",
"batchSize": "",
"timeout": ""
},
"outputAnchors": [
{
"id": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
"name": "openAIEmbeddings",
"label": "OpenAIEmbeddings",
"type": "OpenAIEmbeddings | Embeddings"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 749.4044250705479,
"y": 858.4858399327618
},
"dragging": false
},
{
"width": 300,
"height": 279,
"id": "conversationalRetrievalQAChain_1",
"position": {
"x": 1551.758177447552,
"y": 367.847388647512
},
"type": "customNode",
"data": {
"id": "conversationalRetrievalQAChain_1",
"label": "Conversational Retrieval QA Chain",
"name": "conversationalRetrievalQAChain",
"type": "ConversationalRetrievalQAChain",
"baseClasses": ["ConversationalRetrievalQAChain", "BaseChain", "BaseLangChain"],
"category": "Chains",
"description": "Document QA - built on RetrievalQAChain to provide a chat history component",
"inputParams": [],
"inputAnchors": [
{
"label": "LLM",
"name": "llm",
"type": "BaseLLM",
"id": "conversationalRetrievalQAChain_1-input-llm-BaseLLM"
},
{
"label": "Vector Store Retriever",
"name": "vectorStoreRetriever",
"type": "BaseRetriever",
"id": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever"
}
],
"inputs": {
"llm": "{{openAI_1.data.instance}}",
"vectorStoreRetriever": "{{pineconeUpsert_0.data.instance}}"
},
"outputAnchors": [
{
"id": "conversationalRetrievalQAChain_1-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|BaseLangChain",
"name": "conversationalRetrievalQAChain",
"label": "ConversationalRetrievalQAChain",
"type": "ConversationalRetrievalQAChain | BaseChain | BaseLangChain"
}
],
"outputs": {},
"selected": false
},
"positionAbsolute": {
"x": 1551.758177447552,
"y": 367.847388647512
},
"selected": false,
"dragging": false
},
{
"width": 300,
"height": 410,
"id": "textFile_0",
"position": {
"x": 756.5586098635717,
"y": -121.81747478707992
},
"type": "customNode",
"data": {
"id": "textFile_0",
"label": "Text File",
"name": "textFile",
"type": "Document",
"baseClasses": ["Document"],
"category": "Document Loaders",
"description": "Load data from text files",
"inputParams": [
{
"label": "Txt File",
"name": "txtFile",
"type": "file",
"fileType": ".txt",
"id": "textFile_0-input-txtFile-file"
},
{
"label": "Metadata",
"name": "metadata",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "textFile_0-input-metadata-json"
}
],
"inputAnchors": [
{
"label": "Text Splitter",
"name": "textSplitter",
"type": "TextSplitter",
"optional": true,
"id": "textFile_0-input-textSplitter-TextSplitter"
}
],
"inputs": {
"textSplitter": "{{recursiveCharacterTextSplitter_1.data.instance}}",
"metadata": "{\"id\":\"doc1\"}"
},
"outputAnchors": [
{
"id": "textFile_0-output-textFile-Document",
"name": "textFile",
"label": "Document",
"type": "Document"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 756.5586098635717,
"y": -121.81747478707992
},
"dragging": false
},
{
"width": 300,
"height": 505,
"id": "pdfFile_0",
"position": {
"x": 752.0044222860163,
"y": 318.11704520478617
},
"type": "customNode",
"data": {
"id": "pdfFile_0",
"label": "Pdf File",
"name": "pdfFile",
"type": "Document",
"baseClasses": ["Document"],
"category": "Document Loaders",
"description": "Load data from PDF files",
"inputParams": [
{
"label": "Pdf File",
"name": "pdfFile",
"type": "file",
"fileType": ".pdf",
"id": "pdfFile_0-input-pdfFile-file"
},
{
"label": "Usage",
"name": "usage",
"type": "options",
"options": [
{
"label": "One document per page",
"name": "perPage"
},
{
"label": "One document per file",
"name": "perFile"
}
],
"default": "perPage",
"id": "pdfFile_0-input-usage-options"
},
{
"label": "Metadata",
"name": "metadata",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "pdfFile_0-input-metadata-json"
}
],
"inputAnchors": [
{
"label": "Text Splitter",
"name": "textSplitter",
"type": "TextSplitter",
"optional": true,
"id": "pdfFile_0-input-textSplitter-TextSplitter"
}
],
"inputs": {
"textSplitter": "{{recursiveCharacterTextSplitter_1.data.instance}}",
"usage": "perPage",
"metadata": "{\"id\":\"doc2\"}"
},
"outputAnchors": [
{
"id": "pdfFile_0-output-pdfFile-Document",
"name": "pdfFile",
"label": "Document",
"type": "Document"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 752.0044222860163,
"y": 318.11704520478617
},
"dragging": false
},
{
"width": 300,
"height": 701,
"id": "pineconeUpsert_0",
"position": {
"x": 1161.8813042660154,
"y": 537.0216614326227
},
"type": "customNode",
"data": {
"id": "pineconeUpsert_0",
"label": "Pinecone Upsert Document",
"name": "pineconeUpsert",
"type": "Pinecone",
"baseClasses": ["Pinecone", "VectorStoreRetriever", "BaseRetriever"],
"category": "Vector Stores",
"description": "Upsert documents to Pinecone",
"inputParams": [
{
"label": "Pinecone Api Key",
"name": "pineconeApiKey",
"type": "password",
"id": "pineconeUpsert_0-input-pineconeApiKey-password"
},
{
"label": "Pinecone Environment",
"name": "pineconeEnv",
"type": "string",
"id": "pineconeUpsert_0-input-pineconeEnv-string"
},
{
"label": "Pinecone Index",
"name": "pineconeIndex",
"type": "string",
"id": "pineconeUpsert_0-input-pineconeIndex-string"
},
{
"label": "Pinecone Namespace",
"name": "pineconeNamespace",
"type": "string",
"placeholder": "my-first-namespace",
"optional": true,
"id": "pineconeUpsert_0-input-pineconeNamespace-string"
}
],
"inputAnchors": [
{
"label": "Document",
"name": "document",
"type": "Document",
"list": true,
"id": "pineconeUpsert_0-input-document-Document"
},
{
"label": "Embeddings",
"name": "embeddings",
"type": "Embeddings",
"id": "pineconeUpsert_0-input-embeddings-Embeddings"
}
],
"inputs": {
"document": ["{{pdfFile_0.data.instance}}", "{{textFile_0.data.instance}}"],
"embeddings": "{{openAIEmbeddings_1.data.instance}}",
"pineconeEnv": "northamerica-northeast1-gcp",
"pineconeIndex": "myindex",
"pineconeNamespace": "my-namespace"
},
"outputAnchors": [
{
"name": "output",
"label": "Output",
"type": "options",
"options": [
{
"id": "pineconeUpsert_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
"name": "retriever",
"label": "Pinecone Retriever",
"type": "Pinecone | VectorStoreRetriever | BaseRetriever"
},
{
"id": "pineconeUpsert_0-output-vectorStore-Pinecone|VectorStore",
"name": "vectorStore",
"label": "Pinecone Vector Store",
"type": "Pinecone | VectorStore"
}
],
"default": "retriever"
}
],
"outputs": {
"output": "retriever"
},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 1161.8813042660154,
"y": 537.0216614326227
},
"dragging": false
}
],
"edges": [
{
"source": "openAI_1",
"sourceHandle": "openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain",
"target": "conversationalRetrievalQAChain_1",
"targetHandle": "conversationalRetrievalQAChain_1-input-llm-BaseLLM",
"type": "buttonedge",
"id": "openAI_1-openAI_1-output-openAI-OpenAI|BaseLLM|BaseLanguageModel|BaseLangChain-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-llm-BaseLLM",
"data": {
"label": ""
}
},
{
"source": "recursiveCharacterTextSplitter_1",
"sourceHandle": "recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter",
"target": "textFile_0",
"targetHandle": "textFile_0-input-textSplitter-TextSplitter",
"type": "buttonedge",
"id": "recursiveCharacterTextSplitter_1-recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter-textFile_0-textFile_0-input-textSplitter-TextSplitter",
"data": {
"label": ""
}
},
{
"source": "recursiveCharacterTextSplitter_1",
"sourceHandle": "recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter",
"target": "pdfFile_0",
"targetHandle": "pdfFile_0-input-textSplitter-TextSplitter",
"type": "buttonedge",
"id": "recursiveCharacterTextSplitter_1-recursiveCharacterTextSplitter_1-output-recursiveCharacterTextSplitter-RecursiveCharacterTextSplitter|TextSplitter-pdfFile_0-pdfFile_0-input-textSplitter-TextSplitter",
"data": {
"label": ""
}
},
{
"source": "openAIEmbeddings_1",
"sourceHandle": "openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
"target": "pineconeUpsert_0",
"targetHandle": "pineconeUpsert_0-input-embeddings-Embeddings",
"type": "buttonedge",
"id": "openAIEmbeddings_1-openAIEmbeddings_1-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-pineconeUpsert_0-pineconeUpsert_0-input-embeddings-Embeddings",
"data": {
"label": ""
}
},
{
"source": "pdfFile_0",
"sourceHandle": "pdfFile_0-output-pdfFile-Document",
"target": "pineconeUpsert_0",
"targetHandle": "pineconeUpsert_0-input-document-Document",
"type": "buttonedge",
"id": "pdfFile_0-pdfFile_0-output-pdfFile-Document-pineconeUpsert_0-pineconeUpsert_0-input-document-Document",
"data": {
"label": ""
}
},
{
"source": "textFile_0",
"sourceHandle": "textFile_0-output-textFile-Document",
"target": "pineconeUpsert_0",
"targetHandle": "pineconeUpsert_0-input-document-Document",
"type": "buttonedge",
"id": "textFile_0-textFile_0-output-textFile-Document-pineconeUpsert_0-pineconeUpsert_0-input-document-Document",
"data": {
"label": ""
}
},
{
"source": "pineconeUpsert_0",
"sourceHandle": "pineconeUpsert_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever",
"target": "conversationalRetrievalQAChain_1",
"targetHandle": "conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever",
"type": "buttonedge",
"id": "pineconeUpsert_0-pineconeUpsert_0-output-retriever-Pinecone|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_1-conversationalRetrievalQAChain_1-input-vectorStoreRetriever-BaseRetriever",
"data": {
"label": ""
}
}
]
}

8
packages/server/marketplaces/Multiple VectorDB.json
View File

@ -821,6 +821,14 @@
"placeholder": "my-first-namespace",
"optional": true,
"id": "pineconeExistingIndex_1-input-pineconeNamespace-string"
},
{
"label": "Pinecone Metadata Filter",
"name": "pineconeMetadataFilter",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "pineconeExistingIndex_1-input-pineconeMetadataFilter-json"
}
],
"inputAnchors": [

6
packages/server/src/utils/index.ts
View File

@ -404,8 +404,10 @@ export const isSameOverrideConfig = (
existingOverrideConfig?: ICommonObject,
newOverrideConfig?: ICommonObject
): boolean => {
// Skip check if its internal call
if (isInternal) return true
if (isInternal) {
if (existingOverrideConfig && Object.keys(existingOverrideConfig).length) return false
return true
}
// If existing and new overrideconfig are the same
if (
existingOverrideConfig &&

64
packages/ui/src/ui-component/json/JsonEditor.js Normal file
View File

@ -0,0 +1,64 @@
import { useState } from 'react'
import PropTypes from 'prop-types'
import { FormControl } from '@mui/material'
import ReactJson from 'react-json-view'
export const JsonEditorInput = ({ value, onChange, disabled = false, isDarkMode = false }) => {
const [myValue, setMyValue] = useState(value ? JSON.parse(value) : {})
const onClipboardCopy = (e) => {
const src = e.src
if (Array.isArray(src) || typeof src === 'object') {
navigator.clipboard.writeText(JSON.stringify(src, null, ' '))
} else {
navigator.clipboard.writeText(src)
}
}
return (
<>
<FormControl sx={{ mt: 1, width: '100%' }} size='small'>
{disabled && (
<ReactJson
theme={isDarkMode ? 'ocean' : 'rjv-default'}
style={{ padding: 10, borderRadius: 10 }}
src={myValue}
name={null}
enableClipboard={(e) => onClipboardCopy(e)}
quotesOnKeys={false}
displayDataTypes={false}
/>
)}
{!disabled && (
<ReactJson
theme={isDarkMode ? 'ocean' : 'rjv-default'}
style={{ padding: 10, borderRadius: 10 }}
src={myValue}
name={null}
quotesOnKeys={false}
displayDataTypes={false}
enableClipboard={(e) => onClipboardCopy(e)}
onEdit={(edit) => {
setMyValue(edit.updated_src)
onChange(JSON.stringify(edit.updated_src))
}}
onAdd={() => {
//console.log(add)
}}
onDelete={(deleteobj) => {
setMyValue(deleteobj.updated_src)
onChange(JSON.stringify(deleteobj.updated_src))
}}
/>
)}
</FormControl>
</>
)
}
JsonEditorInput.propTypes = {
value: PropTypes.string,
onChange: PropTypes.func,
disabled: PropTypes.bool,
isDarkMode: PropTypes.bool
}

11
packages/ui/src/views/canvas/NodeInputHandler.js
View File

@ -1,6 +1,7 @@
import PropTypes from 'prop-types'
import { Handle, Position, useUpdateNodeInternals } from 'reactflow'
import { useEffect, useRef, useState, useContext } from 'react'
import { useSelector } from 'react-redux'
// material-ui
import { useTheme, styled } from '@mui/material/styles'
@ -15,6 +16,7 @@ import { File } from 'ui-component/file/File'
import { SwitchInput } from 'ui-component/switch/Switch'
import { flowContext } from 'store/context/ReactFlowContext'
import { isValidConnection, getAvailableNodesForVariable } from 'utils/genericHelper'
import { JsonEditorInput } from 'ui-component/json/JsonEditor'
const CustomWidthTooltip = styled(({ className, ...props }) => <Tooltip {...props} classes={{ popper: className }} />)({
[`& .${tooltipClasses.tooltip}`]: {
@ -26,6 +28,7 @@ const CustomWidthTooltip = styled(({ className, ...props }) => <Tooltip {...prop
const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isAdditionalParams = false }) => {
const theme = useTheme()
const customization = useSelector((state) => state.customization)
const ref = useRef(null)
const { reactFlowInstance } = useContext(flowContext)
const updateNodeInternals = useUpdateNodeInternals()
@ -166,6 +169,14 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA
onDialogConfirm={(newValue, inputParamName) => onExpandDialogSave(newValue, inputParamName)}
/>
)}
{inputParam.type === 'json' && (
<JsonEditorInput
disabled={disabled}
onChange={(newValue) => (data.inputs[inputParam.name] = newValue)}
value={data.inputs[inputParam.name] ?? inputParam.default ?? ''}
isDarkMode={customization.isDarkMode}
/>
)}
{inputParam.type === 'options' && (
<Dropdown
disabled={disabled}