Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature/Ability to omit all metadata keys using asterisk (#2401) 

add ability to omit all metadata keys using asterisk
This commit is contained in:
Henry Heng 2024-05-13 16:30:57 +01:00 committed by GitHub
parent 4fbc3f6cfe
commit b50103021c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 578 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
packages/components/nodes/documentloaders/API/APILoader.ts
View File

@ -82,7 +82,7 @@ class API_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -132,23 +132,31 @@ class API_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Airtable/Airtable.ts
View File

@ -107,7 +107,7 @@ class Airtable_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -162,23 +162,31 @@ class Airtable_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts
View File

@ -106,7 +106,7 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -174,23 +174,31 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
View File

@ -93,7 +93,7 @@ class Cheerio_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -178,23 +178,31 @@ class Cheerio_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Confluence/Confluence.ts
View File

@ -73,7 +73,7 @@ class Confluence_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -128,23 +128,31 @@ class Confluence_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Csv/Csv.ts
View File

@ -59,7 +59,7 @@ class Csv_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -128,23 +128,31 @@ class Csv_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Docx/Docx.ts
View File

@ -51,7 +51,7 @@ class Docx_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -119,23 +119,31 @@ class Docx_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Figma/Figma.ts
View File

@ -74,7 +74,7 @@ class Figma_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -111,23 +111,31 @@ class Figma_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Folder/Folder.ts
View File

@ -79,7 +79,7 @@ class Folder_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -162,23 +162,31 @@ class Folder_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Gitbook/Gitbook.ts
View File

@ -58,7 +58,7 @@ class Gitbook_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -85,23 +85,31 @@ class Gitbook_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Github/Github.ts
View File

@ -100,7 +100,7 @@ class Github_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -145,23 +145,31 @@ class Github_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Json/Json.ts
View File

@ -59,7 +59,7 @@ class Json_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -135,23 +135,31 @@ class Json_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts
View File

@ -58,7 +58,7 @@ class Jsonlines_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -129,23 +129,31 @@ class Jsonlines_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Notion/NotionDB.ts
View File

@ -58,7 +58,7 @@ class NotionDB_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -104,23 +104,31 @@ class NotionDB_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Notion/NotionFolder.ts
View File

@ -51,7 +51,7 @@ class NotionFolder_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -83,23 +83,31 @@ class NotionFolder_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Notion/NotionPage.ts
View File

@ -59,7 +59,7 @@ class NotionPage_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -101,23 +101,31 @@ class NotionPage_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Pdf/Pdf.ts
View File

@ -74,7 +74,7 @@ class Pdf_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -132,23 +132,31 @@ class Pdf_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/PlainText/PlainText.ts
View File

@ -54,7 +54,7 @@ class PlainText_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -104,23 +104,31 @@ class PlainText_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Playwright/Playwright.ts
View File

@ -121,7 +121,7 @@ class Playwright_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -217,23 +217,31 @@ class Playwright_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
View File

@ -122,7 +122,7 @@ class Puppeteer_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -218,23 +218,31 @@ class Puppeteer_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

40
packages/components/nodes/documentloaders/S3File/S3File.ts
View File

@ -427,7 +427,7 @@ class S3_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -561,25 +561,33 @@ class S3_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata, ...parsedMetadata
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey }
}, : omit(
omitMetadataKeys {
) ...doc.metadata,
...parsedMetadata,
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {}
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey : omit(
}, {
omitMetadataKeys ...doc.metadata,
) [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts
View File

@ -68,7 +68,7 @@ class SearchAPI_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -112,23 +112,31 @@ class SearchAPI_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts
View File

@ -58,7 +58,7 @@ class SerpAPI_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -86,23 +86,31 @@ class SerpAPI_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

36
packages/components/nodes/documentloaders/Text/Text.ts
View File

@ -53,7 +53,7 @@ class Text_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -137,23 +137,31 @@ class Text_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata ...parsedMetadata
}, }
omitMetadataKeys : omit(
) {
...doc.metadata,
...parsedMetadata
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata ? {}
}, : omit(
omitMetadataKeys {
) ...doc.metadata
},
omitMetadataKeys
)
})) }))
} }

40
packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts
View File

@ -414,7 +414,7 @@ class UnstructuredFile_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -518,25 +518,33 @@ class UnstructuredFile_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata, ...parsedMetadata
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey }
}, : omit(
omitMetadataKeys {
) ...doc.metadata,
...parsedMetadata,
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {}
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey : omit(
}, {
omitMetadataKeys ...doc.metadata,
) [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
},
omitMetadataKeys
)
})) }))
} }

40
packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts
View File

@ -393,7 +393,7 @@ class UnstructuredFolder_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -456,25 +456,33 @@ class UnstructuredFolder_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {
...parsedMetadata, ...parsedMetadata
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey }
}, : omit(
omitMetadataKeys {
) ...doc.metadata,
...parsedMetadata,
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
},
omitMetadataKeys
)
})) }))
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
{ _omitMetadataKeys === '*'
...doc.metadata, ? {}
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey : omit(
}, {
omitMetadataKeys ...doc.metadata,
) [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey
},
omitMetadataKeys
)
})) }))
} }