Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature/Ability to omit all metadata keys using asterisk (#2401) 

add ability to omit all metadata keys using asterisk
This commit is contained in:
Henry Heng 2024-05-13 16:30:57 +01:00 committed by GitHub
parent 4fbc3f6cfe
commit b50103021c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 578 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
packages/components/nodes/documentloaders/API/APILoader.ts
View File

@ -82,7 +82,7 @@ class API_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -132,7 +132,12 @@ class API_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -143,7 +148,10 @@ class API_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Airtable/Airtable.ts
View File

@ -107,7 +107,7 @@ class Airtable_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -162,7 +162,12 @@ class Airtable_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -173,7 +178,10 @@ class Airtable_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts
View File

@ -106,7 +106,7 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -174,7 +174,12 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -185,7 +190,10 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
View File

@ -93,7 +93,7 @@ class Cheerio_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -178,7 +178,12 @@ class Cheerio_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -189,7 +194,10 @@ class Cheerio_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Confluence/Confluence.ts
View File

@ -73,7 +73,7 @@ class Confluence_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -128,7 +128,12 @@ class Confluence_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -139,7 +144,10 @@ class Confluence_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Csv/Csv.ts
View File

@ -59,7 +59,7 @@ class Csv_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -128,7 +128,12 @@ class Csv_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -139,7 +144,10 @@ class Csv_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Docx/Docx.ts
View File

@ -51,7 +51,7 @@ class Docx_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -119,7 +119,12 @@ class Docx_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -130,7 +135,10 @@ class Docx_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Figma/Figma.ts
View File

@ -74,7 +74,7 @@ class Figma_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -111,7 +111,12 @@ class Figma_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -122,7 +127,10 @@ class Figma_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Folder/Folder.ts
View File

@ -79,7 +79,7 @@ class Folder_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -162,7 +162,12 @@ class Folder_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -173,7 +178,10 @@ class Folder_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Gitbook/Gitbook.ts
View File

@ -58,7 +58,7 @@ class Gitbook_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -85,7 +85,12 @@ class Gitbook_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -96,7 +101,10 @@ class Gitbook_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Github/Github.ts
View File

@ -100,7 +100,7 @@ class Github_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -145,7 +145,12 @@ class Github_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -156,7 +161,10 @@ class Github_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Json/Json.ts
View File

@ -59,7 +59,7 @@ class Json_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -135,7 +135,12 @@ class Json_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -146,7 +151,10 @@ class Json_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts
View File

@ -58,7 +58,7 @@ class Jsonlines_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -129,7 +129,12 @@ class Jsonlines_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -140,7 +145,10 @@ class Jsonlines_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Notion/NotionDB.ts
View File

@ -58,7 +58,7 @@ class NotionDB_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -104,7 +104,12 @@ class NotionDB_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -115,7 +120,10 @@ class NotionDB_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Notion/NotionFolder.ts
View File

@ -51,7 +51,7 @@ class NotionFolder_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -83,7 +83,12 @@ class NotionFolder_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -94,7 +99,10 @@ class NotionFolder_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Notion/NotionPage.ts
View File

@ -59,7 +59,7 @@ class NotionPage_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -101,7 +101,12 @@ class NotionPage_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -112,7 +117,10 @@ class NotionPage_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Pdf/Pdf.ts
View File

@ -74,7 +74,7 @@ class Pdf_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -132,7 +132,12 @@ class Pdf_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -143,7 +148,10 @@ class Pdf_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/PlainText/PlainText.ts
View File

@ -54,7 +54,7 @@ class PlainText_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -104,7 +104,12 @@ class PlainText_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -115,7 +120,10 @@ class PlainText_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Playwright/Playwright.ts
View File

@ -121,7 +121,7 @@ class Playwright_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -217,7 +217,12 @@ class Playwright_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -228,7 +233,10 @@ class Playwright_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
View File

@ -122,7 +122,7 @@ class Puppeteer_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -218,7 +218,12 @@ class Puppeteer_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -229,7 +234,10 @@ class Puppeteer_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/S3File/S3File.ts
View File

@ -427,7 +427,7 @@ class S3_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -561,7 +561,12 @@ class S3_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata, ...parsedMetadata,
@ -573,7 +578,10 @@ class S3_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata, ...doc.metadata,
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey

14
packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts
View File

@ -68,7 +68,7 @@ class SearchAPI_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -112,7 +112,12 @@ class SearchAPI_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -123,7 +128,10 @@ class SearchAPI_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts
View File

@ -58,7 +58,7 @@ class SerpAPI_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -86,7 +86,12 @@ class SerpAPI_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -97,7 +102,10 @@ class SerpAPI_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Text/Text.ts
View File

@ -53,7 +53,7 @@ class Text_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -137,7 +137,12 @@ class Text_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata ...parsedMetadata
@ -148,7 +153,10 @@ class Text_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata ...doc.metadata
}, },

14
packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts
View File

@ -414,7 +414,7 @@ class UnstructuredFile_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -518,7 +518,12 @@ class UnstructuredFile_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata, ...parsedMetadata,
@ -530,7 +535,10 @@ class UnstructuredFile_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata, ...doc.metadata,
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey

14
packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts
View File

@ -393,7 +393,7 @@ class UnstructuredFolder_DocumentLoaders implements INode {
type: 'string', type: 'string',
rows: 4, rows: 4,
description: description:
'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field',
placeholder: 'key1, key2, key3.nestedKey1', placeholder: 'key1, key2, key3.nestedKey1',
optional: true, optional: true,
additionalParams: true additionalParams: true
@ -456,7 +456,12 @@ class UnstructuredFolder_DocumentLoaders implements INode {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {
...parsedMetadata
}
: omit(
{ {
...doc.metadata, ...doc.metadata,
...parsedMetadata, ...parsedMetadata,
@ -468,7 +473,10 @@ class UnstructuredFolder_DocumentLoaders implements INode {
} else { } else {
docs = docs.map((doc) => ({ docs = docs.map((doc) => ({
...doc, ...doc,
metadata: omit( metadata:
_omitMetadataKeys === '*'
? {}
: omit(
{ {
...doc.metadata, ...doc.metadata,
[sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey