diff --git a/packages/components/nodes/documentloaders/API/APILoader.ts b/packages/components/nodes/documentloaders/API/APILoader.ts index dda9cf440..0035e5956 100644 --- a/packages/components/nodes/documentloaders/API/APILoader.ts +++ b/packages/components/nodes/documentloaders/API/APILoader.ts @@ -82,7 +82,7 @@ class API_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -132,23 +132,31 @@ class API_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts index 9c475bf6d..df98639bf 100644 --- a/packages/components/nodes/documentloaders/Airtable/Airtable.ts +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -107,7 +107,7 @@ class Airtable_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -162,23 +162,31 @@ class Airtable_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts b/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts index 207a74f6e..63bcea570 100644 --- a/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts +++ b/packages/components/nodes/documentloaders/ApifyWebsiteContentCrawler/ApifyWebsiteContentCrawler.ts @@ -106,7 +106,7 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -174,23 +174,31 @@ class ApifyWebsiteContentCrawler_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts index 272e226f1..966845b6e 100644 --- a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -93,7 +93,7 @@ class Cheerio_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -178,23 +178,31 @@ class Cheerio_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Confluence/Confluence.ts b/packages/components/nodes/documentloaders/Confluence/Confluence.ts index b2496f675..c77340b8d 100644 --- a/packages/components/nodes/documentloaders/Confluence/Confluence.ts +++ b/packages/components/nodes/documentloaders/Confluence/Confluence.ts @@ -73,7 +73,7 @@ class Confluence_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -128,23 +128,31 @@ class Confluence_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts index ec0fe683a..c9322aba8 100644 --- a/packages/components/nodes/documentloaders/Csv/Csv.ts +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -59,7 +59,7 @@ class Csv_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -128,23 +128,31 @@ class Csv_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Docx/Docx.ts b/packages/components/nodes/documentloaders/Docx/Docx.ts index 3ee6ef34e..e473ae617 100644 --- a/packages/components/nodes/documentloaders/Docx/Docx.ts +++ b/packages/components/nodes/documentloaders/Docx/Docx.ts @@ -51,7 +51,7 @@ class Docx_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -119,23 +119,31 @@ class Docx_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Figma/Figma.ts b/packages/components/nodes/documentloaders/Figma/Figma.ts index e7f88b52d..023168aa7 100644 --- a/packages/components/nodes/documentloaders/Figma/Figma.ts +++ b/packages/components/nodes/documentloaders/Figma/Figma.ts @@ -74,7 +74,7 @@ class Figma_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -111,23 +111,31 @@ class Figma_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Folder/Folder.ts b/packages/components/nodes/documentloaders/Folder/Folder.ts index e3392a77b..15cb10a40 100644 --- a/packages/components/nodes/documentloaders/Folder/Folder.ts +++ b/packages/components/nodes/documentloaders/Folder/Folder.ts @@ -79,7 +79,7 @@ class Folder_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -162,23 +162,31 @@ class Folder_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Gitbook/Gitbook.ts b/packages/components/nodes/documentloaders/Gitbook/Gitbook.ts index 6faba9ef3..e5c524a4e 100644 --- a/packages/components/nodes/documentloaders/Gitbook/Gitbook.ts +++ b/packages/components/nodes/documentloaders/Gitbook/Gitbook.ts @@ -58,7 +58,7 @@ class Gitbook_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -85,23 +85,31 @@ class Gitbook_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Github/Github.ts b/packages/components/nodes/documentloaders/Github/Github.ts index 5b585f2c7..0df11444d 100644 --- a/packages/components/nodes/documentloaders/Github/Github.ts +++ b/packages/components/nodes/documentloaders/Github/Github.ts @@ -100,7 +100,7 @@ class Github_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -145,23 +145,31 @@ class Github_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Json/Json.ts b/packages/components/nodes/documentloaders/Json/Json.ts index 64ef84dd3..7b12fb3a7 100644 --- a/packages/components/nodes/documentloaders/Json/Json.ts +++ b/packages/components/nodes/documentloaders/Json/Json.ts @@ -59,7 +59,7 @@ class Json_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -135,23 +135,31 @@ class Json_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts index 82b11d132..48654c4c0 100644 --- a/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts +++ b/packages/components/nodes/documentloaders/Jsonlines/Jsonlines.ts @@ -58,7 +58,7 @@ class Jsonlines_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -129,23 +129,31 @@ class Jsonlines_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Notion/NotionDB.ts b/packages/components/nodes/documentloaders/Notion/NotionDB.ts index dd21cc8e3..38e48ba48 100644 --- a/packages/components/nodes/documentloaders/Notion/NotionDB.ts +++ b/packages/components/nodes/documentloaders/Notion/NotionDB.ts @@ -58,7 +58,7 @@ class NotionDB_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -104,23 +104,31 @@ class NotionDB_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Notion/NotionFolder.ts b/packages/components/nodes/documentloaders/Notion/NotionFolder.ts index 22184b493..4e473891a 100644 --- a/packages/components/nodes/documentloaders/Notion/NotionFolder.ts +++ b/packages/components/nodes/documentloaders/Notion/NotionFolder.ts @@ -51,7 +51,7 @@ class NotionFolder_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -83,23 +83,31 @@ class NotionFolder_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Notion/NotionPage.ts b/packages/components/nodes/documentloaders/Notion/NotionPage.ts index 3e88a6e35..29e873524 100644 --- a/packages/components/nodes/documentloaders/Notion/NotionPage.ts +++ b/packages/components/nodes/documentloaders/Notion/NotionPage.ts @@ -59,7 +59,7 @@ class NotionPage_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -101,23 +101,31 @@ class NotionPage_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Pdf/Pdf.ts b/packages/components/nodes/documentloaders/Pdf/Pdf.ts index 9199c9bef..13cdf87c8 100644 --- a/packages/components/nodes/documentloaders/Pdf/Pdf.ts +++ b/packages/components/nodes/documentloaders/Pdf/Pdf.ts @@ -74,7 +74,7 @@ class Pdf_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -132,23 +132,31 @@ class Pdf_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/PlainText/PlainText.ts b/packages/components/nodes/documentloaders/PlainText/PlainText.ts index 2d3d89602..03504d7b8 100644 --- a/packages/components/nodes/documentloaders/PlainText/PlainText.ts +++ b/packages/components/nodes/documentloaders/PlainText/PlainText.ts @@ -54,7 +54,7 @@ class PlainText_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -104,23 +104,31 @@ class PlainText_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Playwright/Playwright.ts b/packages/components/nodes/documentloaders/Playwright/Playwright.ts index 4472e6bb3..281fb5424 100644 --- a/packages/components/nodes/documentloaders/Playwright/Playwright.ts +++ b/packages/components/nodes/documentloaders/Playwright/Playwright.ts @@ -121,7 +121,7 @@ class Playwright_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -217,23 +217,31 @@ class Playwright_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts index baca4abe4..b9a80472f 100644 --- a/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts +++ b/packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts @@ -122,7 +122,7 @@ class Puppeteer_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -218,23 +218,31 @@ class Puppeteer_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/S3File/S3File.ts b/packages/components/nodes/documentloaders/S3File/S3File.ts index c01500e1f..fcb438ef0 100644 --- a/packages/components/nodes/documentloaders/S3File/S3File.ts +++ b/packages/components/nodes/documentloaders/S3File/S3File.ts @@ -427,7 +427,7 @@ class S3_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -561,25 +561,33 @@ class S3_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata, - [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata, + [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata, + [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts b/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts index 1a2db0e27..e71307987 100644 --- a/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts +++ b/packages/components/nodes/documentloaders/SearchApi/SearchAPI.ts @@ -68,7 +68,7 @@ class SearchAPI_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -112,23 +112,31 @@ class SearchAPI_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts b/packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts index db735a961..ad536054d 100644 --- a/packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts +++ b/packages/components/nodes/documentloaders/SerpApi/SerpAPI.ts @@ -58,7 +58,7 @@ class SerpAPI_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -86,23 +86,31 @@ class SerpAPI_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Text/Text.ts b/packages/components/nodes/documentloaders/Text/Text.ts index 6589fb75e..c974cae2e 100644 --- a/packages/components/nodes/documentloaders/Text/Text.ts +++ b/packages/components/nodes/documentloaders/Text/Text.ts @@ -53,7 +53,7 @@ class Text_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -137,23 +137,31 @@ class Text_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts index d80f2e5b1..f1cf6c457 100644 --- a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts +++ b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts @@ -414,7 +414,7 @@ class UnstructuredFile_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -518,25 +518,33 @@ class UnstructuredFile_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata, - [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata, + [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata, + [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey + }, + omitMetadataKeys + ) })) } diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts index 83d7c85a8..f04bee358 100644 --- a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts +++ b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts @@ -393,7 +393,7 @@ class UnstructuredFolder_DocumentLoaders implements INode { type: 'string', rows: 4, description: - 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', + 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma. Use * to omit all metadata keys execept the ones you specify in the Additional Metadata field', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true @@ -456,25 +456,33 @@ class UnstructuredFolder_DocumentLoaders implements INode { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - ...parsedMetadata, - [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? { + ...parsedMetadata + } + : omit( + { + ...doc.metadata, + ...parsedMetadata, + [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey + }, + omitMetadataKeys + ) })) } else { docs = docs.map((doc) => ({ ...doc, - metadata: omit( - { - ...doc.metadata, - [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey - }, - omitMetadataKeys - ) + metadata: + _omitMetadataKeys === '*' + ? {} + : omit( + { + ...doc.metadata, + [sourceIdKey]: doc.metadata[sourceIdKey] || sourceIdKey + }, + omitMetadataKeys + ) })) }