import { omit } from 'lodash' import { ICommonObject, IDocument, INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { NotionAPILoader, NotionAPILoaderOptions } from 'langchain/document_loaders/web/notionapi' import { getCredentialData, getCredentialParam } from '../../../src' class NotionPage_DocumentLoaders implements INode { label: string name: string version: number description: string type: string icon: string category: string baseClasses: string[] credential: INodeParams inputs: INodeParams[] constructor() { this.label = 'Notion Page' this.name = 'notionPage' this.version = 1.0 this.type = 'Document' this.icon = 'notion-page.svg' this.category = 'Document Loaders' this.description = 'Load data from Notion Page (including child pages all as separate documents)' this.baseClasses = [this.type] this.credential = { label: 'Connect Credential', name: 'credential', type: 'credential', credentialNames: ['notionApi'] } this.inputs = [ { label: 'Text Splitter', name: 'textSplitter', type: 'TextSplitter', optional: true }, { label: 'Notion Page Id', name: 'pageId', type: 'string', description: 'The last The 32 char hex in the url path. For example: https://www.notion.so/skarard/LangChain-Notion-API-b34ca03f219c4420a6046fc4bdfdf7b4, b34ca03f219c4420a6046fc4bdfdf7b4 is the Page ID' }, { label: 'Additional Metadata', name: 'metadata', type: 'json', description: 'Additional metadata to be added to the extracted documents', optional: true, additionalParams: true }, { label: 'Omit Metadata Keys', name: 'omitMetadataKeys', type: 'string', rows: 4, description: 'Each document loader comes with a default set of metadata keys that are extracted from the document. You can use this field to omit some of the default metadata keys. The value should be a list of keys, seperated by comma', placeholder: 'key1, key2, key3.nestedKey1', optional: true, additionalParams: true } ] } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const pageId = nodeData.inputs?.pageId as string const metadata = nodeData.inputs?.metadata const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string let omitMetadataKeys: string[] = [] if (_omitMetadataKeys) { omitMetadataKeys = _omitMetadataKeys.split(',').map((key) => key.trim()) } const credentialData = await getCredentialData(nodeData.credential ?? '', options) const notionIntegrationToken = getCredentialParam('notionIntegrationToken', credentialData, nodeData) const obj: NotionAPILoaderOptions = { clientOptions: { auth: notionIntegrationToken }, id: pageId, type: 'page' } const loader = new NotionAPILoader(obj) let docs: IDocument[] = [] if (textSplitter) { docs = await loader.loadAndSplit(textSplitter) } else { docs = await loader.load() } if (metadata) { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) docs = docs.map((doc) => ({ ...doc, metadata: omit( { ...doc.metadata, ...parsedMetadata }, omitMetadataKeys ) })) } else { docs = docs.map((doc) => ({ ...doc, metadata: omit( { ...doc.metadata }, omitMetadataKeys ) })) } return docs } } module.exports = { nodeClass: NotionPage_DocumentLoaders }