diff --git a/packages/components/nodes/textsplitters/MarkdownTextSplitter/MarkdownTextSplitter.ts b/packages/components/nodes/textsplitters/MarkdownTextSplitter/MarkdownTextSplitter.ts new file mode 100644 index 000000000..02c37d8d5 --- /dev/null +++ b/packages/components/nodes/textsplitters/MarkdownTextSplitter/MarkdownTextSplitter.ts @@ -0,0 +1,55 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { MarkdownTextSplitter, MarkdownTextSplitterParams } from 'langchain/text_splitter' + +class MarkdownTextSplitter_TextSplitters implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Markdown Text Splitter' + this.name = 'markdownTextSplitter' + this.type = 'MarkdownTextSplitter' + this.icon = 'markdownTextSplitter.svg' + this.category = 'Text Splitters' + this.description = `Split your content into documents based on the Markdown headers` + this.baseClasses = [this.type, ...getBaseClasses(MarkdownTextSplitter)] + this.inputs = [ + { + label: 'Chunk Size', + name: 'chunkSize', + type: 'number', + default: 1000, + optional: true + }, + { + label: 'Chunk Overlap', + name: 'chunkOverlap', + type: 'number', + optional: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const chunkSize = nodeData.inputs?.chunkSize as string + const chunkOverlap = nodeData.inputs?.chunkOverlap as string + + const obj = {} as MarkdownTextSplitterParams + + if (chunkSize) obj.chunkSize = parseInt(chunkSize, 10) + if (chunkOverlap) obj.chunkOverlap = parseInt(chunkOverlap, 10) + + const splitter = new MarkdownTextSplitter(obj) + + return splitter + } +} + +module.exports = { nodeClass: MarkdownTextSplitter_TextSplitters } diff --git a/packages/components/nodes/textsplitters/MarkdownTextSplitter/markdownTextSplitter.svg b/packages/components/nodes/textsplitters/MarkdownTextSplitter/markdownTextSplitter.svg new file mode 100644 index 000000000..f7d45d603 --- /dev/null +++ b/packages/components/nodes/textsplitters/MarkdownTextSplitter/markdownTextSplitter.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file