diff --git a/packages/components/nodes/documentloaders/API/APILoader.ts b/packages/components/nodes/documentloaders/API/APILoader.ts new file mode 100644 index 000000000..30fa31d7a --- /dev/null +++ b/packages/components/nodes/documentloaders/API/APILoader.ts @@ -0,0 +1,198 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { Document } from 'langchain/document' +import axios, { AxiosRequestConfig } from 'axios' + +class API_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs?: INodeParams[] + + constructor() { + this.label = 'API Loader' + this.name = 'apiLoader' + this.type = 'Document' + this.icon = 'api-loader.png' + this.category = 'Document Loaders' + this.description = `Load data from an API` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Method', + name: 'method', + type: 'options', + options: [ + { + label: 'GET', + name: 'GET' + }, + { + label: 'POST', + name: 'POST' + } + ] + }, + { + label: 'URL', + name: 'url', + type: 'string' + }, + { + label: 'Headers', + name: 'headers', + type: 'json', + additionalParams: true, + optional: true + }, + { + label: 'Body', + name: 'body', + type: 'json', + description: + 'JSON body for the POST request. If not specified, agent will try to figure out itself from AIPlugin if provided', + additionalParams: true, + optional: true + } + ] + } + async init(nodeData: INodeData): Promise { + const headers = nodeData.inputs?.headers as string + const url = nodeData.inputs?.url as string + const body = nodeData.inputs?.body as string + const method = nodeData.inputs?.method as string + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata + + const options: ApiLoaderParams = { + url, + method + } + + if (headers) { + const parsedHeaders = typeof headers === 'object' ? headers : JSON.parse(headers) + options.headers = parsedHeaders + } + + if (body) { + const parsedBody = typeof body === 'object' ? body : JSON.parse(body) + options.body = parsedBody + } + + const loader = new ApiLoader(options) + + let docs = [] + + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs + } +} + +interface ApiLoaderParams { + url: string + method: string + headers?: ICommonObject + body?: ICommonObject +} + +class ApiLoader extends BaseDocumentLoader { + public readonly url: string + + public readonly headers?: ICommonObject + + public readonly body?: ICommonObject + + public readonly method: string + + constructor({ url, headers, body, method }: ApiLoaderParams) { + super() + this.url = url + this.headers = headers + this.body = body + this.method = method + } + + public async load(): Promise { + if (this.method === 'POST') { + return this.executePostRequest(this.url, this.headers, this.body) + } else { + return this.executeGetRequest(this.url, this.headers) + } + } + + protected async executeGetRequest(url: string, headers?: ICommonObject): Promise { + try { + const config: AxiosRequestConfig = {} + if (headers) { + config.headers = headers + } + const response = await axios.get(url, config) + const responseJsonString = JSON.stringify(response.data, null, 2) + const doc = new Document({ + pageContent: responseJsonString, + metadata: { + url + } + }) + return [doc] + } catch (error) { + throw new Error(`Failed to fetch ${url}: ${error}`) + } + } + + protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject): Promise { + try { + const config: AxiosRequestConfig = {} + if (headers) { + config.headers = headers + } + const response = await axios.post(url, body ?? {}, config) + const responseJsonString = JSON.stringify(response.data, null, 2) + const doc = new Document({ + pageContent: responseJsonString, + metadata: { + url + } + }) + return [doc] + } catch (error) { + throw new Error(`Failed to post ${url}: ${error}`) + } + } +} + +module.exports = { + nodeClass: API_DocumentLoaders +} diff --git a/packages/components/nodes/documentloaders/API/api-loader.png b/packages/components/nodes/documentloaders/API/api-loader.png new file mode 100644 index 000000000..93668c4cf Binary files /dev/null and b/packages/components/nodes/documentloaders/API/api-loader.png differ