From c68c5be36335f6987ae741b342e36acf37b7ef6a Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 21 Jul 2023 19:22:54 +0100 Subject: [PATCH] add API document loader --- .../nodes/documentloaders/API/APILoader.ts | 198 ++++++++++++++++++ .../nodes/documentloaders/API/api-loader.png | Bin 0 -> 1402 bytes 2 files changed, 198 insertions(+) create mode 100644 packages/components/nodes/documentloaders/API/APILoader.ts create mode 100644 packages/components/nodes/documentloaders/API/api-loader.png diff --git a/packages/components/nodes/documentloaders/API/APILoader.ts b/packages/components/nodes/documentloaders/API/APILoader.ts new file mode 100644 index 000000000..30fa31d7a --- /dev/null +++ b/packages/components/nodes/documentloaders/API/APILoader.ts @@ -0,0 +1,198 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { Document } from 'langchain/document' +import axios, { AxiosRequestConfig } from 'axios' + +class API_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs?: INodeParams[] + + constructor() { + this.label = 'API Loader' + this.name = 'apiLoader' + this.type = 'Document' + this.icon = 'api-loader.png' + this.category = 'Document Loaders' + this.description = `Load data from an API` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Method', + name: 'method', + type: 'options', + options: [ + { + label: 'GET', + name: 'GET' + }, + { + label: 'POST', + name: 'POST' + } + ] + }, + { + label: 'URL', + name: 'url', + type: 'string' + }, + { + label: 'Headers', + name: 'headers', + type: 'json', + additionalParams: true, + optional: true + }, + { + label: 'Body', + name: 'body', + type: 'json', + description: + 'JSON body for the POST request. If not specified, agent will try to figure out itself from AIPlugin if provided', + additionalParams: true, + optional: true + } + ] + } + async init(nodeData: INodeData): Promise { + const headers = nodeData.inputs?.headers as string + const url = nodeData.inputs?.url as string + const body = nodeData.inputs?.body as string + const method = nodeData.inputs?.method as string + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata + + const options: ApiLoaderParams = { + url, + method + } + + if (headers) { + const parsedHeaders = typeof headers === 'object' ? headers : JSON.parse(headers) + options.headers = parsedHeaders + } + + if (body) { + const parsedBody = typeof body === 'object' ? body : JSON.parse(body) + options.body = parsedBody + } + + const loader = new ApiLoader(options) + + let docs = [] + + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs + } +} + +interface ApiLoaderParams { + url: string + method: string + headers?: ICommonObject + body?: ICommonObject +} + +class ApiLoader extends BaseDocumentLoader { + public readonly url: string + + public readonly headers?: ICommonObject + + public readonly body?: ICommonObject + + public readonly method: string + + constructor({ url, headers, body, method }: ApiLoaderParams) { + super() + this.url = url + this.headers = headers + this.body = body + this.method = method + } + + public async load(): Promise { + if (this.method === 'POST') { + return this.executePostRequest(this.url, this.headers, this.body) + } else { + return this.executeGetRequest(this.url, this.headers) + } + } + + protected async executeGetRequest(url: string, headers?: ICommonObject): Promise { + try { + const config: AxiosRequestConfig = {} + if (headers) { + config.headers = headers + } + const response = await axios.get(url, config) + const responseJsonString = JSON.stringify(response.data, null, 2) + const doc = new Document({ + pageContent: responseJsonString, + metadata: { + url + } + }) + return [doc] + } catch (error) { + throw new Error(`Failed to fetch ${url}: ${error}`) + } + } + + protected async executePostRequest(url: string, headers?: ICommonObject, body?: ICommonObject): Promise { + try { + const config: AxiosRequestConfig = {} + if (headers) { + config.headers = headers + } + const response = await axios.post(url, body ?? {}, config) + const responseJsonString = JSON.stringify(response.data, null, 2) + const doc = new Document({ + pageContent: responseJsonString, + metadata: { + url + } + }) + return [doc] + } catch (error) { + throw new Error(`Failed to post ${url}: ${error}`) + } + } +} + +module.exports = { + nodeClass: API_DocumentLoaders +} diff --git a/packages/components/nodes/documentloaders/API/api-loader.png b/packages/components/nodes/documentloaders/API/api-loader.png new file mode 100644 index 0000000000000000000000000000000000000000..93668c4cf1e90538e0c508f4d2f16e5dc69db3de GIT binary patch literal 1402 zcmV-=1%>*FP)z_&000000000000000_>Ycmb-iiHv?1h= zrcfDEC=FB9KFG}YMy~1*YDr)Nr!%cDH!LMgAxm($@;T^a$q>p~^qIb_Y3G|l9)#mn zL(y(n@<3A}m_qfy`KqZ@Yglr1<2fH`M>jnHEm^#BwOU;N=(l7VqhAHG&@hDTYCr%r z0s&Mo0$B27-2xawDF~nf5h_N^ zp$RY-yQnMAa@~7vCcv4CCD)gF^|0MC0siO2L{Z5L<1i!B#`7(QRr zSNFDo0B?Ph_jH6)()xzA^`)VpNFU25Ux4H1it{?+nSHrH1bAt{o@K8bvh~(ejxLt@ z>s7wYik}z(ZeE|!C6|}4X6G+XAOg5u|HR&$>vOCau#+pmvj=VMhbgS!iFCTp$c%v z$NXySLav-l0Uq0(V_)L&JvluzUaP>LvK*=a*RD~ntFS~0uyLfe`lZ##jq5Uc)iHi) zde^7Zp$V|`gADtdi*^DdphGApJP)AoCEIp zoX@wtCuSUc%^Ib1%%bGvU;dhNjW~#lNP-p$i4&mvTRz1+`o*P{&XKdV^8m%nQXd`5 zdL8sazwKI#z5aCq>wusBDY28?;`c8(_nkprq?f;%zd7f|?onr!?HP6SZXcnR3j{@ zij*+Bp)?mIP5>^q6A$gEt#tdx_-mi)^KQ;H%iwsi(xMRzRluM(U`wI2u zQGOnvzD*bcP_ajIOlMDHDQ>|F@N^$v{Zc2A3ZA((&7>e=(k@q0RJ{(O=&QEhw&tUG>;>@JdUEfU`M-4S`jwx&znkEz zVhAwDAv97ssM7WGt#`$M?Yexubwl{Qn%GCc@d>`7rApV&@9*5`9zid*VAn#d1@PK> zLK|ZJ-sha+Z*t&F`2G4=3-Elu`__}YH~ag_fXGLoi3o_^5hhYl^v*CNFw6lWW&+S5 zLs37pnGiFCnyNnnaAcW|^vE(g0XFK9V