implemented serpapi loader integration and added its corresponding template

This commit is contained in:
Hamzah Abdulfattah 2023-08-19 17:23:19 +01:00
parent 260b13bb99
commit 3b0c5b0c0d
3 changed files with 678 additions and 0 deletions

View File

@ -0,0 +1,82 @@
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface';
import { TextSplitter } from 'langchain/text_splitter';
import { SerpAPILoader } from 'langchain/document_loaders/web/serpapi';import { getCredentialData, getCredentialParam } from '../../../src';
class SerpAPI_DocumentLoaders implements INode {
label: string;
name: string;
version: number;
description: string;
type: string;
icon: string;
category: string;
baseClasses: string[];
credential: INodeParams;
inputs: INodeParams[];
constructor() {
this.label = 'SerpApi For Web Search';
this.name = 'serpApi';
this.version = 1.0;
this.type = 'Document';
this.icon = 'serp.png';
this.category = 'Document Loaders';
this.description = 'Load and process data from web search results';
this.baseClasses = [this.type];
this.credential = {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
optional: false,
credentialNames: ['serpApi']
};
this.inputs = [
{
label: 'Query',
name: 'query',
type: 'string'
},
{
label: 'Text Splitter',
name: 'textSplitter',
type: 'TextSplitter',
optional: true
},
{
label: 'Metadata',
name: 'metadata',
type: 'json',
optional: true,
additionalParams: true
}
];
}
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter;
const query = nodeData.inputs?.query as string;
const metadata = nodeData.inputs?.metadata;
const credentialData = await getCredentialData(nodeData.credential ?? '', options);
const serpApiKey = getCredentialParam('serpApiKey', credentialData, nodeData);
const loader = new SerpAPILoader({ q: query, apiKey: serpApiKey });
const docs = textSplitter ? await loader.loadAndSplit() : await loader.load();
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata);
return docs.map((doc) => {
return {
...doc,
metadata: {
...doc.metadata,
...parsedMetadata
}
};
});
}
return docs;
}
}
module.exports = { nodeClass: SerpAPI_DocumentLoaders };

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.3 KiB

View File

@ -0,0 +1,596 @@
{
"nodes": [
{
"width": 300,
"height": 525,
"id": "chatOpenAI_0",
"position": {
"x": 858.6611234371157,
"y": 40.225188014879166
},
"type": "customNode",
"data": {
"id": "chatOpenAI_0",
"label": "ChatOpenAI",
"version": 1,
"name": "chatOpenAI",
"type": "ChatOpenAI",
"baseClasses": [
"ChatOpenAI",
"BaseChatModel",
"BaseLanguageModel",
"Runnable"
],
"category": "Chat Models",
"description": "Wrapper around OpenAI large language models that use the Chat endpoint",
"inputParams": [
{
"label": "Connect Credential",
"name": "credential",
"type": "credential",
"credentialNames": [
"openAIApi"
],
"id": "chatOpenAI_0-input-credential-credential"
},
{
"label": "Model Name",
"name": "modelName",
"type": "options",
"options": [
{
"label": "gpt-4",
"name": "gpt-4"
},
{
"label": "gpt-4-0613",
"name": "gpt-4-0613"
},
{
"label": "gpt-4-32k",
"name": "gpt-4-32k"
},
{
"label": "gpt-4-32k-0613",
"name": "gpt-4-32k-0613"
},
{
"label": "gpt-3.5-turbo",
"name": "gpt-3.5-turbo"
},
{
"label": "gpt-3.5-turbo-0613",
"name": "gpt-3.5-turbo-0613"
},
{
"label": "gpt-3.5-turbo-16k",
"name": "gpt-3.5-turbo-16k"
},
{
"label": "gpt-3.5-turbo-16k-0613",
"name": "gpt-3.5-turbo-16k-0613"
}
],
"default": "gpt-3.5-turbo",
"optional": true,
"id": "chatOpenAI_0-input-modelName-options"
},
{
"label": "Temperature",
"name": "temperature",
"type": "number",
"step": 0.1,
"default": 0.9,
"optional": true,
"id": "chatOpenAI_0-input-temperature-number"
},
{
"label": "Max Tokens",
"name": "maxTokens",
"type": "number",
"step": 1,
"optional": true,
"additionalParams": true,
"id": "chatOpenAI_0-input-maxTokens-number"
},
{
"label": "Top Probability",
"name": "topP",
"type": "number",
"step": 0.1,
"optional": true,
"additionalParams": true,
"id": "chatOpenAI_0-input-topP-number"
},
{
"label": "Frequency Penalty",
"name": "frequencyPenalty",
"type": "number",
"step": 0.1,
"optional": true,
"additionalParams": true,
"id": "chatOpenAI_0-input-frequencyPenalty-number"
},
{
"label": "Presence Penalty",
"name": "presencePenalty",
"type": "number",
"step": 0.1,
"optional": true,
"additionalParams": true,
"id": "chatOpenAI_0-input-presencePenalty-number"
},
{
"label": "Timeout",
"name": "timeout",
"type": "number",
"step": 1,
"optional": true,
"additionalParams": true,
"id": "chatOpenAI_0-input-timeout-number"
},
{
"label": "BasePath",
"name": "basepath",
"type": "string",
"optional": true,
"additionalParams": true,
"id": "chatOpenAI_0-input-basepath-string"
},
{
"label": "BaseOptions",
"name": "baseOptions",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "chatOpenAI_0-input-baseOptions-json"
}
],
"inputAnchors": [],
"inputs": {
"modelName": "gpt-4",
"temperature": 0.9,
"maxTokens": "",
"topP": "",
"frequencyPenalty": "",
"presencePenalty": "",
"timeout": "",
"basepath": "",
"baseOptions": ""
},
"outputAnchors": [
{
"id": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
"name": "chatOpenAI",
"label": "ChatOpenAI",
"type": "ChatOpenAI | BaseChatModel | BaseLanguageModel | Runnable"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 858.6611234371157,
"y": 40.225188014879166
},
"dragging": false
},
{
"width": 300,
"height": 481,
"id": "conversationalRetrievalQAChain_0",
"position": {
"x": 1584.2201037703737,
"y": 207.80548502487522
},
"type": "customNode",
"data": {
"id": "conversationalRetrievalQAChain_0",
"label": "Conversational Retrieval QA Chain",
"version": 1,
"name": "conversationalRetrievalQAChain",
"type": "ConversationalRetrievalQAChain",
"baseClasses": [
"ConversationalRetrievalQAChain",
"BaseChain",
"Runnable"
],
"category": "Chains",
"description": "Document QA - built on RetrievalQAChain to provide a chat history component",
"inputParams": [
{
"label": "Return Source Documents",
"name": "returnSourceDocuments",
"type": "boolean",
"optional": true,
"id": "conversationalRetrievalQAChain_0-input-returnSourceDocuments-boolean"
},
{
"label": "System Message",
"name": "systemMessagePrompt",
"type": "string",
"rows": 4,
"additionalParams": true,
"optional": true,
"placeholder": "I want you to act as a document that I am having a conversation with. Your name is \"AI Assistant\". You will provide me with answers from the given info. If the answer is not included, say exactly \"Hmm, I am not sure.\" and stop after that. Refuse to answer any question not about the info. Never break character.",
"id": "conversationalRetrievalQAChain_0-input-systemMessagePrompt-string"
},
{
"label": "Chain Option",
"name": "chainOption",
"type": "options",
"options": [
{
"label": "MapReduceDocumentsChain",
"name": "map_reduce",
"description": "Suitable for QA tasks over larger documents and can run the preprocessing step in parallel, reducing the running time"
},
{
"label": "RefineDocumentsChain",
"name": "refine",
"description": "Suitable for QA tasks over a large number of documents."
},
{
"label": "StuffDocumentsChain",
"name": "stuff",
"description": "Suitable for QA tasks over a small number of documents."
}
],
"additionalParams": true,
"optional": true,
"id": "conversationalRetrievalQAChain_0-input-chainOption-options"
}
],
"inputAnchors": [
{
"label": "Language Model",
"name": "model",
"type": "BaseLanguageModel",
"id": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel"
},
{
"label": "Vector Store Retriever",
"name": "vectorStoreRetriever",
"type": "BaseRetriever",
"id": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever"
},
{
"label": "Memory",
"name": "memory",
"type": "BaseMemory",
"optional": true,
"description": "If left empty, a default BufferMemory will be used",
"id": "conversationalRetrievalQAChain_0-input-memory-BaseMemory"
}
],
"inputs": {
"model": "{{chatOpenAI_0.data.instance}}",
"vectorStoreRetriever": "{{memoryVectorStore_0.data.instance}}",
"memory": "",
"returnSourceDocuments": true,
"systemMessagePrompt": "",
"chainOption": ""
},
"outputAnchors": [
{
"id": "conversationalRetrievalQAChain_0-output-conversationalRetrievalQAChain-ConversationalRetrievalQAChain|BaseChain|Runnable",
"name": "conversationalRetrievalQAChain",
"label": "ConversationalRetrievalQAChain",
"type": "ConversationalRetrievalQAChain | BaseChain | Runnable"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 1584.2201037703737,
"y": 207.80548502487522
},
"dragging": false
},
{
"width": 300,
"height": 330,
"id": "openAIEmbeddings_0",
"position": {
"x": 59.99572465636521,
"y": 789.1458487580028
},
"type": "customNode",
"data": {
"id": "openAIEmbeddings_0",
"label": "OpenAI Embeddings",
"version": 1,
"name": "openAIEmbeddings",
"type": "OpenAIEmbeddings",
"baseClasses": [
"OpenAIEmbeddings",
"Embeddings"
],
"category": "Embeddings",
"description": "OpenAI API to generate embeddings for a given text",
"inputParams": [
{
"label": "Connect Credential",
"name": "credential",
"type": "credential",
"credentialNames": [
"openAIApi"
],
"id": "openAIEmbeddings_0-input-credential-credential"
},
{
"label": "Strip New Lines",
"name": "stripNewLines",
"type": "boolean",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_0-input-stripNewLines-boolean"
},
{
"label": "Batch Size",
"name": "batchSize",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_0-input-batchSize-number"
},
{
"label": "Timeout",
"name": "timeout",
"type": "number",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_0-input-timeout-number"
},
{
"label": "BasePath",
"name": "basepath",
"type": "string",
"optional": true,
"additionalParams": true,
"id": "openAIEmbeddings_0-input-basepath-string"
}
],
"inputAnchors": [],
"inputs": {
"stripNewLines": "",
"batchSize": "",
"timeout": "",
"basepath": ""
},
"outputAnchors": [
{
"id": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
"name": "openAIEmbeddings",
"label": "OpenAIEmbeddings",
"type": "OpenAIEmbeddings | Embeddings"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 59.99572465636521,
"y": 789.1458487580028
},
"dragging": false
},
{
"width": 300,
"height": 407,
"id": "memoryVectorStore_0",
"position": {
"x": 804.8405953555847,
"y": 780.514305397237
},
"type": "customNode",
"data": {
"id": "memoryVectorStore_0",
"label": "In-Memory Vector Store",
"version": 1,
"name": "memoryVectorStore",
"type": "Memory",
"baseClasses": [
"Memory",
"VectorStoreRetriever",
"BaseRetriever"
],
"category": "Vector Stores",
"description": "In-memory vectorstore that stores embeddings and does an exact, linear search for the most similar embeddings.",
"inputParams": [
{
"label": "Top K",
"name": "topK",
"description": "Number of top results to fetch. Default to 4",
"placeholder": "4",
"type": "number",
"optional": true,
"id": "memoryVectorStore_0-input-topK-number"
}
],
"inputAnchors": [
{
"label": "Document",
"name": "document",
"type": "Document",
"list": true,
"id": "memoryVectorStore_0-input-document-Document"
},
{
"label": "Embeddings",
"name": "embeddings",
"type": "Embeddings",
"id": "memoryVectorStore_0-input-embeddings-Embeddings"
}
],
"inputs": {
"document": [
"{{serpApi_2.data.instance}}"
],
"embeddings": "{{openAIEmbeddings_0.data.instance}}",
"topK": ""
},
"outputAnchors": [
{
"name": "output",
"label": "Output",
"type": "options",
"options": [
{
"id": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever",
"name": "retriever",
"label": "Memory Retriever",
"type": "Memory | VectorStoreRetriever | BaseRetriever"
},
{
"id": "memoryVectorStore_0-output-vectorStore-Memory|VectorStore",
"name": "vectorStore",
"label": "Memory Vector Store",
"type": "Memory | VectorStore"
}
],
"default": "retriever"
}
],
"outputs": {
"output": "retriever"
},
"selected": false
},
"positionAbsolute": {
"x": 804.8405953555847,
"y": 780.514305397237
},
"selected": false,
"dragging": false
},
{
"width": 300,
"height": 480,
"id": "serpApi_2",
"position": {
"x": 199.84883781665212,
"y": 113.02413239018131
},
"type": "customNode",
"data": {
"id": "serpApi_2",
"label": "SerpApi For Web Search",
"version": 1,
"name": "serpApi",
"type": "Document",
"baseClasses": [
"Document"
],
"category": "Document Loaders",
"description": "Load and process data from web search results",
"inputParams": [
{
"label": "Connect Credential",
"name": "credential",
"type": "credential",
"optional": false,
"credentialNames": [
"serpApi"
],
"id": "serpApi_2-input-credential-credential"
},
{
"label": "Query",
"name": "query",
"type": "string",
"id": "serpApi_2-input-query-string"
},
{
"label": "Metadata",
"name": "metadata",
"type": "json",
"optional": true,
"additionalParams": true,
"id": "serpApi_2-input-metadata-json"
}
],
"inputAnchors": [
{
"label": "Text Splitter",
"name": "textSplitter",
"type": "TextSplitter",
"optional": true,
"id": "serpApi_2-input-textSplitter-TextSplitter"
}
],
"inputs": {
"query": "Coffee",
"textSplitter": "",
"metadata": ""
},
"outputAnchors": [
{
"id": "serpApi_2-output-serpApi-Document",
"name": "serpApi",
"label": "Document",
"type": "Document"
}
],
"outputs": {},
"selected": false
},
"selected": false,
"positionAbsolute": {
"x": 199.84883781665212,
"y": 113.02413239018131
},
"dragging": false
}
],
"edges": [
{
"source": "openAIEmbeddings_0",
"sourceHandle": "openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings",
"target": "memoryVectorStore_0",
"targetHandle": "memoryVectorStore_0-input-embeddings-Embeddings",
"type": "buttonedge",
"id": "openAIEmbeddings_0-openAIEmbeddings_0-output-openAIEmbeddings-OpenAIEmbeddings|Embeddings-memoryVectorStore_0-memoryVectorStore_0-input-embeddings-Embeddings",
"data": {
"label": ""
}
},
{
"source": "memoryVectorStore_0",
"sourceHandle": "memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever",
"target": "conversationalRetrievalQAChain_0",
"targetHandle": "conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
"type": "buttonedge",
"id": "memoryVectorStore_0-memoryVectorStore_0-output-retriever-Memory|VectorStoreRetriever|BaseRetriever-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-vectorStoreRetriever-BaseRetriever",
"data": {
"label": ""
}
},
{
"source": "chatOpenAI_0",
"sourceHandle": "chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable",
"target": "conversationalRetrievalQAChain_0",
"targetHandle": "conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
"type": "buttonedge",
"id": "chatOpenAI_0-chatOpenAI_0-output-chatOpenAI-ChatOpenAI|BaseChatModel|BaseLanguageModel|Runnable-conversationalRetrievalQAChain_0-conversationalRetrievalQAChain_0-input-model-BaseLanguageModel",
"data": {
"label": ""
}
},
{
"source": "serpApi_2",
"sourceHandle": "serpApi_2-output-serpApi-Document",
"target": "memoryVectorStore_0",
"targetHandle": "memoryVectorStore_0-input-document-Document",
"type": "buttonedge",
"id": "serpApi_2-serpApi_2-output-serpApi-Document-memoryVectorStore_0-memoryVectorStore_0-input-document-Document",
"data": {
"label": ""
}
}
]
}