Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add faiss

This commit is contained in:
Henry 2023-06-05 11:02:22 +01:00
parent 3fe1f5c4e2
commit d98854d85e
7 changed files with 206 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
packages/components/nodes/vectorstores/Faiss_Existing/Faiss_Existing.ts Normal file
View File

@ -0,0 +1,70 @@
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { FaissStore } from 'langchain/vectorstores/faiss'
import { Embeddings } from 'langchain/embeddings/base'
import { getBaseClasses } from '../../../src/utils'
class Faiss_Existing_VectorStores implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
outputs: INodeOutputsValue[]
constructor() {
this.label = 'Faiss Load Existing Index'
this.name = 'faissExistingIndex'
this.type = 'Faiss'
this.icon = 'faiss.svg'
this.category = 'Vector Stores'
this.description = 'Load existing index from Faiss (i.e: Document has been upserted)'
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
this.inputs = [
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Base Path to load',
name: 'basePath',
description: 'Path to load faiss.index file',
placeholder: `C:\\Users\\User\\Desktop`,
type: 'string'
}
]
this.outputs = [
{
label: 'Faiss Retriever',
name: 'retriever',
baseClasses: this.baseClasses
},
{
label: 'Faiss Vector Store',
name: 'vectorStore',
baseClasses: [this.type, ...getBaseClasses(FaissStore)]
}
]
}
async init(nodeData: INodeData): Promise<any> {
const embeddings = nodeData.inputs?.embeddings as Embeddings
const basePath = nodeData.inputs?.basePath as string
const output = nodeData.outputs?.output as string
const vectorStore = await FaissStore.load(basePath, embeddings)
if (output === 'retriever') {
const retriever = vectorStore.asRetriever()
return retriever
} else if (output === 'vectorStore') {
return vectorStore
}
return vectorStore
}
}
module.exports = { nodeClass: Faiss_Existing_VectorStores }

10
packages/components/nodes/vectorstores/Faiss_Existing/faiss.svg Normal file
View File

@ -0,0 +1,10 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-chart-dots-3" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M5 7m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
<path d="M16 15m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
<path d="M18 6m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
<path d="M6 18m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
<path d="M9 17l5 -1.5"></path>
<path d="M6.5 8.5l7.81 5.37"></path>
<path d="M7 7l8 -1"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 648 B

85
packages/components/nodes/vectorstores/Faiss_Upsert/Faiss_Upsert.ts Normal file
View File

@ -0,0 +1,85 @@
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { Embeddings } from 'langchain/embeddings/base'
import { Document } from 'langchain/document'
import { getBaseClasses } from '../../../src/utils'
import { FaissStore } from 'langchain/vectorstores/faiss'
class FaissUpsert_VectorStores implements INode {
label: string
name: string
description: string
type: string
icon: string
category: string
baseClasses: string[]
inputs: INodeParams[]
outputs: INodeOutputsValue[]
constructor() {
this.label = 'Faiss Upsert Document'
this.name = 'faissUpsert'
this.type = 'Faiss'
this.icon = 'faiss.svg'
this.category = 'Vector Stores'
this.description = 'Upsert documents to Faiss'
this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever']
this.inputs = [
{
label: 'Document',
name: 'document',
type: 'Document',
list: true
},
{
label: 'Embeddings',
name: 'embeddings',
type: 'Embeddings'
},
{
label: 'Base Path to store',
name: 'basePath',
description: 'Path to store faiss.index file',
placeholder: `C:\\Users\\User\\Desktop`,
type: 'string'
}
]
this.outputs = [
{
label: 'Faiss Retriever',
name: 'retriever',
baseClasses: this.baseClasses
},
{
label: 'Faiss Vector Store',
name: 'vectorStore',
baseClasses: [this.type, ...getBaseClasses(FaissStore)]
}
]
}
async init(nodeData: INodeData): Promise<any> {
const docs = nodeData.inputs?.document as Document[]
const embeddings = nodeData.inputs?.embeddings as Embeddings
const output = nodeData.outputs?.output as string
const basePath = nodeData.inputs?.basePath as string
const flattenDocs = docs && docs.length ? docs.flat() : []
const finalDocs = []
for (let i = 0; i < flattenDocs.length; i += 1) {
finalDocs.push(new Document(flattenDocs[i]))
}
const vectorStore = await FaissStore.fromDocuments(finalDocs, embeddings)
await vectorStore.save(basePath)
if (output === 'retriever') {
const retriever = vectorStore.asRetriever()
return retriever
} else if (output === 'vectorStore') {
return vectorStore
}
return vectorStore
}
}
module.exports = { nodeClass: FaissUpsert_VectorStores }

10
packages/components/nodes/vectorstores/Faiss_Upsert/faiss.svg Normal file
View File

@ -0,0 +1,10 @@
<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-chart-dots-3" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round">
<path stroke="none" d="M0 0h24v24H0z" fill="none"></path>
<path d="M5 7m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
<path d="M16 15m-2 0a2 2 0 1 0 4 0a2 2 0 1 0 -4 0"></path>
<path d="M18 6m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
<path d="M6 18m-3 0a3 3 0 1 0 6 0a3 3 0 1 0 -6 0"></path>
<path d="M9 17l5 -1.5"></path>
<path d="M6.5 8.5l7.81 5.37"></path>
<path d="M7 7l8 -1"></path>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 648 B

1
packages/components/package.json
View File

@ -28,6 +28,7 @@
"d3-dsv": "2",
"dotenv": "^16.0.0",
"express": "^4.17.3",
"faiss-node": "^0.2.1",
"form-data": "^4.0.0",
"graphql": "^16.6.0",
"langchain": "^0.0.84",

4
packages/server/src/index.ts
View File

@ -34,7 +34,8 @@ import {
findAvailableConfigs,
isSameOverrideConfig,
replaceAllAPIKeys,
isFlowValidForStream
isFlowValidForStream,
isVectorStoreFaiss
} from './utils'
import { cloneDeep } from 'lodash'
import { getDataSource } from './DataSource'
@ -634,6 +635,7 @@ export class App {
const nodeModule = await import(nodeInstanceFilePath)
const nodeInstance = new nodeModule.nodeClass()
isStreamValid = isStreamValid && !isVectorStoreFaiss(nodeToExecuteData)
const result = isStreamValid
? await nodeInstance.run(nodeToExecuteData, incomingInput.question, {
chatHistory: incomingInput.history,

30
packages/server/src/utils/index.ts
View File

@ -14,7 +14,7 @@ import {
INodeData,
IOverrideConfig
} from '../Interface'
import { cloneDeep, get } from 'lodash'
import { cloneDeep, get, omit, merge } from 'lodash'
import { ICommonObject, getInputVariables } from 'flowise-components'
import { scryptSync, randomBytes, timingSafeEqual } from 'crypto'
@ -317,6 +317,25 @@ export const getVariableValue = (paramValue: string, reactFlowNodes: IReactFlowN
return returnVal
}
/**
* Temporarily disable streaming if vectorStore is Faiss
* @param {INodeData} flowNodeData
* @returns {boolean}
*/
export const isVectorStoreFaiss = (flowNodeData: INodeData) => {
if (flowNodeData.inputs && flowNodeData.inputs.vectorStoreRetriever) {
const vectorStoreRetriever = flowNodeData.inputs.vectorStoreRetriever
if (typeof vectorStoreRetriever === 'string' && vectorStoreRetriever.includes('faiss')) return true
if (
typeof vectorStoreRetriever === 'object' &&
vectorStoreRetriever.vectorStore &&
vectorStoreRetriever.vectorStore.constructor.name === 'FaissStore'
)
return true
}
return false
}
/**
* Loop through each inputs and resolve variable if neccessary
* @param {INodeData} reactFlowNodeData
@ -325,7 +344,12 @@ export const getVariableValue = (paramValue: string, reactFlowNodes: IReactFlowN
* @returns {INodeData}
*/
export const resolveVariables = (reactFlowNodeData: INodeData, reactFlowNodes: IReactFlowNode[], question: string): INodeData => {
const flowNodeData = cloneDeep(reactFlowNodeData)
let flowNodeData = cloneDeep(reactFlowNodeData)
if (reactFlowNodeData.instance && isVectorStoreFaiss(reactFlowNodeData)) {
// omit and merge because cloneDeep of instance gives "Illegal invocation" Exception
const flowNodeDataWithoutInstance = cloneDeep(omit(reactFlowNodeData, ['instance']))
flowNodeData = merge(flowNodeDataWithoutInstance, { instance: reactFlowNodeData.instance })
}
const types = 'inputs'
const getParamValues = (paramsObj: ICommonObject) => {
@ -633,5 +657,5 @@ export const isFlowValidForStream = (reactFlowNodes: IReactFlowNode[], endingNod
}
}
return isChatOrLLMsExist && endingNodeData.category === 'Chains'
return isChatOrLLMsExist && endingNodeData.category === 'Chains' && !isVectorStoreFaiss(endingNodeData)
}