Compare commits
1 Commits
main
...
feature/Ge
| Author | SHA1 | Date |
|---|---|---|
Henry
|
d499e92887 |
|
|
@ -62,6 +62,7 @@ class MySQLRecordManager_RecordManager implements INode {
|
|||
label: 'Namespace',
|
||||
name: 'namespace',
|
||||
type: 'string',
|
||||
description: 'If not specified, chatflowid will be used',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
|
|
@ -218,16 +219,7 @@ class MySQLRecordManager implements RecordManagerInterface {
|
|||
unique key \`unique_key_namespace\` (\`key\`,
|
||||
\`namespace\`));`)
|
||||
|
||||
// Add doc_id column if it doesn't exist (migration for existing tables)
|
||||
const checkColumn = await queryRunner.manager.query(
|
||||
`SELECT COUNT(1) ColumnExists FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE table_schema=DATABASE() AND table_name='${tableName}' AND column_name='doc_id';`
|
||||
)
|
||||
if (checkColumn[0].ColumnExists === 0) {
|
||||
await queryRunner.manager.query(`ALTER TABLE \`${tableName}\` ADD COLUMN \`doc_id\` longtext;`)
|
||||
}
|
||||
|
||||
const columns = [`updated_at`, `key`, `namespace`, `group_id`, `doc_id`]
|
||||
const columns = [`updated_at`, `key`, `namespace`, `group_id`]
|
||||
for (const column of columns) {
|
||||
// MySQL does not support 'IF NOT EXISTS' function for Index
|
||||
const Check = await queryRunner.manager.query(
|
||||
|
|
@ -269,7 +261,7 @@ class MySQLRecordManager implements RecordManagerInterface {
|
|||
}
|
||||
}
|
||||
|
||||
async update(keys: Array<{ uid: string; docId: string }> | string[], updateOptions?: UpdateOptions): Promise<void> {
|
||||
async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
|
||||
if (keys.length === 0) {
|
||||
return
|
||||
}
|
||||
|
|
@ -285,23 +277,23 @@ class MySQLRecordManager implements RecordManagerInterface {
|
|||
throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
|
||||
}
|
||||
|
||||
// Handle both new format (objects with uid and docId) and old format (strings)
|
||||
const isNewFormat = keys.length > 0 && typeof keys[0] === 'object' && 'uid' in keys[0]
|
||||
const keyStrings = isNewFormat ? (keys as Array<{ uid: string; docId: string }>).map((k) => k.uid) : (keys as string[])
|
||||
const docIds = isNewFormat ? (keys as Array<{ uid: string; docId: string }>).map((k) => k.docId) : keys.map(() => null)
|
||||
const groupIds = _groupIds ?? keys.map(() => null)
|
||||
|
||||
const groupIds = _groupIds ?? keyStrings.map(() => null)
|
||||
|
||||
if (groupIds.length !== keyStrings.length) {
|
||||
throw new Error(`Number of keys (${keyStrings.length}) does not match number of group_ids (${groupIds.length})`)
|
||||
if (groupIds.length !== keys.length) {
|
||||
throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`)
|
||||
}
|
||||
|
||||
const recordsToUpsert = keyStrings.map((key, i) => [key, this.namespace, updatedAt, groupIds[i] ?? null, docIds[i] ?? null])
|
||||
const recordsToUpsert = keys.map((key, i) => [
|
||||
key,
|
||||
this.namespace,
|
||||
updatedAt,
|
||||
groupIds[i] ?? null // Ensure groupIds[i] is null if undefined
|
||||
])
|
||||
|
||||
const query = `
|
||||
INSERT INTO \`${tableName}\` (\`key\`, \`namespace\`, \`updated_at\`, \`group_id\`, \`doc_id\`)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE \`updated_at\` = VALUES(\`updated_at\`), \`doc_id\` = VALUES(\`doc_id\`)`
|
||||
INSERT INTO \`${tableName}\` (\`key\`, \`namespace\`, \`updated_at\`, \`group_id\`)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE \`updated_at\` = VALUES(\`updated_at\`)`
|
||||
|
||||
// To handle multiple files upsert
|
||||
try {
|
||||
|
|
@ -357,13 +349,13 @@ class MySQLRecordManager implements RecordManagerInterface {
|
|||
}
|
||||
}
|
||||
|
||||
async listKeys(options?: ListKeyOptions & { docId?: string }): Promise<string[]> {
|
||||
async listKeys(options?: ListKeyOptions): Promise<string[]> {
|
||||
const dataSource = await this.getDataSource()
|
||||
const queryRunner = dataSource.createQueryRunner()
|
||||
const tableName = this.sanitizeTableName(this.tableName)
|
||||
|
||||
try {
|
||||
const { before, after, limit, groupIds, docId } = options ?? {}
|
||||
const { before, after, limit, groupIds } = options ?? {}
|
||||
let query = `SELECT \`key\` FROM \`${tableName}\` WHERE \`namespace\` = ?`
|
||||
const values: (string | number | string[])[] = [this.namespace]
|
||||
|
||||
|
|
@ -390,11 +382,6 @@ class MySQLRecordManager implements RecordManagerInterface {
|
|||
values.push(...groupIds.filter((gid): gid is string => gid !== null))
|
||||
}
|
||||
|
||||
if (docId) {
|
||||
query += ` AND \`doc_id\` = ?`
|
||||
values.push(docId)
|
||||
}
|
||||
|
||||
query += ';'
|
||||
|
||||
// Directly using try/catch with async/await for cleaner flow
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ class PostgresRecordManager_RecordManager implements INode {
|
|||
label: 'Namespace',
|
||||
name: 'namespace',
|
||||
type: 'string',
|
||||
description: 'If not specified, chatflowid will be used',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
|
|
@ -240,19 +241,6 @@ class PostgresRecordManager implements RecordManagerInterface {
|
|||
CREATE INDEX IF NOT EXISTS namespace_index ON "${tableName}" (namespace);
|
||||
CREATE INDEX IF NOT EXISTS group_id_index ON "${tableName}" (group_id);`)
|
||||
|
||||
// Add doc_id column if it doesn't exist (migration for existing tables)
|
||||
await queryRunner.manager.query(`
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = '${tableName}' AND column_name = 'doc_id'
|
||||
) THEN
|
||||
ALTER TABLE "${tableName}" ADD COLUMN doc_id TEXT;
|
||||
CREATE INDEX IF NOT EXISTS doc_id_index ON "${tableName}" (doc_id);
|
||||
END IF;
|
||||
END $$;`)
|
||||
|
||||
await queryRunner.release()
|
||||
} catch (e: any) {
|
||||
// This error indicates that the table already exists
|
||||
|
|
@ -298,7 +286,7 @@ class PostgresRecordManager implements RecordManagerInterface {
|
|||
return `(${placeholders.join(', ')})`
|
||||
}
|
||||
|
||||
async update(keys: Array<{ uid: string; docId: string }> | string[], updateOptions?: UpdateOptions): Promise<void> {
|
||||
async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
|
||||
if (keys.length === 0) {
|
||||
return
|
||||
}
|
||||
|
|
@ -314,22 +302,17 @@ class PostgresRecordManager implements RecordManagerInterface {
|
|||
throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
|
||||
}
|
||||
|
||||
// Handle both new format (objects with uid and docId) and old format (strings)
|
||||
const isNewFormat = keys.length > 0 && typeof keys[0] === 'object' && 'uid' in keys[0]
|
||||
const keyStrings = isNewFormat ? (keys as Array<{ uid: string; docId: string }>).map((k) => k.uid) : (keys as string[])
|
||||
const docIds = isNewFormat ? (keys as Array<{ uid: string; docId: string }>).map((k) => k.docId) : keys.map(() => null)
|
||||
const groupIds = _groupIds ?? keys.map(() => null)
|
||||
|
||||
const groupIds = _groupIds ?? keyStrings.map(() => null)
|
||||
|
||||
if (groupIds.length !== keyStrings.length) {
|
||||
throw new Error(`Number of keys (${keyStrings.length}) does not match number of group_ids ${groupIds.length})`)
|
||||
if (groupIds.length !== keys.length) {
|
||||
throw new Error(`Number of keys (${keys.length}) does not match number of group_ids ${groupIds.length})`)
|
||||
}
|
||||
|
||||
const recordsToUpsert = keyStrings.map((key, i) => [key, this.namespace, updatedAt, groupIds[i], docIds[i]])
|
||||
const recordsToUpsert = keys.map((key, i) => [key, this.namespace, updatedAt, groupIds[i]])
|
||||
|
||||
const valuesPlaceholders = recordsToUpsert.map((_, j) => this.generatePlaceholderForRowAt(j, recordsToUpsert[0].length)).join(', ')
|
||||
|
||||
const query = `INSERT INTO "${tableName}" (key, namespace, updated_at, group_id, doc_id) VALUES ${valuesPlaceholders} ON CONFLICT (key, namespace) DO UPDATE SET updated_at = EXCLUDED.updated_at, doc_id = EXCLUDED.doc_id;`
|
||||
const query = `INSERT INTO "${tableName}" (key, namespace, updated_at, group_id) VALUES ${valuesPlaceholders} ON CONFLICT (key, namespace) DO UPDATE SET updated_at = EXCLUDED.updated_at;`
|
||||
try {
|
||||
await queryRunner.manager.query(query, recordsToUpsert.flat())
|
||||
await queryRunner.release()
|
||||
|
|
@ -368,8 +351,8 @@ class PostgresRecordManager implements RecordManagerInterface {
|
|||
}
|
||||
}
|
||||
|
||||
async listKeys(options?: ListKeyOptions & { docId?: string }): Promise<string[]> {
|
||||
const { before, after, limit, groupIds, docId } = options ?? {}
|
||||
async listKeys(options?: ListKeyOptions): Promise<string[]> {
|
||||
const { before, after, limit, groupIds } = options ?? {}
|
||||
const tableName = this.sanitizeTableName(this.tableName)
|
||||
|
||||
let query = `SELECT key FROM "${tableName}" WHERE namespace = $1`
|
||||
|
|
@ -400,12 +383,6 @@ class PostgresRecordManager implements RecordManagerInterface {
|
|||
index += 1
|
||||
}
|
||||
|
||||
if (docId) {
|
||||
values.push(docId)
|
||||
query += ` AND doc_id = $${index}`
|
||||
index += 1
|
||||
}
|
||||
|
||||
query += ';'
|
||||
|
||||
const dataSource = await this.getDataSource()
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ class SQLiteRecordManager_RecordManager implements INode {
|
|||
label: 'Namespace',
|
||||
name: 'namespace',
|
||||
type: 'string',
|
||||
description: 'If not specified, chatflowid will be used',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
|
|
@ -197,15 +198,6 @@ CREATE INDEX IF NOT EXISTS key_index ON "${tableName}" (key);
|
|||
CREATE INDEX IF NOT EXISTS namespace_index ON "${tableName}" (namespace);
|
||||
CREATE INDEX IF NOT EXISTS group_id_index ON "${tableName}" (group_id);`)
|
||||
|
||||
// Add doc_id column if it doesn't exist (migration for existing tables)
|
||||
const checkColumn = await queryRunner.manager.query(
|
||||
`SELECT COUNT(*) as count FROM pragma_table_info('${tableName}') WHERE name='doc_id';`
|
||||
)
|
||||
if (checkColumn[0].count === 0) {
|
||||
await queryRunner.manager.query(`ALTER TABLE "${tableName}" ADD COLUMN doc_id TEXT;`)
|
||||
await queryRunner.manager.query(`CREATE INDEX IF NOT EXISTS doc_id_index ON "${tableName}" (doc_id);`)
|
||||
}
|
||||
|
||||
await queryRunner.release()
|
||||
} catch (e: any) {
|
||||
// This error indicates that the table already exists
|
||||
|
|
@ -236,7 +228,7 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${tableName}" (group_id);`)
|
|||
}
|
||||
}
|
||||
|
||||
async update(keys: Array<{ uid: string; docId: string }> | string[], updateOptions?: UpdateOptions): Promise<void> {
|
||||
async update(keys: string[], updateOptions?: UpdateOptions): Promise<void> {
|
||||
if (keys.length === 0) {
|
||||
return
|
||||
}
|
||||
|
|
@ -251,23 +243,23 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${tableName}" (group_id);`)
|
|||
throw new Error(`Time sync issue with database ${updatedAt} < ${timeAtLeast}`)
|
||||
}
|
||||
|
||||
// Handle both new format (objects with uid and docId) and old format (strings)
|
||||
const isNewFormat = keys.length > 0 && typeof keys[0] === 'object' && 'uid' in keys[0]
|
||||
const keyStrings = isNewFormat ? (keys as Array<{ uid: string; docId: string }>).map((k) => k.uid) : (keys as string[])
|
||||
const docIds = isNewFormat ? (keys as Array<{ uid: string; docId: string }>).map((k) => k.docId) : keys.map(() => null)
|
||||
const groupIds = _groupIds ?? keys.map(() => null)
|
||||
|
||||
const groupIds = _groupIds ?? keyStrings.map(() => null)
|
||||
|
||||
if (groupIds.length !== keyStrings.length) {
|
||||
throw new Error(`Number of keys (${keyStrings.length}) does not match number of group_ids (${groupIds.length})`)
|
||||
if (groupIds.length !== keys.length) {
|
||||
throw new Error(`Number of keys (${keys.length}) does not match number of group_ids (${groupIds.length})`)
|
||||
}
|
||||
|
||||
const recordsToUpsert = keyStrings.map((key, i) => [key, this.namespace, updatedAt, groupIds[i] ?? null, docIds[i] ?? null])
|
||||
const recordsToUpsert = keys.map((key, i) => [
|
||||
key,
|
||||
this.namespace,
|
||||
updatedAt,
|
||||
groupIds[i] ?? null // Ensure groupIds[i] is null if undefined
|
||||
])
|
||||
|
||||
const query = `
|
||||
INSERT INTO "${tableName}" (key, namespace, updated_at, group_id, doc_id)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at, doc_id = excluded.doc_id`
|
||||
INSERT INTO "${tableName}" (key, namespace, updated_at, group_id)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at`
|
||||
|
||||
try {
|
||||
// To handle multiple files upsert
|
||||
|
|
@ -322,8 +314,8 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${tableName}" (group_id);`)
|
|||
}
|
||||
}
|
||||
|
||||
async listKeys(options?: ListKeyOptions & { docId?: string }): Promise<string[]> {
|
||||
const { before, after, limit, groupIds, docId } = options ?? {}
|
||||
async listKeys(options?: ListKeyOptions): Promise<string[]> {
|
||||
const { before, after, limit, groupIds } = options ?? {}
|
||||
const tableName = this.sanitizeTableName(this.tableName)
|
||||
|
||||
let query = `SELECT key FROM "${tableName}" WHERE namespace = ?`
|
||||
|
|
@ -352,11 +344,6 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${tableName}" (group_id);`)
|
|||
values.push(...groupIds.filter((gid): gid is string => gid !== null))
|
||||
}
|
||||
|
||||
if (docId) {
|
||||
query += ` AND doc_id = ?`
|
||||
values.push(docId)
|
||||
}
|
||||
|
||||
query += ';'
|
||||
|
||||
const dataSource = await this.getDataSource()
|
||||
|
|
|
|||
|
|
@ -186,11 +186,7 @@ class Chroma_VectorStores implements INode {
|
|||
const vectorStoreName = collectionName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
const chromaStore = new ChromaExtended(embeddings, obj)
|
||||
|
||||
|
|
|
|||
|
|
@ -198,11 +198,7 @@ class Elasticsearch_VectorStores implements INode {
|
|||
const vectorStoreName = indexName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await vectorStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
|
|
|
|||
|
|
@ -212,11 +212,7 @@ class Pinecone_VectorStores implements INode {
|
|||
const vectorStoreName = pineconeNamespace
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await pineconeStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ class Postgres_VectorStores implements INode {
|
|||
constructor() {
|
||||
this.label = 'Postgres'
|
||||
this.name = 'postgres'
|
||||
this.version = 7.1
|
||||
this.version = 7.0
|
||||
this.type = 'Postgres'
|
||||
this.icon = 'postgres.svg'
|
||||
this.category = 'Vector Stores'
|
||||
|
|
@ -173,15 +173,6 @@ class Postgres_VectorStores implements INode {
|
|||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Upsert Batch Size',
|
||||
name: 'batchSize',
|
||||
type: 'number',
|
||||
step: 1,
|
||||
description: 'Upsert in batches of size N',
|
||||
additionalParams: true,
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Additional Configuration',
|
||||
name: 'additionalConfig',
|
||||
|
|
@ -241,7 +232,6 @@ class Postgres_VectorStores implements INode {
|
|||
const docs = nodeData.inputs?.document as Document[]
|
||||
const recordManager = nodeData.inputs?.recordManager
|
||||
const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean
|
||||
const _batchSize = nodeData.inputs?.batchSize
|
||||
const vectorStoreDriver: VectorStoreDriver = Postgres_VectorStores.getDriverFromConfig(nodeData, options)
|
||||
|
||||
const flattenDocs = docs && docs.length ? flatten(docs) : []
|
||||
|
|
@ -274,16 +264,8 @@ class Postgres_VectorStores implements INode {
|
|||
})
|
||||
|
||||
return res
|
||||
} else {
|
||||
if (_batchSize) {
|
||||
const batchSize = parseInt(_batchSize, 10)
|
||||
for (let i = 0; i < finalDocs.length; i += batchSize) {
|
||||
const batch = finalDocs.slice(i, i + batchSize)
|
||||
await vectorStoreDriver.fromDocuments(batch)
|
||||
}
|
||||
} else {
|
||||
await vectorStoreDriver.fromDocuments(finalDocs)
|
||||
}
|
||||
|
||||
return { numAdded: finalDocs.length, addedDocs: finalDocs }
|
||||
}
|
||||
|
|
@ -303,11 +285,7 @@ class Postgres_VectorStores implements INode {
|
|||
const vectorStoreName = tableName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await vectorStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
|
|
|
|||
|
|
@ -5,11 +5,6 @@ import { TypeORMVectorStore, TypeORMVectorStoreArgs, TypeORMVectorStoreDocument
|
|||
import { VectorStore } from '@langchain/core/vectorstores'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { Pool } from 'pg'
|
||||
import { v4 as uuid } from 'uuid'
|
||||
|
||||
type TypeORMAddDocumentOptions = {
|
||||
ids?: string[]
|
||||
}
|
||||
|
||||
export class TypeORMDriver extends VectorStoreDriver {
|
||||
protected _postgresConnectionOptions: DataSourceOptions
|
||||
|
|
@ -100,45 +95,15 @@ export class TypeORMDriver extends VectorStoreDriver {
|
|||
try {
|
||||
instance.appDataSource.getRepository(instance.documentEntity).delete(ids)
|
||||
} catch (e) {
|
||||
console.error('Failed to delete', e)
|
||||
console.error('Failed to delete')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
instance.addVectors = async (
|
||||
vectors: number[][],
|
||||
documents: Document[],
|
||||
documentOptions?: TypeORMAddDocumentOptions
|
||||
): Promise<void> => {
|
||||
const rows = vectors.map((embedding, idx) => {
|
||||
const embeddingString = `[${embedding.join(',')}]`
|
||||
const documentRow = {
|
||||
id: documentOptions?.ids?.length ? documentOptions.ids[idx] : uuid(),
|
||||
pageContent: documents[idx].pageContent,
|
||||
embedding: embeddingString,
|
||||
metadata: documents[idx].metadata
|
||||
}
|
||||
return documentRow
|
||||
})
|
||||
const baseAddVectorsFn = instance.addVectors.bind(instance)
|
||||
|
||||
const documentRepository = instance.appDataSource.getRepository(instance.documentEntity)
|
||||
const _batchSize = this.nodeData.inputs?.batchSize
|
||||
const chunkSize = _batchSize ? parseInt(_batchSize, 10) : 500
|
||||
|
||||
for (let i = 0; i < rows.length; i += chunkSize) {
|
||||
const chunk = rows.slice(i, i + chunkSize)
|
||||
try {
|
||||
await documentRepository.save(chunk)
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
throw new Error(`Error inserting: ${chunk[0].pageContent}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
instance.addDocuments = async (documents: Document[], options?: { ids?: string[] }): Promise<void> => {
|
||||
const texts = documents.map(({ pageContent }) => pageContent)
|
||||
return (instance.addVectors as any)(await this.getEmbeddings().embedDocuments(texts), documents, options)
|
||||
instance.addVectors = async (vectors, documents) => {
|
||||
return baseAddVectorsFn(vectors, this.sanitizeDocuments(documents))
|
||||
}
|
||||
|
||||
return instance
|
||||
|
|
|
|||
|
|
@ -385,11 +385,7 @@ class Qdrant_VectorStores implements INode {
|
|||
const vectorStoreName = collectionName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await vectorStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
|
|
|
|||
|
|
@ -197,11 +197,7 @@ class Supabase_VectorStores implements INode {
|
|||
const vectorStoreName = tableName + '_' + queryName
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await supabaseStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
|
|
|
|||
|
|
@ -187,11 +187,7 @@ class Upstash_VectorStores implements INode {
|
|||
const vectorStoreName = UPSTASH_VECTOR_REST_URL
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await upstashStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
|
|
|
|||
|
|
@ -252,11 +252,7 @@ class Weaviate_VectorStores implements INode {
|
|||
const vectorStoreName = weaviateTextKey ? weaviateIndex + '_' + weaviateTextKey : weaviateIndex
|
||||
await recordManager.createSchema()
|
||||
;(recordManager as any).namespace = (recordManager as any).namespace + '_' + vectorStoreName
|
||||
const filterKeys: ICommonObject = {}
|
||||
if (options.docId) {
|
||||
filterKeys.docId = options.docId
|
||||
}
|
||||
const keys: string[] = await recordManager.listKeys(filterKeys)
|
||||
const keys: string[] = await recordManager.listKeys({})
|
||||
|
||||
await weaviateStore.delete({ ids: keys })
|
||||
await recordManager.deleteKeys(keys)
|
||||
|
|
|
|||
|
|
@ -8,10 +8,6 @@ import { IndexingResult } from './Interface'
|
|||
|
||||
type Metadata = Record<string, unknown>
|
||||
|
||||
export interface ExtendedRecordManagerInterface extends RecordManagerInterface {
|
||||
update(keys: Array<{ uid: string; docId: string }> | string[], updateOptions?: Record<string, any>): Promise<void>
|
||||
}
|
||||
|
||||
type StringOrDocFunc = string | ((doc: DocumentInterface) => string)
|
||||
|
||||
export interface HashedDocumentInterface extends DocumentInterface {
|
||||
|
|
@ -211,7 +207,7 @@ export const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {
|
|||
|
||||
interface IndexArgs {
|
||||
docsSource: BaseDocumentLoader | DocumentInterface[]
|
||||
recordManager: ExtendedRecordManagerInterface
|
||||
recordManager: RecordManagerInterface
|
||||
vectorStore: VectorStore
|
||||
options?: IndexOptions
|
||||
}
|
||||
|
|
@ -279,7 +275,7 @@ export async function index(args: IndexArgs): Promise<IndexingResult> {
|
|||
|
||||
const uids: string[] = []
|
||||
const docsToIndex: DocumentInterface[] = []
|
||||
const docsToUpdate: Array<{ uid: string; docId: string }> = []
|
||||
const docsToUpdate: string[] = []
|
||||
const seenDocs = new Set<string>()
|
||||
hashedDocs.forEach((hashedDoc, i) => {
|
||||
const docExists = batchExists[i]
|
||||
|
|
@ -287,7 +283,7 @@ export async function index(args: IndexArgs): Promise<IndexingResult> {
|
|||
if (forceUpdate) {
|
||||
seenDocs.add(hashedDoc.uid)
|
||||
} else {
|
||||
docsToUpdate.push({ uid: hashedDoc.uid, docId: hashedDoc.metadata.docId as string })
|
||||
docsToUpdate.push(hashedDoc.uid)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
@ -312,7 +308,7 @@ export async function index(args: IndexArgs): Promise<IndexingResult> {
|
|||
}
|
||||
|
||||
await recordManager.update(
|
||||
hashedDocs.map((doc) => ({ uid: doc.uid, docId: doc.metadata.docId as string })),
|
||||
hashedDocs.map((doc) => doc.uid),
|
||||
{ timeAtLeast: indexStartDt, groupIds: sourceIds }
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -465,10 +465,9 @@ const insertIntoVectorStore = async (req: Request, res: Response, next: NextFunc
|
|||
}
|
||||
const subscriptionId = req.user?.activeOrganizationSubscriptionId || ''
|
||||
const body = req.body
|
||||
const isStrictSave = body.isStrictSave ?? false
|
||||
const apiResponse = await documentStoreService.insertIntoVectorStoreMiddleware(
|
||||
body,
|
||||
isStrictSave,
|
||||
false,
|
||||
orgId,
|
||||
workspaceId,
|
||||
subscriptionId,
|
||||
|
|
@ -514,11 +513,7 @@ const deleteVectorStoreFromStore = async (req: Request, res: Response, next: Nex
|
|||
`Error: documentStoreController.deleteVectorStoreFromStore - workspaceId not provided!`
|
||||
)
|
||||
}
|
||||
const apiResponse = await documentStoreService.deleteVectorStoreFromStore(
|
||||
req.params.storeId,
|
||||
workspaceId,
|
||||
(req.query.docId as string) || undefined
|
||||
)
|
||||
const apiResponse = await documentStoreService.deleteVectorStoreFromStore(req.params.storeId, workspaceId)
|
||||
return res.json(apiResponse)
|
||||
} catch (error) {
|
||||
next(error)
|
||||
|
|
|
|||
|
|
@ -391,7 +391,7 @@ const deleteDocumentStoreFileChunk = async (storeId: string, docId: string, chun
|
|||
}
|
||||
}
|
||||
|
||||
const deleteVectorStoreFromStore = async (storeId: string, workspaceId: string, docId?: string) => {
|
||||
const deleteVectorStoreFromStore = async (storeId: string, workspaceId: string) => {
|
||||
try {
|
||||
const appServer = getRunningExpressApp()
|
||||
const componentNodes = appServer.nodesPool.componentNodes
|
||||
|
|
@ -461,7 +461,7 @@ const deleteVectorStoreFromStore = async (storeId: string, workspaceId: string,
|
|||
|
||||
// Call the delete method of the vector store
|
||||
if (vectorStoreObj.vectorStoreMethods.delete) {
|
||||
await vectorStoreObj.vectorStoreMethods.delete(vStoreNodeData, idsToDelete, { ...options, docId })
|
||||
await vectorStoreObj.vectorStoreMethods.delete(vStoreNodeData, idsToDelete, options)
|
||||
}
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
|
|
@ -1157,18 +1157,6 @@ const updateVectorStoreConfigOnly = async (data: ICommonObject, workspaceId: str
|
|||
)
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Saves vector store configuration to the document store entity.
|
||||
* Handles embedding, vector store, and record manager configurations.
|
||||
*
|
||||
* @example
|
||||
* // Strict mode: Only save what's provided, clear the rest
|
||||
* await saveVectorStoreConfig(ds, { storeId, embeddingName, embeddingConfig }, true, wsId)
|
||||
*
|
||||
* @example
|
||||
* // Lenient mode: Reuse existing configs if not provided
|
||||
* await saveVectorStoreConfig(ds, { storeId, vectorStoreName, vectorStoreConfig }, false, wsId)
|
||||
*/
|
||||
const saveVectorStoreConfig = async (appDataSource: DataSource, data: ICommonObject, isStrictSave = true, workspaceId: string) => {
|
||||
try {
|
||||
const entity = await appDataSource.getRepository(DocumentStore).findOneBy({
|
||||
|
|
@ -1233,15 +1221,6 @@ const saveVectorStoreConfig = async (appDataSource: DataSource, data: ICommonObj
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts documents from document store into the configured vector store.
|
||||
*
|
||||
* Process:
|
||||
* 1. Saves vector store configuration (embedding, vector store, record manager)
|
||||
* 2. Sets document store status to UPSERTING
|
||||
* 3. Performs the actual vector store upsert operation
|
||||
* 4. Updates status to UPSERTED upon completion
|
||||
*/
|
||||
export const insertIntoVectorStore = async ({
|
||||
appDataSource,
|
||||
componentNodes,
|
||||
|
|
@ -1252,16 +1231,19 @@ export const insertIntoVectorStore = async ({
|
|||
workspaceId
|
||||
}: IExecuteVectorStoreInsert) => {
|
||||
try {
|
||||
// Step 1: Save configuration based on isStrictSave mode
|
||||
const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave, workspaceId)
|
||||
|
||||
// Step 2: Mark as UPSERTING before starting the operation
|
||||
entity.status = DocumentStoreStatus.UPSERTING
|
||||
await appDataSource.getRepository(DocumentStore).save(entity)
|
||||
|
||||
// Step 3: Perform the actual vector store upsert
|
||||
// Note: Configuration already saved above, worker thread just retrieves and uses it
|
||||
const indexResult = await _insertIntoVectorStoreWorkerThread(appDataSource, componentNodes, telemetry, data, orgId, workspaceId)
|
||||
const indexResult = await _insertIntoVectorStoreWorkerThread(
|
||||
appDataSource,
|
||||
componentNodes,
|
||||
telemetry,
|
||||
data,
|
||||
isStrictSave,
|
||||
orgId,
|
||||
workspaceId
|
||||
)
|
||||
return indexResult
|
||||
} catch (error) {
|
||||
throw new InternalFlowiseError(
|
||||
|
|
@ -1326,18 +1308,12 @@ const _insertIntoVectorStoreWorkerThread = async (
|
|||
componentNodes: IComponentNodes,
|
||||
telemetry: Telemetry,
|
||||
data: ICommonObject,
|
||||
isStrictSave = true,
|
||||
orgId: string,
|
||||
workspaceId: string
|
||||
) => {
|
||||
try {
|
||||
// Configuration already saved by insertIntoVectorStore, just retrieve the entity
|
||||
const entity = await appDataSource.getRepository(DocumentStore).findOneBy({
|
||||
id: data.storeId,
|
||||
workspaceId: workspaceId
|
||||
})
|
||||
if (!entity) {
|
||||
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Document store ${data.storeId} not found`)
|
||||
}
|
||||
const entity = await saveVectorStoreConfig(appDataSource, data, isStrictSave, workspaceId)
|
||||
let upsertHistory: Record<string, any> = {}
|
||||
const chatflowid = data.storeId // fake chatflowid because this is not tied to any chatflow
|
||||
|
||||
|
|
@ -1374,10 +1350,7 @@ const _insertIntoVectorStoreWorkerThread = async (
|
|||
const docs: Document[] = chunks.map((chunk: DocumentStoreFileChunk) => {
|
||||
return new Document({
|
||||
pageContent: chunk.pageContent,
|
||||
metadata: {
|
||||
...JSON.parse(chunk.metadata),
|
||||
docId: chunk.docId
|
||||
}
|
||||
metadata: JSON.parse(chunk.metadata)
|
||||
})
|
||||
})
|
||||
vStoreNodeData.inputs.document = docs
|
||||
|
|
@ -1938,8 +1911,6 @@ const upsertDocStore = async (
|
|||
recordManagerConfig
|
||||
}
|
||||
|
||||
// Use isStrictSave: false to preserve existing configurations during upsert
|
||||
// This allows the operation to reuse existing embedding/vector store/record manager configs
|
||||
const res = await insertIntoVectorStore({
|
||||
appDataSource,
|
||||
componentNodes,
|
||||
|
|
|
|||
|
|
@ -22,10 +22,7 @@ const refreshLoader = (storeId) => client.post(`/document-store/refresh/${storeI
|
|||
const insertIntoVectorStore = (body) => client.post(`/document-store/vectorstore/insert`, body)
|
||||
const saveVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/save`, body)
|
||||
const updateVectorStoreConfig = (body) => client.post(`/document-store/vectorstore/update`, body)
|
||||
const deleteVectorStoreDataFromStore = (storeId, docId) => {
|
||||
const url = docId ? `/document-store/vectorstore/${storeId}?docId=${docId}` : `/document-store/vectorstore/${storeId}`
|
||||
return client.delete(url)
|
||||
}
|
||||
const deleteVectorStoreDataFromStore = (storeId) => client.delete(`/document-store/vectorstore/${storeId}`)
|
||||
const queryVectorStore = (body) => client.post(`/document-store/vectorstore/query`, body)
|
||||
const getVectorStoreProviders = () => client.get('/document-store/components/vectorstore')
|
||||
const getEmbeddingProviders = () => client.get('/document-store/components/embeddings')
|
||||
|
|
|
|||
|
|
@ -18,15 +18,11 @@ import {
|
|||
TableContainer,
|
||||
TableRow,
|
||||
TableCell,
|
||||
DialogActions,
|
||||
Card,
|
||||
Stack,
|
||||
Link
|
||||
Checkbox,
|
||||
FormControlLabel,
|
||||
DialogActions
|
||||
} from '@mui/material'
|
||||
import { useTheme } from '@mui/material/styles'
|
||||
import ExpandMoreIcon from '@mui/icons-material/ExpandMore'
|
||||
import SettingsIcon from '@mui/icons-material/Settings'
|
||||
import { IconAlertTriangle } from '@tabler/icons-react'
|
||||
import { TableViewOnly } from '@/ui-component/table/Table'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
|
||||
|
|
@ -40,13 +36,12 @@ import { initNode } from '@/utils/genericHelper'
|
|||
|
||||
const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
||||
const portalElement = document.getElementById('portal')
|
||||
const theme = useTheme()
|
||||
const [nodeConfigExpanded, setNodeConfigExpanded] = useState({})
|
||||
const [removeFromVS, setRemoveFromVS] = useState(false)
|
||||
const [vsFlowData, setVSFlowData] = useState([])
|
||||
const [rmFlowData, setRMFlowData] = useState([])
|
||||
|
||||
const getVectorStoreNodeApi = useApi(nodesApi.getSpecificNode)
|
||||
const getRecordManagerNodeApi = useApi(nodesApi.getSpecificNode)
|
||||
const getSpecificNodeApi = useApi(nodesApi.getSpecificNode)
|
||||
|
||||
const handleAccordionChange = (nodeName) => (event, isExpanded) => {
|
||||
const accordianNodes = { ...nodeConfigExpanded }
|
||||
|
|
@ -57,37 +52,42 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
useEffect(() => {
|
||||
if (dialogProps.recordManagerConfig) {
|
||||
const nodeName = dialogProps.recordManagerConfig.name
|
||||
if (nodeName) getRecordManagerNodeApi.request(nodeName)
|
||||
}
|
||||
if (nodeName) getSpecificNodeApi.request(nodeName)
|
||||
|
||||
if (dialogProps.vectorStoreConfig) {
|
||||
const nodeName = dialogProps.vectorStoreConfig.name
|
||||
if (nodeName) getVectorStoreNodeApi.request(nodeName)
|
||||
if (nodeName) getSpecificNodeApi.request(nodeName)
|
||||
}
|
||||
}
|
||||
|
||||
return () => {
|
||||
setNodeConfigExpanded({})
|
||||
setRemoveFromVS(false)
|
||||
setVSFlowData([])
|
||||
setRMFlowData([])
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [dialogProps])
|
||||
|
||||
// Process Vector Store node data
|
||||
useEffect(() => {
|
||||
if (getVectorStoreNodeApi.data && dialogProps.vectorStoreConfig) {
|
||||
const nodeData = cloneDeep(initNode(getVectorStoreNodeApi.data, uuidv4()))
|
||||
if (getSpecificNodeApi.data) {
|
||||
const nodeData = cloneDeep(initNode(getSpecificNodeApi.data, uuidv4()))
|
||||
|
||||
let config = 'vectorStoreConfig'
|
||||
if (nodeData.category === 'Record Manager') config = 'recordManagerConfig'
|
||||
|
||||
const paramValues = []
|
||||
|
||||
for (const inputName in dialogProps.vectorStoreConfig.config) {
|
||||
for (const inputName in dialogProps[config].config) {
|
||||
const inputParam = nodeData.inputParams.find((inp) => inp.name === inputName)
|
||||
|
||||
if (!inputParam) continue
|
||||
|
||||
if (inputParam.type === 'credential') continue
|
||||
|
||||
const inputValue = dialogProps.vectorStoreConfig.config[inputName]
|
||||
let paramValue = {}
|
||||
|
||||
const inputValue = dialogProps[config].config[inputName]
|
||||
|
||||
if (!inputValue) continue
|
||||
|
||||
|
|
@ -95,14 +95,16 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
continue
|
||||
}
|
||||
|
||||
paramValues.push({
|
||||
paramValue = {
|
||||
label: inputParam?.label,
|
||||
name: inputParam?.name,
|
||||
type: inputParam?.type,
|
||||
value: inputValue
|
||||
})
|
||||
}
|
||||
paramValues.push(paramValue)
|
||||
}
|
||||
|
||||
if (config === 'vectorStoreConfig') {
|
||||
setVSFlowData([
|
||||
{
|
||||
label: nodeData.label,
|
||||
|
|
@ -112,41 +114,7 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
paramValues
|
||||
}
|
||||
])
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getVectorStoreNodeApi.data])
|
||||
|
||||
// Process Record Manager node data
|
||||
useEffect(() => {
|
||||
if (getRecordManagerNodeApi.data && dialogProps.recordManagerConfig) {
|
||||
const nodeData = cloneDeep(initNode(getRecordManagerNodeApi.data, uuidv4()))
|
||||
|
||||
const paramValues = []
|
||||
|
||||
for (const inputName in dialogProps.recordManagerConfig.config) {
|
||||
const inputParam = nodeData.inputParams.find((inp) => inp.name === inputName)
|
||||
|
||||
if (!inputParam) continue
|
||||
|
||||
if (inputParam.type === 'credential') continue
|
||||
|
||||
const inputValue = dialogProps.recordManagerConfig.config[inputName]
|
||||
|
||||
if (!inputValue) continue
|
||||
|
||||
if (typeof inputValue === 'string' && inputValue.startsWith('{{') && inputValue.endsWith('}}')) {
|
||||
continue
|
||||
}
|
||||
|
||||
paramValues.push({
|
||||
label: inputParam?.label,
|
||||
name: inputParam?.name,
|
||||
type: inputParam?.type,
|
||||
value: inputValue
|
||||
})
|
||||
}
|
||||
|
||||
} else if (config === 'recordManagerConfig') {
|
||||
setRMFlowData([
|
||||
{
|
||||
label: nodeData.label,
|
||||
|
|
@ -157,9 +125,10 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
}
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getRecordManagerNodeApi.data])
|
||||
}, [getSpecificNodeApi.data])
|
||||
|
||||
const component = show ? (
|
||||
<Dialog
|
||||
|
|
@ -173,54 +142,17 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
<DialogTitle sx={{ fontSize: '1rem', p: 3, pb: 0 }} id='alert-dialog-title'>
|
||||
{dialogProps.title}
|
||||
</DialogTitle>
|
||||
<DialogContent
|
||||
sx={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: 2,
|
||||
maxHeight: '75vh',
|
||||
position: 'relative',
|
||||
px: 3,
|
||||
pb: 3,
|
||||
overflow: 'auto'
|
||||
}}
|
||||
>
|
||||
<DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxHeight: '75vh', position: 'relative', px: 3, pb: 3 }}>
|
||||
<span style={{ marginTop: '20px' }}>{dialogProps.description}</span>
|
||||
{dialogProps.vectorStoreConfig && !dialogProps.recordManagerConfig && (
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'row',
|
||||
alignItems: 'center',
|
||||
borderRadius: 10,
|
||||
background: 'rgb(254,252,191)',
|
||||
padding: 10
|
||||
}}
|
||||
>
|
||||
<IconAlertTriangle size={70} color='orange' />
|
||||
<span style={{ color: 'rgb(116,66,16)', marginLeft: 10 }}>
|
||||
<strong>Note:</strong> Without a Record Manager configured, only the document chunks will be removed from the
|
||||
document store. The actual vector embeddings in your vector store database will remain unchanged. To enable
|
||||
automatic cleanup of vector store data, please configure a Record Manager.{' '}
|
||||
<Link
|
||||
href='https://docs.flowiseai.com/integrations/langchain/record-managers'
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
sx={{ fontWeight: 500, color: 'rgb(116,66,16)', textDecoration: 'underline' }}
|
||||
>
|
||||
Learn more
|
||||
</Link>
|
||||
</span>
|
||||
</div>
|
||||
{dialogProps.type === 'STORE' && dialogProps.recordManagerConfig && (
|
||||
<FormControlLabel
|
||||
control={<Checkbox checked={removeFromVS} onChange={(event) => setRemoveFromVS(event.target.checked)} />}
|
||||
label='Remove data from vector store and record manager'
|
||||
/>
|
||||
)}
|
||||
{vsFlowData && vsFlowData.length > 0 && rmFlowData && rmFlowData.length > 0 && (
|
||||
<Card sx={{ borderColor: theme.palette.primary[200] + 75, p: 2 }} variant='outlined'>
|
||||
<Stack sx={{ mt: 1, mb: 2, ml: 1, alignItems: 'center' }} direction='row' spacing={2}>
|
||||
<SettingsIcon />
|
||||
<Typography variant='h4'>Configuration</Typography>
|
||||
</Stack>
|
||||
<Stack direction='column'>
|
||||
<TableContainer component={Paper} sx={{ maxHeight: '400px', overflow: 'auto' }}>
|
||||
{removeFromVS && (
|
||||
<div>
|
||||
<TableContainer component={Paper}>
|
||||
<Table sx={{ minWidth: 650 }} aria-label='simple table'>
|
||||
<TableBody>
|
||||
<TableRow sx={{ '& td': { border: 0 } }}>
|
||||
|
|
@ -229,7 +161,7 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
{([...vsFlowData, ...rmFlowData] || []).map((node, index) => {
|
||||
return (
|
||||
<Accordion
|
||||
expanded={nodeConfigExpanded[node.name] || false}
|
||||
expanded={nodeConfigExpanded[node.name] || true}
|
||||
onChange={handleAccordionChange(node.name)}
|
||||
key={index}
|
||||
disableGutters
|
||||
|
|
@ -240,11 +172,7 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
id={`nodes-accordian-header-${node.name}`}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'row',
|
||||
alignItems: 'center'
|
||||
}}
|
||||
style={{ display: 'flex', flexDirection: 'row', alignItems: 'center' }}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
|
|
@ -270,7 +198,7 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
<Typography variant='h5'>{node.label}</Typography>
|
||||
</div>
|
||||
</AccordionSummary>
|
||||
<AccordionDetails sx={{ p: 0 }}>
|
||||
<AccordionDetails>
|
||||
{node.paramValues[0] && (
|
||||
<TableViewOnly
|
||||
sx={{ minWidth: 150 }}
|
||||
|
|
@ -288,15 +216,17 @@ const DeleteDocStoreDialog = ({ show, dialogProps, onCancel, onDelete }) => {
|
|||
</TableBody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
</Stack>
|
||||
</Card>
|
||||
<span style={{ marginTop: '30px', fontStyle: 'italic', color: '#b35702' }}>
|
||||
* Only data that were upserted with Record Manager will be deleted from vector store
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</DialogContent>
|
||||
<DialogActions sx={{ pr: 3, pb: 3 }}>
|
||||
<Button onClick={onCancel} color='primary'>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant='contained' onClick={() => onDelete(dialogProps.type, dialogProps.file)} color='error'>
|
||||
<Button variant='contained' onClick={() => onDelete(dialogProps.type, dialogProps.file, removeFromVS)} color='error'>
|
||||
Delete
|
||||
</Button>
|
||||
</DialogActions>
|
||||
|
|
|
|||
|
|
@ -186,19 +186,19 @@ const DocumentStoreDetails = () => {
|
|||
setShowDocumentLoaderListDialog(true)
|
||||
}
|
||||
|
||||
const deleteVectorStoreDataFromStore = async (storeId, docId) => {
|
||||
const deleteVectorStoreDataFromStore = async (storeId) => {
|
||||
try {
|
||||
await documentsApi.deleteVectorStoreDataFromStore(storeId, docId)
|
||||
await documentsApi.deleteVectorStoreDataFromStore(storeId)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
}
|
||||
}
|
||||
|
||||
const onDocStoreDelete = async (type, file) => {
|
||||
const onDocStoreDelete = async (type, file, removeFromVectorStore) => {
|
||||
setBackdropLoading(true)
|
||||
setShowDeleteDocStoreDialog(false)
|
||||
if (type === 'STORE') {
|
||||
if (documentStore.recordManagerConfig) {
|
||||
if (removeFromVectorStore) {
|
||||
await deleteVectorStoreDataFromStore(storeId)
|
||||
}
|
||||
try {
|
||||
|
|
@ -239,9 +239,6 @@ const DocumentStoreDetails = () => {
|
|||
})
|
||||
}
|
||||
} else if (type === 'LOADER') {
|
||||
if (documentStore.recordManagerConfig) {
|
||||
await deleteVectorStoreDataFromStore(storeId, file.id)
|
||||
}
|
||||
try {
|
||||
const deleteResp = await documentsApi.deleteLoaderFromStore(storeId, file.id)
|
||||
setBackdropLoading(false)
|
||||
|
|
@ -283,40 +280,9 @@ const DocumentStoreDetails = () => {
|
|||
}
|
||||
|
||||
const onLoaderDelete = (file, vectorStoreConfig, recordManagerConfig) => {
|
||||
// Get the display name in the format "LoaderName (sourceName)"
|
||||
const loaderName = file.loaderName || 'Unknown'
|
||||
let sourceName = ''
|
||||
|
||||
// Prefer files.name when files array exists and has items
|
||||
if (file.files && Array.isArray(file.files) && file.files.length > 0) {
|
||||
sourceName = file.files.map((f) => f.name).join(', ')
|
||||
} else if (file.source) {
|
||||
// Fallback to source logic
|
||||
if (typeof file.source === 'string' && file.source.includes('base64')) {
|
||||
sourceName = getFileName(file.source)
|
||||
} else if (typeof file.source === 'string' && file.source.startsWith('[') && file.source.endsWith(']')) {
|
||||
sourceName = JSON.parse(file.source).join(', ')
|
||||
} else if (typeof file.source === 'string') {
|
||||
sourceName = file.source
|
||||
}
|
||||
}
|
||||
|
||||
const displayName = sourceName ? `${loaderName} (${sourceName})` : loaderName
|
||||
|
||||
let description = `Delete "${displayName}"? This will delete all the associated document chunks from the document store.`
|
||||
|
||||
if (
|
||||
recordManagerConfig &&
|
||||
vectorStoreConfig &&
|
||||
Object.keys(recordManagerConfig).length > 0 &&
|
||||
Object.keys(vectorStoreConfig).length > 0
|
||||
) {
|
||||
description = `Delete "${displayName}"? This will delete all the associated document chunks from the document store and remove the actual data from the vector store database.`
|
||||
}
|
||||
|
||||
const props = {
|
||||
title: `Delete`,
|
||||
description,
|
||||
description: `Delete Loader ${file.loaderName} ? This will delete all the associated document chunks.`,
|
||||
vectorStoreConfig,
|
||||
recordManagerConfig,
|
||||
type: 'LOADER',
|
||||
|
|
@ -328,20 +294,9 @@ const DocumentStoreDetails = () => {
|
|||
}
|
||||
|
||||
const onStoreDelete = (vectorStoreConfig, recordManagerConfig) => {
|
||||
let description = `Delete Store ${getSpecificDocumentStore.data?.name}? This will delete all the associated loaders and document chunks from the document store.`
|
||||
|
||||
if (
|
||||
recordManagerConfig &&
|
||||
vectorStoreConfig &&
|
||||
Object.keys(recordManagerConfig).length > 0 &&
|
||||
Object.keys(vectorStoreConfig).length > 0
|
||||
) {
|
||||
description = `Delete Store ${getSpecificDocumentStore.data?.name}? This will delete all the associated loaders and document chunks from the document store, and remove the actual data from the vector store database.`
|
||||
}
|
||||
|
||||
const props = {
|
||||
title: `Delete`,
|
||||
description,
|
||||
description: `Delete Store ${getSpecificDocumentStore.data?.name} ? This will delete all the associated loaders and document chunks.`,
|
||||
vectorStoreConfig,
|
||||
recordManagerConfig,
|
||||
type: 'STORE'
|
||||
|
|
@ -526,10 +481,7 @@ const DocumentStoreDetails = () => {
|
|||
>
|
||||
<MenuItem
|
||||
disabled={documentStore?.totalChunks <= 0 || documentStore?.status === 'UPSERTING'}
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
showStoredChunks('all')
|
||||
}}
|
||||
onClick={() => showStoredChunks('all')}
|
||||
disableRipple
|
||||
>
|
||||
<FileChunksIcon />
|
||||
|
|
@ -538,10 +490,7 @@ const DocumentStoreDetails = () => {
|
|||
<Available permission={'documentStores:upsert-config'}>
|
||||
<MenuItem
|
||||
disabled={documentStore?.totalChunks <= 0 || documentStore?.status === 'UPSERTING'}
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
showVectorStore(documentStore.id)
|
||||
}}
|
||||
onClick={() => showVectorStore(documentStore.id)}
|
||||
disableRipple
|
||||
>
|
||||
<NoteAddIcon />
|
||||
|
|
@ -550,10 +499,7 @@ const DocumentStoreDetails = () => {
|
|||
</Available>
|
||||
<MenuItem
|
||||
disabled={documentStore?.totalChunks <= 0 || documentStore?.status !== 'UPSERTED'}
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
showVectorStoreQuery(documentStore.id)
|
||||
}}
|
||||
onClick={() => showVectorStoreQuery(documentStore.id)}
|
||||
disableRipple
|
||||
>
|
||||
<SearchIcon />
|
||||
|
|
@ -572,10 +518,7 @@ const DocumentStoreDetails = () => {
|
|||
</Available>
|
||||
<Divider sx={{ my: 0.5 }} />
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
onStoreDelete(documentStore.vectorStoreConfig, documentStore.recordManagerConfig)
|
||||
}}
|
||||
onClick={() => onStoreDelete(documentStore.vectorStoreConfig, documentStore.recordManagerConfig)}
|
||||
disableRipple
|
||||
>
|
||||
<FileDeleteIcon />
|
||||
|
|
@ -813,26 +756,20 @@ function LoaderRow(props) {
|
|||
setAnchorEl(null)
|
||||
}
|
||||
|
||||
const formatSources = (files, source, loaderName) => {
|
||||
let sourceName = ''
|
||||
|
||||
const formatSources = (files, source) => {
|
||||
// Prefer files.name when files array exists and has items
|
||||
if (files && Array.isArray(files) && files.length > 0) {
|
||||
sourceName = files.map((file) => file.name).join(', ')
|
||||
} else if (source && typeof source === 'string' && source.includes('base64')) {
|
||||
// Fallback to original source logic
|
||||
sourceName = getFileName(source)
|
||||
} else if (source && typeof source === 'string' && source.startsWith('[') && source.endsWith(']')) {
|
||||
sourceName = JSON.parse(source).join(', ')
|
||||
} else if (source) {
|
||||
sourceName = source
|
||||
return files.map((file) => file.name).join(', ')
|
||||
}
|
||||
|
||||
// Return format: "LoaderName (sourceName)" or just "LoaderName" if no source
|
||||
if (!sourceName) {
|
||||
return loaderName || 'No source'
|
||||
// Fallback to original source logic
|
||||
if (source && typeof source === 'string' && source.includes('base64')) {
|
||||
return getFileName(source)
|
||||
}
|
||||
return loaderName ? `${loaderName} (${sourceName})` : sourceName
|
||||
if (source && typeof source === 'string' && source.startsWith('[') && source.endsWith(']')) {
|
||||
return JSON.parse(source).join(', ')
|
||||
}
|
||||
return source || 'No source'
|
||||
}
|
||||
|
||||
return (
|
||||
|
|
@ -886,62 +823,32 @@ function LoaderRow(props) {
|
|||
onClose={handleClose}
|
||||
>
|
||||
<Available permission={'documentStores:preview-process'}>
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
props.onEditClick()
|
||||
}}
|
||||
disableRipple
|
||||
>
|
||||
<MenuItem onClick={props.onEditClick} disableRipple>
|
||||
<FileEditIcon />
|
||||
Preview & Process
|
||||
</MenuItem>
|
||||
</Available>
|
||||
<Available permission={'documentStores:preview-process'}>
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
props.onViewChunksClick()
|
||||
}}
|
||||
disableRipple
|
||||
>
|
||||
<MenuItem onClick={props.onViewChunksClick} disableRipple>
|
||||
<FileChunksIcon />
|
||||
View & Edit Chunks
|
||||
</MenuItem>
|
||||
</Available>
|
||||
<Available permission={'documentStores:preview-process'}>
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
props.onChunkUpsert()
|
||||
}}
|
||||
disableRipple
|
||||
>
|
||||
<MenuItem onClick={props.onChunkUpsert} disableRipple>
|
||||
<NoteAddIcon />
|
||||
Upsert Chunks
|
||||
</MenuItem>
|
||||
</Available>
|
||||
<Available permission={'documentStores:preview-process'}>
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
props.onViewUpsertAPI()
|
||||
}}
|
||||
disableRipple
|
||||
>
|
||||
<MenuItem onClick={props.onViewUpsertAPI} disableRipple>
|
||||
<CodeIcon />
|
||||
View API
|
||||
</MenuItem>
|
||||
</Available>
|
||||
<Divider sx={{ my: 0.5 }} />
|
||||
<Available permission={'documentStores:delete-loader'}>
|
||||
<MenuItem
|
||||
onClick={() => {
|
||||
handleClose()
|
||||
props.onDeleteClick()
|
||||
}}
|
||||
disableRipple
|
||||
>
|
||||
<MenuItem onClick={props.onDeleteClick} disableRipple>
|
||||
<FileDeleteIcon />
|
||||
Delete
|
||||
</MenuItem>
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import useApi from '@/hooks/useApi'
|
|||
import useConfirm from '@/hooks/useConfirm'
|
||||
import useNotifier from '@/utils/useNotifier'
|
||||
import { useAuth } from '@/hooks/useAuth'
|
||||
import { getFileName } from '@/utils/genericHelper'
|
||||
|
||||
// store
|
||||
import { closeSnackbar as closeSnackbarAction, enqueueSnackbar as enqueueSnackbarAction } from '@/store/actions'
|
||||
|
|
@ -77,7 +76,6 @@ const ShowStoredChunks = () => {
|
|||
const [showExpandedChunkDialog, setShowExpandedChunkDialog] = useState(false)
|
||||
const [expandedChunkDialogProps, setExpandedChunkDialogProps] = useState({})
|
||||
const [fileNames, setFileNames] = useState([])
|
||||
const [loaderDisplayName, setLoaderDisplayName] = useState('')
|
||||
|
||||
const chunkSelected = (chunkId) => {
|
||||
const selectedChunk = documentChunks.find((chunk) => chunk.id === chunkId)
|
||||
|
|
@ -214,34 +212,15 @@ const ShowStoredChunks = () => {
|
|||
setCurrentPage(data.currentPage)
|
||||
setStart(data.currentPage * 50 - 49)
|
||||
setEnd(data.currentPage * 50 > data.count ? data.count : data.currentPage * 50)
|
||||
|
||||
// Build the loader display name in format "LoaderName (sourceName)"
|
||||
const loaderName = data.file?.loaderName || data.storeName || ''
|
||||
let sourceName = ''
|
||||
|
||||
if (data.file?.files && data.file.files.length > 0) {
|
||||
const fileNames = []
|
||||
for (const attachedFile of data.file.files) {
|
||||
fileNames.push(attachedFile.name)
|
||||
}
|
||||
setFileNames(fileNames)
|
||||
sourceName = fileNames.join(', ')
|
||||
} else if (data.file?.source) {
|
||||
const source = data.file.source
|
||||
if (typeof source === 'string' && source.includes('base64')) {
|
||||
sourceName = getFileName(source)
|
||||
} else if (typeof source === 'string' && source.startsWith('[') && source.endsWith(']')) {
|
||||
sourceName = JSON.parse(source).join(', ')
|
||||
} else if (typeof source === 'string') {
|
||||
sourceName = source
|
||||
}
|
||||
}
|
||||
|
||||
// Set display name in format "LoaderName (sourceName)" or just "LoaderName"
|
||||
const displayName = sourceName ? `${loaderName} (${sourceName})` : loaderName
|
||||
setLoaderDisplayName(displayName)
|
||||
}
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [getChunksApi.data])
|
||||
|
||||
|
|
@ -255,7 +234,7 @@ const ShowStoredChunks = () => {
|
|||
<ViewHeader
|
||||
isBackButton={true}
|
||||
search={false}
|
||||
title={loaderDisplayName}
|
||||
title={getChunksApi.data?.file?.loaderName || getChunksApi.data?.storeName}
|
||||
description={getChunksApi.data?.file?.splitterName || getChunksApi.data?.description}
|
||||
onBack={() => navigate(-1)}
|
||||
></ViewHeader>
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ import Storage from '@mui/icons-material/Storage'
|
|||
import DynamicFeed from '@mui/icons-material/Filter1'
|
||||
|
||||
// utils
|
||||
import { initNode, showHideInputParams, getFileName } from '@/utils/genericHelper'
|
||||
import { initNode, showHideInputParams } from '@/utils/genericHelper'
|
||||
import useNotifier from '@/utils/useNotifier'
|
||||
|
||||
// const
|
||||
|
|
@ -69,7 +69,6 @@ const VectorStoreConfigure = () => {
|
|||
const [loading, setLoading] = useState(true)
|
||||
const [documentStore, setDocumentStore] = useState({})
|
||||
const [dialogProps, setDialogProps] = useState({})
|
||||
const [currentLoader, setCurrentLoader] = useState(null)
|
||||
|
||||
const [showEmbeddingsListDialog, setShowEmbeddingsListDialog] = useState(false)
|
||||
const [selectedEmbeddingsProvider, setSelectedEmbeddingsProvider] = useState({})
|
||||
|
|
@ -246,8 +245,7 @@ const VectorStoreConfigure = () => {
|
|||
const prepareConfigData = () => {
|
||||
const data = {
|
||||
storeId: storeId,
|
||||
docId: docId,
|
||||
isStrictSave: true
|
||||
docId: docId
|
||||
}
|
||||
// Set embedding config
|
||||
if (selectedEmbeddingsProvider.inputs) {
|
||||
|
|
@ -355,39 +353,6 @@ const VectorStoreConfigure = () => {
|
|||
return Object.keys(selectedEmbeddingsProvider).length === 0
|
||||
}
|
||||
|
||||
const getLoaderDisplayName = (loader) => {
|
||||
if (!loader) return ''
|
||||
|
||||
const loaderName = loader.loaderName || 'Unknown'
|
||||
let sourceName = ''
|
||||
|
||||
// Prefer files.name when files array exists and has items
|
||||
if (loader.files && Array.isArray(loader.files) && loader.files.length > 0) {
|
||||
sourceName = loader.files.map((file) => file.name).join(', ')
|
||||
} else if (loader.source) {
|
||||
// Fallback to source logic
|
||||
if (typeof loader.source === 'string' && loader.source.includes('base64')) {
|
||||
sourceName = getFileName(loader.source)
|
||||
} else if (typeof loader.source === 'string' && loader.source.startsWith('[') && loader.source.endsWith(']')) {
|
||||
sourceName = JSON.parse(loader.source).join(', ')
|
||||
} else if (typeof loader.source === 'string') {
|
||||
sourceName = loader.source
|
||||
}
|
||||
}
|
||||
|
||||
// Return format: "LoaderName (sourceName)" or just "LoaderName" if no source
|
||||
return sourceName ? `${loaderName} (${sourceName})` : loaderName
|
||||
}
|
||||
|
||||
const getViewHeaderTitle = () => {
|
||||
const storeName = getSpecificDocumentStoreApi.data?.name || ''
|
||||
if (docId && currentLoader) {
|
||||
const loaderName = getLoaderDisplayName(currentLoader)
|
||||
return `${storeName} / ${loaderName}`
|
||||
}
|
||||
return storeName
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (saveVectorStoreConfigApi.data) {
|
||||
setLoading(false)
|
||||
|
|
@ -446,15 +411,6 @@ const VectorStoreConfigure = () => {
|
|||
return
|
||||
}
|
||||
setDocumentStore(docStore)
|
||||
|
||||
// Find the current loader if docId is provided
|
||||
if (docId && docStore.loaders) {
|
||||
const loader = docStore.loaders.find((l) => l.id === docId)
|
||||
if (loader) {
|
||||
setCurrentLoader(loader)
|
||||
}
|
||||
}
|
||||
|
||||
if (docStore.embeddingConfig) {
|
||||
getEmbeddingNodeDetailsApi.request(docStore.embeddingConfig.name)
|
||||
}
|
||||
|
|
@ -517,7 +473,7 @@ const VectorStoreConfigure = () => {
|
|||
<ViewHeader
|
||||
isBackButton={true}
|
||||
search={false}
|
||||
title={getViewHeaderTitle()}
|
||||
title={getSpecificDocumentStoreApi.data?.name}
|
||||
description='Configure Embeddings, Vector Store and Record Manager'
|
||||
onBack={() => navigate(-1)}
|
||||
>
|
||||
|
|
|
|||
Loading…
Reference in New Issue