Bugfix/Prevent open connections on typeorm datasource (#3652)
prevent open connections on typeorm datasource
This commit is contained in:
parent
680fe8dee1
commit
ddca80d4e0
|
|
@ -21,6 +21,13 @@ export class MySQLSaver extends BaseCheckpointSaver implements MemoryMethods {
|
||||||
|
|
||||||
private async getDataSource(): Promise<DataSource> {
|
private async getDataSource(): Promise<DataSource> {
|
||||||
const { datasourceOptions } = this.config
|
const { datasourceOptions } = this.config
|
||||||
|
if (!datasourceOptions) {
|
||||||
|
throw new Error('No datasource options provided')
|
||||||
|
}
|
||||||
|
// Prevent using default Postgres port, otherwise will throw uncaught error and crashing the app
|
||||||
|
if (datasourceOptions.port === 5432) {
|
||||||
|
throw new Error('Invalid port number')
|
||||||
|
}
|
||||||
const dataSource = new DataSource(datasourceOptions)
|
const dataSource = new DataSource(datasourceOptions)
|
||||||
await dataSource.initialize()
|
await dataSource.initialize()
|
||||||
return dataSource
|
return dataSource
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,13 @@ export class PostgresSaver extends BaseCheckpointSaver implements MemoryMethods
|
||||||
|
|
||||||
private async getDataSource(): Promise<DataSource> {
|
private async getDataSource(): Promise<DataSource> {
|
||||||
const { datasourceOptions } = this.config
|
const { datasourceOptions } = this.config
|
||||||
|
if (!datasourceOptions) {
|
||||||
|
throw new Error('No datasource options provided')
|
||||||
|
}
|
||||||
|
// Prevent using default MySQL port, otherwise will throw uncaught error and crashing the app
|
||||||
|
if (datasourceOptions.port === 3006) {
|
||||||
|
throw new Error('Invalid port number')
|
||||||
|
}
|
||||||
const dataSource = new DataSource(datasourceOptions)
|
const dataSource = new DataSource(datasourceOptions)
|
||||||
await dataSource.initialize()
|
await dataSource.initialize()
|
||||||
return dataSource
|
return dataSource
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||||
import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
|
import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
|
||||||
import { DataSource, QueryRunner } from 'typeorm'
|
import { DataSource } from 'typeorm'
|
||||||
|
|
||||||
class MySQLRecordManager_RecordManager implements INode {
|
class MySQLRecordManager_RecordManager implements INode {
|
||||||
label: string
|
label: string
|
||||||
|
|
@ -167,29 +167,37 @@ type MySQLRecordManagerOptions = {
|
||||||
|
|
||||||
class MySQLRecordManager implements RecordManagerInterface {
|
class MySQLRecordManager implements RecordManagerInterface {
|
||||||
lc_namespace = ['langchain', 'recordmanagers', 'mysql']
|
lc_namespace = ['langchain', 'recordmanagers', 'mysql']
|
||||||
|
config: MySQLRecordManagerOptions
|
||||||
datasource: DataSource
|
|
||||||
|
|
||||||
queryRunner: QueryRunner
|
|
||||||
|
|
||||||
tableName: string
|
tableName: string
|
||||||
|
|
||||||
namespace: string
|
namespace: string
|
||||||
|
|
||||||
constructor(namespace: string, config: MySQLRecordManagerOptions) {
|
constructor(namespace: string, config: MySQLRecordManagerOptions) {
|
||||||
const { mysqlOptions, tableName } = config
|
const { tableName } = config
|
||||||
this.namespace = namespace
|
this.namespace = namespace
|
||||||
this.tableName = tableName || 'upsertion_records'
|
this.tableName = tableName || 'upsertion_records'
|
||||||
this.datasource = new DataSource(mysqlOptions)
|
this.config = config
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getDataSource(): Promise<DataSource> {
|
||||||
|
const { mysqlOptions } = this.config
|
||||||
|
if (!mysqlOptions) {
|
||||||
|
throw new Error('No datasource options provided')
|
||||||
|
}
|
||||||
|
// Prevent using default Postgres port, otherwise will throw uncaught error and crashing the app
|
||||||
|
if (mysqlOptions.port === 5432) {
|
||||||
|
throw new Error('Invalid port number')
|
||||||
|
}
|
||||||
|
const dataSource = new DataSource(mysqlOptions)
|
||||||
|
await dataSource.initialize()
|
||||||
|
return dataSource
|
||||||
}
|
}
|
||||||
|
|
||||||
async createSchema(): Promise<void> {
|
async createSchema(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const appDataSource = await this.datasource.initialize()
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
this.queryRunner = appDataSource.createQueryRunner()
|
await queryRunner.manager.query(`create table if not exists \`${this.tableName}\` (
|
||||||
|
|
||||||
await this.queryRunner.manager.query(`create table if not exists \`${this.tableName}\` (
|
|
||||||
\`uuid\` varchar(36) primary key default (UUID()),
|
\`uuid\` varchar(36) primary key default (UUID()),
|
||||||
\`key\` varchar(255) not null,
|
\`key\` varchar(255) not null,
|
||||||
\`namespace\` varchar(255) not null,
|
\`namespace\` varchar(255) not null,
|
||||||
|
|
@ -197,17 +205,20 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
\`group_id\` longtext,
|
\`group_id\` longtext,
|
||||||
unique key \`unique_key_namespace\` (\`key\`,
|
unique key \`unique_key_namespace\` (\`key\`,
|
||||||
\`namespace\`));`)
|
\`namespace\`));`)
|
||||||
|
|
||||||
const columns = [`updated_at`, `key`, `namespace`, `group_id`]
|
const columns = [`updated_at`, `key`, `namespace`, `group_id`]
|
||||||
for (const column of columns) {
|
for (const column of columns) {
|
||||||
// MySQL does not support 'IF NOT EXISTS' function for Index
|
// MySQL does not support 'IF NOT EXISTS' function for Index
|
||||||
const Check = await this.queryRunner.manager.query(
|
const Check = await queryRunner.manager.query(
|
||||||
`SELECT COUNT(1) IndexIsThere FROM INFORMATION_SCHEMA.STATISTICS
|
`SELECT COUNT(1) IndexIsThere FROM INFORMATION_SCHEMA.STATISTICS
|
||||||
WHERE table_schema=DATABASE() AND table_name='${this.tableName}' AND index_name='${column}_index';`
|
WHERE table_schema=DATABASE() AND table_name='${this.tableName}' AND index_name='${column}_index';`
|
||||||
)
|
)
|
||||||
if (Check[0].IndexIsThere === 0)
|
if (Check[0].IndexIsThere === 0)
|
||||||
await this.queryRunner.manager.query(`CREATE INDEX \`${column}_index\`
|
await queryRunner.manager.query(`CREATE INDEX \`${column}_index\`
|
||||||
ON \`${this.tableName}\` (\`${column}\`);`)
|
ON \`${this.tableName}\` (\`${column}\`);`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await queryRunner.release()
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
// This error indicates that the table already exists
|
// This error indicates that the table already exists
|
||||||
// Due to asynchronous nature of the code, it is possible that
|
// Due to asynchronous nature of the code, it is possible that
|
||||||
|
|
@ -221,12 +232,17 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
}
|
}
|
||||||
|
|
||||||
async getTime(): Promise<number> {
|
async getTime(): Promise<number> {
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
try {
|
try {
|
||||||
const res = await this.queryRunner.manager.query(`SELECT UNIX_TIMESTAMP(NOW()) AS epoch`)
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
const res = await queryRunner.manager.query(`SELECT UNIX_TIMESTAMP(NOW()) AS epoch`)
|
||||||
|
await queryRunner.release()
|
||||||
return Number.parseFloat(res[0].epoch)
|
return Number.parseFloat(res[0].epoch)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error getting time in MySQLRecordManager:')
|
console.error('Error getting time in MySQLRecordManager:')
|
||||||
throw error
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -235,6 +251,9 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
const updatedAt = await this.getTime()
|
const updatedAt = await this.getTime()
|
||||||
const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
|
const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
|
||||||
|
|
||||||
|
|
@ -261,9 +280,18 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
ON DUPLICATE KEY UPDATE \`updated_at\` = VALUES(\`updated_at\`)`
|
ON DUPLICATE KEY UPDATE \`updated_at\` = VALUES(\`updated_at\`)`
|
||||||
|
|
||||||
// To handle multiple files upsert
|
// To handle multiple files upsert
|
||||||
for (const record of recordsToUpsert) {
|
try {
|
||||||
// Consider using a transaction for batch operations
|
for (const record of recordsToUpsert) {
|
||||||
await this.queryRunner.manager.query(query, record.flat())
|
// Consider using a transaction for batch operations
|
||||||
|
await queryRunner.manager.query(query, record.flat())
|
||||||
|
}
|
||||||
|
|
||||||
|
await queryRunner.release()
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error updating in MySQLRecordManager:')
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -272,6 +300,9 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
// Prepare the placeholders and the query
|
// Prepare the placeholders and the query
|
||||||
const placeholders = keys.map(() => `?`).join(', ')
|
const placeholders = keys.map(() => `?`).join(', ')
|
||||||
const query = `
|
const query = `
|
||||||
|
|
@ -284,21 +315,27 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Execute the query
|
// Execute the query
|
||||||
const rows = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
|
const rows = await queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
|
||||||
// Create a set of existing keys for faster lookup
|
// Create a set of existing keys for faster lookup
|
||||||
const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
|
const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
|
||||||
// Map the input keys to booleans indicating if they exist
|
// Map the input keys to booleans indicating if they exist
|
||||||
keys.forEach((key, index) => {
|
keys.forEach((key, index) => {
|
||||||
existsArray[index] = existingKeysSet.has(key)
|
existsArray[index] = existingKeysSet.has(key)
|
||||||
})
|
})
|
||||||
|
await queryRunner.release()
|
||||||
return existsArray
|
return existsArray
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error checking existence of keys')
|
console.error('Error checking existence of keys')
|
||||||
throw error // Allow the caller to handle the error
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async listKeys(options?: ListKeyOptions): Promise<string[]> {
|
async listKeys(options?: ListKeyOptions): Promise<string[]> {
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { before, after, limit, groupIds } = options ?? {}
|
const { before, after, limit, groupIds } = options ?? {}
|
||||||
let query = `SELECT \`key\` FROM \`${this.tableName}\` WHERE \`namespace\` = ?`
|
let query = `SELECT \`key\` FROM \`${this.tableName}\` WHERE \`namespace\` = ?`
|
||||||
|
|
@ -330,11 +367,14 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
query += ';'
|
query += ';'
|
||||||
|
|
||||||
// Directly using try/catch with async/await for cleaner flow
|
// Directly using try/catch with async/await for cleaner flow
|
||||||
const result = await this.queryRunner.manager.query(query, values)
|
const result = await queryRunner.manager.query(query, values)
|
||||||
|
await queryRunner.release()
|
||||||
return result.map((row: { key: string }) => row.key)
|
return result.map((row: { key: string }) => row.key)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('MySQLRecordManager listKeys Error: ')
|
console.error('MySQLRecordManager listKeys Error: ')
|
||||||
throw error // Re-throw the error to be handled by the caller
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -343,16 +383,22 @@ class MySQLRecordManager implements RecordManagerInterface {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
const placeholders = keys.map(() => '?').join(', ')
|
const placeholders = keys.map(() => '?').join(', ')
|
||||||
const query = `DELETE FROM \`${this.tableName}\` WHERE \`namespace\` = ? AND \`key\` IN (${placeholders});`
|
const query = `DELETE FROM \`${this.tableName}\` WHERE \`namespace\` = ? AND \`key\` IN (${placeholders});`
|
||||||
const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
|
const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
|
||||||
|
|
||||||
// Directly using try/catch with async/await for cleaner flow
|
// Directly using try/catch with async/await for cleaner flow
|
||||||
try {
|
try {
|
||||||
await this.queryRunner.manager.query(query, values)
|
await queryRunner.manager.query(query, values)
|
||||||
|
await queryRunner.release()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error deleting keys')
|
console.error('Error deleting keys')
|
||||||
throw error // Re-throw the error to be handled by the caller
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||||
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||||
import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
|
import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
|
||||||
import { DataSource, QueryRunner } from 'typeorm'
|
import { DataSource } from 'typeorm'
|
||||||
import { getHost } from '../../vectorstores/Postgres/utils'
|
import { getHost } from '../../vectorstores/Postgres/utils'
|
||||||
import { getDatabase, getPort, getTableName } from './utils'
|
import { getDatabase, getPort, getTableName } from './utils'
|
||||||
|
|
||||||
|
|
@ -175,29 +175,37 @@ type PostgresRecordManagerOptions = {
|
||||||
|
|
||||||
class PostgresRecordManager implements RecordManagerInterface {
|
class PostgresRecordManager implements RecordManagerInterface {
|
||||||
lc_namespace = ['langchain', 'recordmanagers', 'postgres']
|
lc_namespace = ['langchain', 'recordmanagers', 'postgres']
|
||||||
|
config: PostgresRecordManagerOptions
|
||||||
datasource: DataSource
|
|
||||||
|
|
||||||
queryRunner: QueryRunner
|
|
||||||
|
|
||||||
tableName: string
|
tableName: string
|
||||||
|
|
||||||
namespace: string
|
namespace: string
|
||||||
|
|
||||||
constructor(namespace: string, config: PostgresRecordManagerOptions) {
|
constructor(namespace: string, config: PostgresRecordManagerOptions) {
|
||||||
const { postgresConnectionOptions, tableName } = config
|
const { tableName } = config
|
||||||
this.namespace = namespace
|
this.namespace = namespace
|
||||||
this.datasource = new DataSource(postgresConnectionOptions)
|
|
||||||
this.tableName = tableName
|
this.tableName = tableName
|
||||||
|
this.config = config
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getDataSource(): Promise<DataSource> {
|
||||||
|
const { postgresConnectionOptions } = this.config
|
||||||
|
if (!postgresConnectionOptions) {
|
||||||
|
throw new Error('No datasource options provided')
|
||||||
|
}
|
||||||
|
// Prevent using default MySQL port, otherwise will throw uncaught error and crashing the app
|
||||||
|
if (postgresConnectionOptions.port === 3006) {
|
||||||
|
throw new Error('Invalid port number')
|
||||||
|
}
|
||||||
|
const dataSource = new DataSource(postgresConnectionOptions)
|
||||||
|
await dataSource.initialize()
|
||||||
|
return dataSource
|
||||||
}
|
}
|
||||||
|
|
||||||
async createSchema(): Promise<void> {
|
async createSchema(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const appDataSource = await this.datasource.initialize()
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
this.queryRunner = appDataSource.createQueryRunner()
|
await queryRunner.manager.query(`
|
||||||
|
|
||||||
await this.queryRunner.manager.query(`
|
|
||||||
CREATE TABLE IF NOT EXISTS "${this.tableName}" (
|
CREATE TABLE IF NOT EXISTS "${this.tableName}" (
|
||||||
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
key TEXT NOT NULL,
|
key TEXT NOT NULL,
|
||||||
|
|
@ -210,6 +218,8 @@ class PostgresRecordManager implements RecordManagerInterface {
|
||||||
CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
|
CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
|
||||||
CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
|
CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
|
||||||
CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
|
|
||||||
|
await queryRunner.release()
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
// This error indicates that the table already exists
|
// This error indicates that the table already exists
|
||||||
// Due to asynchronous nature of the code, it is possible that
|
// Due to asynchronous nature of the code, it is possible that
|
||||||
|
|
@ -223,8 +233,18 @@ class PostgresRecordManager implements RecordManagerInterface {
|
||||||
}
|
}
|
||||||
|
|
||||||
async getTime(): Promise<number> {
|
async getTime(): Promise<number> {
|
||||||
const res = await this.queryRunner.manager.query('SELECT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)')
|
const dataSource = await this.getDataSource()
|
||||||
return Number.parseFloat(res[0].extract)
|
try {
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
const res = await queryRunner.manager.query('SELECT EXTRACT(EPOCH FROM CURRENT_TIMESTAMP)')
|
||||||
|
await queryRunner.release()
|
||||||
|
return Number.parseFloat(res[0].extract)
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error getting time in PostgresRecordManager:')
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -247,6 +267,9 @@ class PostgresRecordManager implements RecordManagerInterface {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
const updatedAt = await this.getTime()
|
const updatedAt = await this.getTime()
|
||||||
const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
|
const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
|
||||||
|
|
||||||
|
|
@ -265,7 +288,15 @@ class PostgresRecordManager implements RecordManagerInterface {
|
||||||
const valuesPlaceholders = recordsToUpsert.map((_, j) => this.generatePlaceholderForRowAt(j, recordsToUpsert[0].length)).join(', ')
|
const valuesPlaceholders = recordsToUpsert.map((_, j) => this.generatePlaceholderForRowAt(j, recordsToUpsert[0].length)).join(', ')
|
||||||
|
|
||||||
const query = `INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) VALUES ${valuesPlaceholders} ON CONFLICT (key, namespace) DO UPDATE SET updated_at = EXCLUDED.updated_at;`
|
const query = `INSERT INTO "${this.tableName}" (key, namespace, updated_at, group_id) VALUES ${valuesPlaceholders} ON CONFLICT (key, namespace) DO UPDATE SET updated_at = EXCLUDED.updated_at;`
|
||||||
await this.queryRunner.manager.query(query, recordsToUpsert.flat())
|
try {
|
||||||
|
await queryRunner.manager.query(query, recordsToUpsert.flat())
|
||||||
|
await queryRunner.release()
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error updating in PostgresRecordManager:')
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async exists(keys: string[]): Promise<boolean[]> {
|
async exists(keys: string[]): Promise<boolean[]> {
|
||||||
|
|
@ -273,14 +304,25 @@ class PostgresRecordManager implements RecordManagerInterface {
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
const startIndex = 2
|
const startIndex = 2
|
||||||
const arrayPlaceholders = keys.map((_, i) => `$${i + startIndex}`).join(', ')
|
const arrayPlaceholders = keys.map((_, i) => `$${i + startIndex}`).join(', ')
|
||||||
|
|
||||||
const query = `
|
const query = `
|
||||||
SELECT k, (key is not null) ex from unnest(ARRAY[${arrayPlaceholders}]) k left join "${this.tableName}" on k=key and namespace = $1;
|
SELECT k, (key is not null) ex from unnest(ARRAY[${arrayPlaceholders}]) k left join "${this.tableName}" on k=key and namespace = $1;
|
||||||
`
|
`
|
||||||
const res = await this.queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
|
try {
|
||||||
return res.map((row: { ex: boolean }) => row.ex)
|
const res = await queryRunner.manager.query(query, [this.namespace, ...keys.flat()])
|
||||||
|
await queryRunner.release()
|
||||||
|
return res.map((row: { ex: boolean }) => row.ex)
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error checking existence of keys in PostgresRecordManager:')
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async listKeys(options?: ListKeyOptions): Promise<string[]> {
|
async listKeys(options?: ListKeyOptions): Promise<string[]> {
|
||||||
|
|
@ -314,8 +356,20 @@ class PostgresRecordManager implements RecordManagerInterface {
|
||||||
}
|
}
|
||||||
|
|
||||||
query += ';'
|
query += ';'
|
||||||
const res = await this.queryRunner.manager.query(query, values)
|
|
||||||
return res.map((row: { key: string }) => row.key)
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await queryRunner.manager.query(query, values)
|
||||||
|
await queryRunner.release()
|
||||||
|
return res.map((row: { key: string }) => row.key)
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error listing keys in PostgresRecordManager:')
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteKeys(keys: string[]): Promise<void> {
|
async deleteKeys(keys: string[]): Promise<void> {
|
||||||
|
|
@ -323,16 +377,19 @@ class PostgresRecordManager implements RecordManagerInterface {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
const query = `DELETE FROM "${this.tableName}" WHERE namespace = $1 AND key = ANY($2);`
|
const dataSource = await this.getDataSource()
|
||||||
await this.queryRunner.manager.query(query, [this.namespace, keys])
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
try {
|
||||||
* Terminates the connection pool.
|
const query = `DELETE FROM "${this.tableName}" WHERE namespace = $1 AND key = ANY($2);`
|
||||||
* @returns {Promise<void>}
|
await queryRunner.manager.query(query, [this.namespace, keys])
|
||||||
*/
|
await queryRunner.release()
|
||||||
async end(): Promise<void> {
|
} catch (error) {
|
||||||
if (this.datasource && this.datasource.isInitialized) await this.datasource.destroy()
|
console.error('Error deleting keys')
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||||
import { getBaseClasses } from '../../../src/utils'
|
import { getBaseClasses, getUserHome } from '../../../src/utils'
|
||||||
import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
|
import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
|
||||||
import { DataSource, QueryRunner } from 'typeorm'
|
import { DataSource } from 'typeorm'
|
||||||
import path from 'path'
|
import path from 'path'
|
||||||
|
|
||||||
class SQLiteRecordManager_RecordManager implements INode {
|
class SQLiteRecordManager_RecordManager implements INode {
|
||||||
|
|
@ -19,19 +19,19 @@ class SQLiteRecordManager_RecordManager implements INode {
|
||||||
constructor() {
|
constructor() {
|
||||||
this.label = 'SQLite Record Manager'
|
this.label = 'SQLite Record Manager'
|
||||||
this.name = 'SQLiteRecordManager'
|
this.name = 'SQLiteRecordManager'
|
||||||
this.version = 1.0
|
this.version = 1.1
|
||||||
this.type = 'SQLite RecordManager'
|
this.type = 'SQLite RecordManager'
|
||||||
this.icon = 'sqlite.png'
|
this.icon = 'sqlite.png'
|
||||||
this.category = 'Record Manager'
|
this.category = 'Record Manager'
|
||||||
this.description = 'Use SQLite to keep track of document writes into the vector databases'
|
this.description = 'Use SQLite to keep track of document writes into the vector databases'
|
||||||
this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(SQLiteRecordManager)]
|
this.baseClasses = [this.type, 'RecordManager', ...getBaseClasses(SQLiteRecordManager)]
|
||||||
this.inputs = [
|
this.inputs = [
|
||||||
{
|
/*{
|
||||||
label: 'Database File Path',
|
label: 'Database File Path',
|
||||||
name: 'databaseFilePath',
|
name: 'databaseFilePath',
|
||||||
type: 'string',
|
type: 'string',
|
||||||
placeholder: 'C:\\Users\\User\\.flowise\\database.sqlite'
|
placeholder: 'C:\\Users\\User\\.flowise\\database.sqlite'
|
||||||
},
|
},*/
|
||||||
{
|
{
|
||||||
label: 'Additional Connection Configuration',
|
label: 'Additional Connection Configuration',
|
||||||
name: 'additionalConfig',
|
name: 'additionalConfig',
|
||||||
|
|
@ -106,7 +106,6 @@ class SQLiteRecordManager_RecordManager implements INode {
|
||||||
const cleanup = nodeData.inputs?.cleanup as string
|
const cleanup = nodeData.inputs?.cleanup as string
|
||||||
const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
|
const _sourceIdKey = nodeData.inputs?.sourceIdKey as string
|
||||||
const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
|
const sourceIdKey = _sourceIdKey ? _sourceIdKey : 'source'
|
||||||
const databaseFilePath = nodeData.inputs?.databaseFilePath as string
|
|
||||||
|
|
||||||
let additionalConfiguration = {}
|
let additionalConfiguration = {}
|
||||||
if (additionalConfig) {
|
if (additionalConfig) {
|
||||||
|
|
@ -117,10 +116,12 @@ class SQLiteRecordManager_RecordManager implements INode {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const database = path.join(process.env.DATABASE_PATH ?? path.join(getUserHome(), '.flowise'), 'database.sqlite')
|
||||||
|
|
||||||
const sqliteOptions = {
|
const sqliteOptions = {
|
||||||
|
database,
|
||||||
...additionalConfiguration,
|
...additionalConfiguration,
|
||||||
type: 'sqlite',
|
type: 'sqlite'
|
||||||
database: path.resolve(databaseFilePath)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const args = {
|
const args = {
|
||||||
|
|
@ -144,29 +145,33 @@ type SQLiteRecordManagerOptions = {
|
||||||
|
|
||||||
class SQLiteRecordManager implements RecordManagerInterface {
|
class SQLiteRecordManager implements RecordManagerInterface {
|
||||||
lc_namespace = ['langchain', 'recordmanagers', 'sqlite']
|
lc_namespace = ['langchain', 'recordmanagers', 'sqlite']
|
||||||
|
|
||||||
datasource: DataSource
|
|
||||||
|
|
||||||
queryRunner: QueryRunner
|
|
||||||
|
|
||||||
tableName: string
|
tableName: string
|
||||||
|
|
||||||
namespace: string
|
namespace: string
|
||||||
|
config: SQLiteRecordManagerOptions
|
||||||
|
|
||||||
constructor(namespace: string, config: SQLiteRecordManagerOptions) {
|
constructor(namespace: string, config: SQLiteRecordManagerOptions) {
|
||||||
const { sqliteOptions, tableName } = config
|
const { tableName } = config
|
||||||
this.namespace = namespace
|
this.namespace = namespace
|
||||||
this.tableName = tableName || 'upsertion_records'
|
this.tableName = tableName || 'upsertion_records'
|
||||||
this.datasource = new DataSource(sqliteOptions)
|
this.config = config
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getDataSource(): Promise<DataSource> {
|
||||||
|
const { sqliteOptions } = this.config
|
||||||
|
if (!sqliteOptions) {
|
||||||
|
throw new Error('No datasource options provided')
|
||||||
|
}
|
||||||
|
const dataSource = new DataSource(sqliteOptions)
|
||||||
|
await dataSource.initialize()
|
||||||
|
return dataSource
|
||||||
}
|
}
|
||||||
|
|
||||||
async createSchema(): Promise<void> {
|
async createSchema(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const appDataSource = await this.datasource.initialize()
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
this.queryRunner = appDataSource.createQueryRunner()
|
await queryRunner.manager.query(`
|
||||||
|
|
||||||
await this.queryRunner.manager.query(`
|
|
||||||
CREATE TABLE IF NOT EXISTS "${this.tableName}" (
|
CREATE TABLE IF NOT EXISTS "${this.tableName}" (
|
||||||
uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
uuid TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(16)))),
|
||||||
key TEXT NOT NULL,
|
key TEXT NOT NULL,
|
||||||
|
|
@ -179,6 +184,8 @@ CREATE INDEX IF NOT EXISTS updated_at_index ON "${this.tableName}" (updated_at);
|
||||||
CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
|
CREATE INDEX IF NOT EXISTS key_index ON "${this.tableName}" (key);
|
||||||
CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
|
CREATE INDEX IF NOT EXISTS namespace_index ON "${this.tableName}" (namespace);
|
||||||
CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
|
|
||||||
|
await queryRunner.release()
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
// This error indicates that the table already exists
|
// This error indicates that the table already exists
|
||||||
// Due to asynchronous nature of the code, it is possible that
|
// Due to asynchronous nature of the code, it is possible that
|
||||||
|
|
@ -192,12 +199,17 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
}
|
}
|
||||||
|
|
||||||
async getTime(): Promise<number> {
|
async getTime(): Promise<number> {
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
try {
|
try {
|
||||||
const res = await this.queryRunner.manager.query(`SELECT strftime('%s', 'now') AS epoch`)
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
const res = await queryRunner.manager.query(`SELECT strftime('%s', 'now') AS epoch`)
|
||||||
|
await queryRunner.release()
|
||||||
return Number.parseFloat(res[0].epoch)
|
return Number.parseFloat(res[0].epoch)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error getting time in SQLiteRecordManager:')
|
console.error('Error getting time in SQLiteRecordManager:')
|
||||||
throw error
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -205,6 +217,8 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
if (keys.length === 0) {
|
if (keys.length === 0) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
const updatedAt = await this.getTime()
|
const updatedAt = await this.getTime()
|
||||||
const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
|
const { timeAtLeast, groupIds: _groupIds } = updateOptions ?? {}
|
||||||
|
|
@ -231,10 +245,18 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
VALUES (?, ?, ?, ?)
|
VALUES (?, ?, ?, ?)
|
||||||
ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at`
|
ON CONFLICT (key, namespace) DO UPDATE SET updated_at = excluded.updated_at`
|
||||||
|
|
||||||
// To handle multiple files upsert
|
try {
|
||||||
for (const record of recordsToUpsert) {
|
// To handle multiple files upsert
|
||||||
// Consider using a transaction for batch operations
|
for (const record of recordsToUpsert) {
|
||||||
await this.queryRunner.manager.query(query, record.flat())
|
// Consider using a transaction for batch operations
|
||||||
|
await queryRunner.manager.query(query, record.flat())
|
||||||
|
}
|
||||||
|
await queryRunner.release()
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error updating in SQLiteRecordManager:')
|
||||||
|
throw error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -253,19 +275,25 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
// Initialize an array to fill with the existence checks
|
// Initialize an array to fill with the existence checks
|
||||||
const existsArray = new Array(keys.length).fill(false)
|
const existsArray = new Array(keys.length).fill(false)
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Execute the query
|
// Execute the query
|
||||||
const rows = await this.queryRunner.manager.query(sql, [this.namespace, ...keys.flat()])
|
const rows = await queryRunner.manager.query(sql, [this.namespace, ...keys.flat()])
|
||||||
// Create a set of existing keys for faster lookup
|
// Create a set of existing keys for faster lookup
|
||||||
const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
|
const existingKeysSet = new Set(rows.map((row: { key: string }) => row.key))
|
||||||
// Map the input keys to booleans indicating if they exist
|
// Map the input keys to booleans indicating if they exist
|
||||||
keys.forEach((key, index) => {
|
keys.forEach((key, index) => {
|
||||||
existsArray[index] = existingKeysSet.has(key)
|
existsArray[index] = existingKeysSet.has(key)
|
||||||
})
|
})
|
||||||
|
await queryRunner.release()
|
||||||
return existsArray
|
return existsArray
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error checking existence of keys')
|
console.error('Error checking existence of keys')
|
||||||
throw error // Allow the caller to handle the error
|
throw error // Allow the caller to handle the error
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -299,13 +327,19 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
|
|
||||||
query += ';'
|
query += ';'
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
// Directly using try/catch with async/await for cleaner flow
|
// Directly using try/catch with async/await for cleaner flow
|
||||||
try {
|
try {
|
||||||
const result = await this.queryRunner.manager.query(query, values)
|
const result = await queryRunner.manager.query(query, values)
|
||||||
|
await queryRunner.release()
|
||||||
return result.map((row: { key: string }) => row.key)
|
return result.map((row: { key: string }) => row.key)
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error listing keys.')
|
console.error('Error listing keys.')
|
||||||
throw error // Re-throw the error to be handled by the caller
|
throw error // Re-throw the error to be handled by the caller
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -314,16 +348,22 @@ CREATE INDEX IF NOT EXISTS group_id_index ON "${this.tableName}" (group_id);`)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const dataSource = await this.getDataSource()
|
||||||
|
const queryRunner = dataSource.createQueryRunner()
|
||||||
|
|
||||||
const placeholders = keys.map(() => '?').join(', ')
|
const placeholders = keys.map(() => '?').join(', ')
|
||||||
const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});`
|
const query = `DELETE FROM "${this.tableName}" WHERE namespace = ? AND key IN (${placeholders});`
|
||||||
const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
|
const values = [this.namespace, ...keys].map((v) => (typeof v !== 'string' ? `${v}` : v))
|
||||||
|
|
||||||
// Directly using try/catch with async/await for cleaner flow
|
// Directly using try/catch with async/await for cleaner flow
|
||||||
try {
|
try {
|
||||||
await this.queryRunner.manager.query(query, values)
|
await queryRunner.manager.query(query, values)
|
||||||
|
await queryRunner.release()
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error deleting keys')
|
console.error('Error deleting keys')
|
||||||
throw error // Re-throw the error to be handled by the caller
|
throw error // Re-throw the error to be handled by the caller
|
||||||
|
} finally {
|
||||||
|
await dataSource.destroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -33,6 +33,11 @@ export class PGVectorDriver extends VectorStoreDriver {
|
||||||
password: password,
|
password: password,
|
||||||
database: this.getDatabase()
|
database: this.getDatabase()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Prevent using default MySQL port, otherwise will throw uncaught error and crashing the app
|
||||||
|
if (this.getHost() === '3006') {
|
||||||
|
throw new Error('Invalid port number')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return this._postgresConnectionOptions
|
return this._postgresConnectionOptions
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,11 @@ export class TypeORMDriver extends VectorStoreDriver {
|
||||||
password: password,
|
password: password,
|
||||||
database: this.getDatabase()
|
database: this.getDatabase()
|
||||||
} as DataSourceOptions
|
} as DataSourceOptions
|
||||||
|
|
||||||
|
// Prevent using default MySQL port, otherwise will throw uncaught error and crashing the app
|
||||||
|
if (this.getHost() === '3006') {
|
||||||
|
throw new Error('Invalid port number')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return this._postgresConnectionOptions
|
return this._postgresConnectionOptions
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue