From 03ef28afbc71c44892a9822d7323c1254124b2e2 Mon Sep 17 00:00:00 2001 From: Kushagra Wadhwa <130084175+KushagraaWadhwa@users.noreply.github.com> Date: Tue, 25 Nov 2025 18:42:22 +0530 Subject: [PATCH] fix: change DocumentStoreFileChunk columns to LONGTEXT (#5465) * fix: change DocumentStoreFileChunk columns to LONGTEXT - Fixes 'Unterminated string in JSON' error with large documents - Changes pageContent and metadata columns from TEXT (64KB) to LONGTEXT (4GB) - Adds database migrations for MySQL and MariaDB - Resolves issue #2352 * docs: add data loss warning to migration rollback - Add warning comment in down() migration methods - Clarifies potential data loss when reverting LONGTEXT to TEXT - Helps future developers understand rollback risks * lint fix * lint fix * Update DocumentStoreFileChunk.ts --------- Co-authored-by: KushagraaWadhwa Co-authored-by: Henry Heng Co-authored-by: Henry --- ...5000000000-FixDocumentStoreFileChunkLongText.ts | 14 ++++++++++++++ .../src/database/migrations/mariadb/index.ts | 4 +++- ...5000000000-FixDocumentStoreFileChunkLongText.ts | 14 ++++++++++++++ .../server/src/database/migrations/mysql/index.ts | 4 +++- 4 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 packages/server/src/database/migrations/mariadb/1765000000000-FixDocumentStoreFileChunkLongText.ts create mode 100644 packages/server/src/database/migrations/mysql/1765000000000-FixDocumentStoreFileChunkLongText.ts diff --git a/packages/server/src/database/migrations/mariadb/1765000000000-FixDocumentStoreFileChunkLongText.ts b/packages/server/src/database/migrations/mariadb/1765000000000-FixDocumentStoreFileChunkLongText.ts new file mode 100644 index 000000000..130f59745 --- /dev/null +++ b/packages/server/src/database/migrations/mariadb/1765000000000-FixDocumentStoreFileChunkLongText.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class FixDocumentStoreFileChunkLongText1765000000000 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` LONGTEXT NOT NULL;`) + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` LONGTEXT NULL;`) + } + + public async down(queryRunner: QueryRunner): Promise { + // WARNING: Reverting to TEXT may cause data loss if content exceeds the 64KB limit. + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` TEXT NOT NULL;`) + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` TEXT NULL;`) + } +} diff --git a/packages/server/src/database/migrations/mariadb/index.ts b/packages/server/src/database/migrations/mariadb/index.ts index d3a7bbe85..07ddb6ed0 100644 --- a/packages/server/src/database/migrations/mariadb/index.ts +++ b/packages/server/src/database/migrations/mariadb/index.ts @@ -40,6 +40,7 @@ import { AddTextToSpeechToChatFlow1754986457485 } from './1754986457485-AddTextT import { ModifyChatflowType1755066758601 } from './1755066758601-ModifyChatflowType' import { AddTextToSpeechToChatFlow1759419231100 } from './1759419231100-AddTextToSpeechToChatFlow' import { AddChatFlowNameIndex1759424809984 } from './1759424809984-AddChatFlowNameIndex' +import { FixDocumentStoreFileChunkLongText1765000000000 } from './1765000000000-FixDocumentStoreFileChunkLongText' import { AddAuthTables1720230151482 } from '../../../enterprise/database/migrations/mariadb/1720230151482-AddAuthTables' import { AddWorkspace1725437498242 } from '../../../enterprise/database/migrations/mariadb/1725437498242-AddWorkspace' @@ -106,5 +107,6 @@ export const mariadbMigrations = [ AddTextToSpeechToChatFlow1754986457485, ModifyChatflowType1755066758601, AddTextToSpeechToChatFlow1759419231100, - AddChatFlowNameIndex1759424809984 + AddChatFlowNameIndex1759424809984, + FixDocumentStoreFileChunkLongText1765000000000 ] diff --git a/packages/server/src/database/migrations/mysql/1765000000000-FixDocumentStoreFileChunkLongText.ts b/packages/server/src/database/migrations/mysql/1765000000000-FixDocumentStoreFileChunkLongText.ts new file mode 100644 index 000000000..130f59745 --- /dev/null +++ b/packages/server/src/database/migrations/mysql/1765000000000-FixDocumentStoreFileChunkLongText.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from 'typeorm' + +export class FixDocumentStoreFileChunkLongText1765000000000 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` LONGTEXT NOT NULL;`) + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` LONGTEXT NULL;`) + } + + public async down(queryRunner: QueryRunner): Promise { + // WARNING: Reverting to TEXT may cause data loss if content exceeds the 64KB limit. + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` TEXT NOT NULL;`) + await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` TEXT NULL;`) + } +} diff --git a/packages/server/src/database/migrations/mysql/index.ts b/packages/server/src/database/migrations/mysql/index.ts index be6c97949..c7f5d2eba 100644 --- a/packages/server/src/database/migrations/mysql/index.ts +++ b/packages/server/src/database/migrations/mysql/index.ts @@ -41,6 +41,7 @@ import { AddTextToSpeechToChatFlow1754986468397 } from './1754986468397-AddTextT import { ModifyChatflowType1755066758601 } from './1755066758601-ModifyChatflowType' import { AddTextToSpeechToChatFlow1759419216034 } from './1759419216034-AddTextToSpeechToChatFlow' import { AddChatFlowNameIndex1759424828558 } from './1759424828558-AddChatFlowNameIndex' +import { FixDocumentStoreFileChunkLongText1765000000000 } from './1765000000000-FixDocumentStoreFileChunkLongText' import { AddAuthTables1720230151482 } from '../../../enterprise/database/migrations/mysql/1720230151482-AddAuthTables' import { AddWorkspace1720230151484 } from '../../../enterprise/database/migrations/mysql/1720230151484-AddWorkspace' @@ -108,5 +109,6 @@ export const mysqlMigrations = [ AddTextToSpeechToChatFlow1754986468397, ModifyChatflowType1755066758601, AddTextToSpeechToChatFlow1759419216034, - AddChatFlowNameIndex1759424828558 + AddChatFlowNameIndex1759424828558, + FixDocumentStoreFileChunkLongText1765000000000 ]