fix: change DocumentStoreFileChunk columns to LONGTEXT (#5465)

* fix: change DocumentStoreFileChunk columns to LONGTEXT

- Fixes 'Unterminated string in JSON' error with large documents
- Changes pageContent and metadata columns from TEXT (64KB) to LONGTEXT (4GB)
- Adds database migrations for MySQL and MariaDB
- Resolves issue #2352

* docs: add data loss warning to migration rollback

- Add warning comment in down() migration methods
- Clarifies potential data loss when reverting LONGTEXT to TEXT
- Helps future developers understand rollback risks

* lint fix

* lint fix

* Update DocumentStoreFileChunk.ts

---------

Co-authored-by: KushagraaWadhwa <kushagraa@users.noreply.github.com>
Co-authored-by: Henry Heng <henryheng@flowiseai.com>
Co-authored-by: Henry <hzj94@hotmail.com>

packages/server/src/database/migrations/mariadb/1765000000000-FixDocumentStoreFileChunkLongText.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class FixDocumentStoreFileChunkLongText1765000000000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` LONGTEXT NOT NULL;`)
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` LONGTEXT NULL;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // WARNING: Reverting to TEXT may cause data loss if content exceeds the 64KB limit.
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` TEXT NOT NULL;`)
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` TEXT NULL;`)
+    }
+}

packages/server/src/database/migrations/mariadb/index.ts

@@ -40,6 +40,7 @@ import { AddTextToSpeechToChatFlow1754986457485 } from './1754986457485-AddTextT
 import { ModifyChatflowType1755066758601 } from './1755066758601-ModifyChatflowType'
 import { AddTextToSpeechToChatFlow1759419231100 } from './1759419231100-AddTextToSpeechToChatFlow'
 import { AddChatFlowNameIndex1759424809984 } from './1759424809984-AddChatFlowNameIndex'
+import { FixDocumentStoreFileChunkLongText1765000000000 } from './1765000000000-FixDocumentStoreFileChunkLongText'
 
 import { AddAuthTables1720230151482 } from '../../../enterprise/database/migrations/mariadb/1720230151482-AddAuthTables'
 import { AddWorkspace1725437498242 } from '../../../enterprise/database/migrations/mariadb/1725437498242-AddWorkspace'
@@ -106,5 +107,6 @@ export const mariadbMigrations = [
     AddTextToSpeechToChatFlow1754986457485,
     ModifyChatflowType1755066758601,
     AddTextToSpeechToChatFlow1759419231100,
-    AddChatFlowNameIndex1759424809984
+    AddChatFlowNameIndex1759424809984,
+    FixDocumentStoreFileChunkLongText1765000000000
 ]

packages/server/src/database/migrations/mysql/1765000000000-FixDocumentStoreFileChunkLongText.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class FixDocumentStoreFileChunkLongText1765000000000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` LONGTEXT NOT NULL;`)
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` LONGTEXT NULL;`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // WARNING: Reverting to TEXT may cause data loss if content exceeds the 64KB limit.
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` TEXT NOT NULL;`)
+        await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` TEXT NULL;`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -41,6 +41,7 @@ import { AddTextToSpeechToChatFlow1754986468397 } from './1754986468397-AddTextT
 import { ModifyChatflowType1755066758601 } from './1755066758601-ModifyChatflowType'
 import { AddTextToSpeechToChatFlow1759419216034 } from './1759419216034-AddTextToSpeechToChatFlow'
 import { AddChatFlowNameIndex1759424828558 } from './1759424828558-AddChatFlowNameIndex'
+import { FixDocumentStoreFileChunkLongText1765000000000 } from './1765000000000-FixDocumentStoreFileChunkLongText'
 
 import { AddAuthTables1720230151482 } from '../../../enterprise/database/migrations/mysql/1720230151482-AddAuthTables'
 import { AddWorkspace1720230151484 } from '../../../enterprise/database/migrations/mysql/1720230151484-AddWorkspace'
@@ -108,5 +109,6 @@ export const mysqlMigrations = [
     AddTextToSpeechToChatFlow1754986468397,
     ModifyChatflowType1755066758601,
     AddTextToSpeechToChatFlow1759419216034,
-    AddChatFlowNameIndex1759424828558
+    AddChatFlowNameIndex1759424828558,
+    FixDocumentStoreFileChunkLongText1765000000000
 ]