Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: change DocumentStoreFileChunk columns to LONGTEXT (#5465) 

* fix: change DocumentStoreFileChunk columns to LONGTEXT

- Fixes 'Unterminated string in JSON' error with large documents
- Changes pageContent and metadata columns from TEXT (64KB) to LONGTEXT (4GB)
- Adds database migrations for MySQL and MariaDB
- Resolves issue #2352

* docs: add data loss warning to migration rollback

- Add warning comment in down() migration methods
- Clarifies potential data loss when reverting LONGTEXT to TEXT
- Helps future developers understand rollback risks

* lint fix

* lint fix

* Update DocumentStoreFileChunk.ts

---------

Co-authored-by: KushagraaWadhwa <kushagraa@users.noreply.github.com>
Co-authored-by: Henry Heng <henryheng@flowiseai.com>
Co-authored-by: Henry <hzj94@hotmail.com>
This commit is contained in:
Kushagra Wadhwa 2025-11-25 18:42:22 +05:30 committed by GitHub
parent 0cc7b3036e
commit 03ef28afbc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 34 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
packages/server/src/database/migrations/mariadb/1765000000000-FixDocumentStoreFileChunkLongText.ts Normal file
View File

@ -0,0 +1,14 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class FixDocumentStoreFileChunkLongText1765000000000 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` LONGTEXT NOT NULL;`)
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` LONGTEXT NULL;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
// WARNING: Reverting to TEXT may cause data loss if content exceeds the 64KB limit.
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` TEXT NOT NULL;`)
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` TEXT NULL;`)
}
}

4
packages/server/src/database/migrations/mariadb/index.ts
View File

@ -40,6 +40,7 @@ import { AddTextToSpeechToChatFlow1754986457485 } from './1754986457485-AddTextT
import { ModifyChatflowType1755066758601 } from './1755066758601-ModifyChatflowType'
import { AddTextToSpeechToChatFlow1759419231100 } from './1759419231100-AddTextToSpeechToChatFlow'
import { AddChatFlowNameIndex1759424809984 } from './1759424809984-AddChatFlowNameIndex'
import { FixDocumentStoreFileChunkLongText1765000000000 } from './1765000000000-FixDocumentStoreFileChunkLongText'
import { AddAuthTables1720230151482 } from '../../../enterprise/database/migrations/mariadb/1720230151482-AddAuthTables'
import { AddWorkspace1725437498242 } from '../../../enterprise/database/migrations/mariadb/1725437498242-AddWorkspace'
@ -106,5 +107,6 @@ export const mariadbMigrations = [
AddTextToSpeechToChatFlow1754986457485,
ModifyChatflowType1755066758601,
AddTextToSpeechToChatFlow1759419231100,
AddChatFlowNameIndex1759424809984
AddChatFlowNameIndex1759424809984,
FixDocumentStoreFileChunkLongText1765000000000
]

14
packages/server/src/database/migrations/mysql/1765000000000-FixDocumentStoreFileChunkLongText.ts Normal file
View File

@ -0,0 +1,14 @@
import { MigrationInterface, QueryRunner } from 'typeorm'
export class FixDocumentStoreFileChunkLongText1765000000000 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` LONGTEXT NOT NULL;`)
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` LONGTEXT NULL;`)
}
public async down(queryRunner: QueryRunner): Promise<void> {
// WARNING: Reverting to TEXT may cause data loss if content exceeds the 64KB limit.
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`pageContent\` TEXT NOT NULL;`)
await queryRunner.query(`ALTER TABLE \`document_store_file_chunk\` MODIFY \`metadata\` TEXT NULL;`)
}
}

4
packages/server/src/database/migrations/mysql/index.ts
View File

@ -41,6 +41,7 @@ import { AddTextToSpeechToChatFlow1754986468397 } from './1754986468397-AddTextT
import { ModifyChatflowType1755066758601 } from './1755066758601-ModifyChatflowType'
import { AddTextToSpeechToChatFlow1759419216034 } from './1759419216034-AddTextToSpeechToChatFlow'
import { AddChatFlowNameIndex1759424828558 } from './1759424828558-AddChatFlowNameIndex'
import { FixDocumentStoreFileChunkLongText1765000000000 } from './1765000000000-FixDocumentStoreFileChunkLongText'
import { AddAuthTables1720230151482 } from '../../../enterprise/database/migrations/mysql/1720230151482-AddAuthTables'
import { AddWorkspace1720230151484 } from '../../../enterprise/database/migrations/mysql/1720230151484-AddWorkspace'
@ -108,5 +109,6 @@ export const mysqlMigrations = [
AddTextToSpeechToChatFlow1754986468397,
ModifyChatflowType1755066758601,
AddTextToSpeechToChatFlow1759419216034,
AddChatFlowNameIndex1759424828558
AddChatFlowNameIndex1759424828558,
FixDocumentStoreFileChunkLongText1765000000000
]