feat: add JSONPathExtractor tool (#5052)

* feat: add JSONPathExtractor tool with lodash-based path extraction

  - Implement JSONPathExtractor tool for extracting values from JSON using path notation
  - Use lodash.get for robust path extraction supporting edge cases (numeric string keys, array indexing)
  - Add configurable error handling with returnNullOnError parameter
  - Include comprehensive test suite with 34 tests covering all scenarios
  - Support JSON strings, objects, and arrays as input

* fix lint

* Update pnpm-lock.yaml

* fix: exclude test files from TypeScript compilation

Prevents test files from being included in the dist folder which was causing
"jest is not defined" errors during server startup.

---------

Co-authored-by: Henry Heng <henryheng@flowiseai.com>

packages/components/jest.config.js

@@ -0,0 +1,15 @@
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    roots: ['<rootDir>/nodes'],
+    transform: {
+        '^.+\\.tsx?$': 'ts-jest'
+    },
+    testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.tsx?$',
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+    verbose: true,
+    testPathIgnorePatterns: ['/node_modules/', '/dist/'],
+    moduleNameMapper: {
+        '^../../../src/(.*)$': '<rootDir>/src/$1'
+    }
+}

packages/components/nodes/tools/JSONPathExtractor/JSONPathExtractor.test.ts

@@ -0,0 +1,261 @@
+const { nodeClass: JSONPathExtractor_Tools } = require('./JSONPathExtractor')
+import { INodeData } from '../../../src/Interface'
+
+// Mock the getBaseClasses function
+jest.mock('../../../src/utils', () => ({
+    getBaseClasses: jest.fn(() => ['Tool', 'StructuredTool'])
+}))
+
+// Helper function to create a valid INodeData object
+function createNodeData(id: string, inputs: any): INodeData {
+    return {
+        id: id,
+        label: 'JSON Path Extractor',
+        name: 'jsonPathExtractor',
+        type: 'JSONPathExtractor',
+        icon: 'jsonpathextractor.svg',
+        version: 1.0,
+        category: 'Tools',
+        baseClasses: ['JSONPathExtractor', 'Tool'],
+        inputs: inputs
+    }
+}
+
+describe('JSONPathExtractor', () => {
+    let nodeClass: any
+
+    beforeEach(() => {
+        nodeClass = new JSONPathExtractor_Tools()
+    })
+
+    describe('Tool Initialization', () => {
+        it('should throw error when path is not provided', async () => {
+            const nodeData = createNodeData('test-node-1', {
+                path: ''
+            })
+
+            await expect(nodeClass.init(nodeData, '')).rejects.toThrow('JSON Path is required')
+        })
+
+        it('should initialize tool with path and default returnNullOnError', async () => {
+            const nodeData = createNodeData('test-node-2', {
+                path: 'data.value'
+            })
+
+            const tool = await nodeClass.init(nodeData, '')
+            expect(tool).toBeDefined()
+            expect(tool.name).toBe('json_path_extractor')
+        })
+
+        it('should initialize tool with custom returnNullOnError', async () => {
+            const nodeData = createNodeData('test-node-3', {
+                path: 'data.value',
+                returnNullOnError: true
+            })
+
+            const tool = await nodeClass.init(nodeData, '')
+            expect(tool).toBeDefined()
+        })
+    })
+
+    describe('JSONPathExtractorTool Functionality', () => {
+        describe('Positive test cases - Path extraction', () => {
+            const successCases = [
+                {
+                    name: 'simple path from object',
+                    path: 'data.value',
+                    input: { data: { value: 'test' } },
+                    expected: 'test'
+                },
+                {
+                    name: 'nested path from object',
+                    path: 'user.profile.name',
+                    input: { user: { profile: { name: 'John' } } },
+                    expected: 'John'
+                },
+                {
+                    name: 'array index access',
+                    path: 'items[0].name',
+                    input: { items: [{ name: 'first' }, { name: 'second' }] },
+                    expected: 'first'
+                },
+                {
+                    name: 'multi-dimensional array',
+                    path: 'matrix[0][1]',
+                    input: {
+                        matrix: [
+                            ['a', 'b'],
+                            ['c', 'd']
+                        ]
+                    },
+                    expected: 'b'
+                },
+                {
+                    name: 'object return (stringified)',
+                    path: 'data',
+                    input: { data: { nested: 'object' } },
+                    expected: '{"nested":"object"}'
+                },
+                {
+                    name: 'array return (stringified)',
+                    path: 'tags',
+                    input: { tags: ['a', 'b', 'c'] },
+                    expected: '["a","b","c"]'
+                },
+                {
+                    name: 'deep nesting',
+                    path: 'a.b.c.d.e',
+                    input: { a: { b: { c: { d: { e: 'deep' } } } } },
+                    expected: 'deep'
+                },
+                {
+                    name: 'array at root with index',
+                    path: '[1]',
+                    input: ['first', 'second', 'third'],
+                    expected: 'second'
+                }
+            ]
+
+            test.each(successCases)('should extract $name', async ({ path, input, expected }) => {
+                const nodeData = createNodeData(`test-node-${path}`, {
+                    path: path,
+                    returnNullOnError: false
+                })
+                const tool = await nodeClass.init(nodeData, '')
+                const result = await tool._call({ json: input })
+                expect(result).toBe(expected)
+            })
+        })
+
+        describe('Primitive value handling', () => {
+            const primitiveTests = [
+                { name: 'string', path: 'val', input: { val: 'text' }, expected: 'text' },
+                { name: 'number', path: 'val', input: { val: 42 }, expected: '42' },
+                { name: 'zero', path: 'val', input: { val: 0 }, expected: '0' },
+                { name: 'boolean true', path: 'val', input: { val: true }, expected: 'true' },
+                { name: 'boolean false', path: 'val', input: { val: false }, expected: 'false' },
+                { name: 'null', path: 'val', input: { val: null }, expected: 'null' },
+                { name: 'empty string', path: 'val', input: { val: '' }, expected: '' }
+            ]
+
+            test.each(primitiveTests)('should handle $name value', async ({ path, input, expected }) => {
+                const nodeData = createNodeData(`test-primitive`, {
+                    path: path,
+                    returnNullOnError: false
+                })
+                const tool = await nodeClass.init(nodeData, '')
+                const result = await tool._call({ json: input })
+                expect(result).toBe(expected)
+            })
+        })
+
+        describe('Special characters in keys', () => {
+            const specialCharTests = [
+                { name: 'dashes', path: 'data.key-with-dash', input: { data: { 'key-with-dash': 'value' } } },
+                { name: 'spaces', path: 'data.key with spaces', input: { data: { 'key with spaces': 'value' } } },
+                { name: 'unicode', path: 'data.emoji🔑', input: { data: { 'emoji🔑': 'value' } } },
+                { name: 'numeric strings', path: 'data.123', input: { data: { '123': 'value' } } }
+            ]
+
+            test.each(specialCharTests)('should handle $name in keys', async ({ path, input }) => {
+                const nodeData = createNodeData(`test-special`, {
+                    path: path,
+                    returnNullOnError: false
+                })
+                const tool = await nodeClass.init(nodeData, '')
+                const result = await tool._call({ json: input })
+                expect(result).toBe('value')
+            })
+        })
+
+        describe('Error handling - throw mode', () => {
+            const errorCases = [
+                {
+                    name: 'path not found',
+                    path: 'data.value',
+                    input: { data: { other: 'value' } },
+                    errorPattern: /Path "data.value" not found in JSON/
+                },
+                {
+                    name: 'invalid JSON string',
+                    path: 'data',
+                    input: 'invalid json',
+                    errorPattern: /Invalid JSON string/
+                },
+                {
+                    name: 'array index on object',
+                    path: 'data[0]',
+                    input: { data: { key: 'value' } },
+                    errorPattern: /Path "data\[0\]" not found in JSON/
+                },
+                {
+                    name: 'out of bounds array',
+                    path: 'items[10]',
+                    input: { items: ['a', 'b'] },
+                    errorPattern: /Path "items\[10\]" not found in JSON/
+                }
+            ]
+
+            test.each(errorCases)('should throw error for $name', async ({ path, input, errorPattern }) => {
+                const nodeData = createNodeData(`test-error`, {
+                    path: path,
+                    returnNullOnError: false
+                })
+                const tool = await nodeClass.init(nodeData, '')
+                await expect(tool._call({ json: input })).rejects.toThrow(errorPattern)
+            })
+        })
+
+        describe('Error handling - null mode', () => {
+            const nullCases = [
+                { name: 'path not found', path: 'missing.path', input: { data: 'value' } },
+                { name: 'invalid JSON string', path: 'data', input: 'invalid json' },
+                { name: 'null in path', path: 'data.nested.value', input: { data: { nested: null } } },
+                { name: 'empty array access', path: 'items[0]', input: { items: [] } },
+                { name: 'property on primitive', path: 'value.nested', input: { value: 'string' } }
+            ]
+
+            test.each(nullCases)('should return null for $name', async ({ path, input }) => {
+                const nodeData = createNodeData(`test-null`, {
+                    path: path,
+                    returnNullOnError: true
+                })
+                const tool = await nodeClass.init(nodeData, '')
+                const result = await tool._call({ json: input })
+                expect(result).toBe('null')
+            })
+
+            it('should still extract valid paths when returnNullOnError is true', async () => {
+                const nodeData = createNodeData('test-valid-null-mode', {
+                    path: 'data.value',
+                    returnNullOnError: true
+                })
+                const tool = await nodeClass.init(nodeData, '')
+                const result = await tool._call({
+                    json: { data: { value: 'test' } }
+                })
+                expect(result).toBe('test')
+            })
+        })
+
+        describe('Complex structures', () => {
+            it('should handle deeply nested arrays and objects', async () => {
+                const nodeData = createNodeData('test-complex', {
+                    path: 'users[0].addresses[1].city',
+                    returnNullOnError: false
+                })
+                const tool = await nodeClass.init(nodeData, '')
+                const result = await tool._call({
+                    json: {
+                        users: [
+                            {
+                                addresses: [{ city: 'New York' }, { city: 'Los Angeles' }]
+                            }
+                        ]
+                    }
+                })
+                expect(result).toBe('Los Angeles')
+            })
+        })
+    })
+})

packages/components/nodes/tools/JSONPathExtractor/JSONPathExtractor.ts

@@ -0,0 +1,125 @@
+import { z } from 'zod'
+import { StructuredTool } from '@langchain/core/tools'
+import { INode, INodeData, INodeParams } from '../../../src/Interface'
+import { getBaseClasses } from '../../../src/utils'
+import { get } from 'lodash'
+
+/**
+ * Tool that extracts values from JSON using path
+ */
+class JSONPathExtractorTool extends StructuredTool {
+    name = 'json_path_extractor'
+    description = 'Extract value from JSON using configured path'
+
+    schema = z.object({
+        json: z
+            .union([z.string().describe('JSON string'), z.record(z.any()).describe('JSON object'), z.array(z.any()).describe('JSON array')])
+            .describe('JSON data to extract value from')
+    })
+
+    private readonly path: string
+    private readonly returnNullOnError: boolean
+
+    constructor(path: string, returnNullOnError: boolean = false) {
+        super()
+        this.path = path
+        this.returnNullOnError = returnNullOnError
+    }
+
+    async _call({ json }: z.infer<typeof this.schema>): Promise<string> {
+        // Validate that path is configured
+        if (!this.path) {
+            if (this.returnNullOnError) {
+                return 'null'
+            }
+            throw new Error('No extraction path configured')
+        }
+
+        let data: any
+
+        // Parse JSON string if needed
+        if (typeof json === 'string') {
+            try {
+                data = JSON.parse(json)
+            } catch (error) {
+                if (this.returnNullOnError) {
+                    return 'null'
+                }
+                throw new Error(`Invalid JSON string: ${error instanceof Error ? error.message : 'Parse error'}`)
+            }
+        } else {
+            data = json
+        }
+
+        // Extract value using lodash get
+        const value = get(data, this.path)
+
+        if (value === undefined) {
+            if (this.returnNullOnError) {
+                return 'null'
+            }
+            const jsonPreview = JSON.stringify(data, null, 2)
+            const preview = jsonPreview.length > 200 ? jsonPreview.substring(0, 200) + '...' : jsonPreview
+            throw new Error(`Path "${this.path}" not found in JSON. Received: ${preview}`)
+        }
+
+        return typeof value === 'string' ? value : JSON.stringify(value)
+    }
+}
+
+/**
+ * Node implementation for JSON Path Extractor tool
+ */
+class JSONPathExtractor_Tools implements INode {
+    label: string
+    name: string
+    version: number
+    type: string
+    icon: string
+    category: string
+    description: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'JSON Path Extractor'
+        this.name = 'jsonPathExtractor'
+        this.version = 1.0
+        this.type = 'JSONPathExtractor'
+        this.icon = 'jsonpathextractor.svg'
+        this.category = 'Tools'
+        this.description = 'Extract values from JSON using path expressions'
+        this.baseClasses = [this.type, ...getBaseClasses(JSONPathExtractorTool)]
+        this.inputs = [
+            {
+                label: 'JSON Path',
+                name: 'path',
+                type: 'string',
+                description: 'Path to extract. Examples: data, user.name, items[0].id',
+                placeholder: 'data'
+            },
+            {
+                label: 'Return Null on Error',
+                name: 'returnNullOnError',
+                type: 'boolean',
+                default: false,
+                description: 'Return null instead of throwing error when extraction fails',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+    }
+
+    async init(nodeData: INodeData, _: string): Promise<any> {
+        const path = (nodeData.inputs?.path as string) || ''
+        const returnNullOnError = (nodeData.inputs?.returnNullOnError as boolean) || false
+
+        if (!path) {
+            throw new Error('JSON Path is required')
+        }
+
+        return new JSONPathExtractorTool(path, returnNullOnError)
+    }
+}
+
+module.exports = { nodeClass: JSONPathExtractor_Tools }

packages/components/nodes/tools/JSONPathExtractor/jsonpathextractor.svg

@@ -0,0 +1,17 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none">
+    <!-- JSON document -->
+    <rect x="4" y="3" width="16" height="18" rx="2" stroke="currentColor" stroke-width="2" fill="none"/>
+    
+    <!-- Lines representing JSON content -->
+    <line x1="7" y1="7" x2="11" y2="7" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/>
+    <line x1="7" y1="10" x2="9" y2="10" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/>
+    <line x1="11" y1="10" x2="15" y2="10" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/>
+    <line x1="7" y1="13" x2="9" y2="13" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"/>
+    
+    <!-- Magnifying glass for extraction -->
+    <circle cx="15" cy="15" r="3" stroke="currentColor" stroke-width="2" fill="currentColor" fill-opacity="0.1"/>
+    <path d="M17.5 17.5l2 2" stroke="currentColor" stroke-width="2" stroke-linecap="round"/>
+    
+    <!-- Dot indicating selected value -->
+    <circle cx="15" cy="15" r="1" fill="currentColor"/>
+</svg>

packages/components/package.json

@@ -8,7 +8,10 @@
         "build": "tsc && gulp",
         "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
         "clean": "rimraf dist",
-        "nuke": "rimraf dist node_modules .turbo"
+        "nuke": "rimraf dist node_modules .turbo",
+        "test": "jest",
+        "test:watch": "jest --watch",
+        "test:coverage": "jest --coverage"
     },
     "keywords": [],
     "homepage": "https://flowiseai.com",
@@ -150,6 +153,7 @@
         "@swc/core": "^1.3.99",
         "@types/crypto-js": "^4.1.1",
         "@types/gulp": "4.0.9",
+        "@types/jest": "^29.5.14",
         "@types/lodash": "^4.17.20",
         "@types/node-fetch": "2.6.2",
         "@types/object-hash": "^3.0.2",
@@ -157,7 +161,9 @@
         "@types/pg": "^8.10.2",
         "@types/ws": "^8.5.3",
         "gulp": "^4.0.2",
+        "jest": "^29.7.0",
         "rimraf": "^5.0.5",
+        "ts-jest": "^29.3.2",
         "tsc-watch": "^6.0.4",
         "tslib": "^2.6.2",
         "typescript": "^5.4.5"

packages/components/tsconfig.json

@@ -17,5 +17,5 @@
         "module": "commonjs"
     },
     "include": ["src", "nodes", "credentials"],
-    "exclude": ["gulpfile.ts", "node_modules", "dist"]
+    "exclude": ["gulpfile.ts", "node_modules", "dist", "**/*.test.ts", "**/*.test.js", "**/*.spec.ts", "**/*.spec.js"]
 }