From 736c2b11a1ea3bef22065cc6bc86d8ce022ce74b Mon Sep 17 00:00:00 2001 From: anatolii burtsev Date: Thu, 11 Sep 2025 14:34:12 -0700 Subject: [PATCH] feat: Add AWS DynamoDB KV Storage tool (#5111) * feat: Add AWS DynamoDB KV Storage tool - Add AWS DynamoDB key-value storage tool for persistent data storage - Add utility functions for AWS tools configuration * update SNS tool --------- Co-authored-by: Henry --- .../AWSDynamoDBKVStorage.test.ts | 479 ++++++++++++++++++ .../AWSDynamoDBKVStorage.ts | 375 ++++++++++++++ .../dynamodbkvstorage.svg | 29 ++ .../components/nodes/tools/AWSSNS/AWSSNS.ts | 85 +--- packages/components/src/awsToolsUtils.ts | 65 +++ 5 files changed, 959 insertions(+), 74 deletions(-) create mode 100644 packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.test.ts create mode 100644 packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.ts create mode 100644 packages/components/nodes/tools/AWSDynamoDBKVStorage/dynamodbkvstorage.svg create mode 100644 packages/components/src/awsToolsUtils.ts diff --git a/packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.test.ts b/packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.test.ts new file mode 100644 index 000000000..c69b48aeb --- /dev/null +++ b/packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.test.ts @@ -0,0 +1,479 @@ +// Mock AWS SDK DynamoDB client +jest.mock('@aws-sdk/client-dynamodb', () => { + const mockSend = jest.fn() + + // Create mock constructors that capture inputs + const PutItemCommandMock = jest.fn((input) => ({ input, _type: 'PutItemCommand' })) + const QueryCommandMock = jest.fn((input) => ({ input, _type: 'QueryCommand' })) + + return { + DynamoDBClient: jest.fn().mockImplementation(() => ({ + send: mockSend + })), + DescribeTableCommand: jest.fn(), + ListTablesCommand: jest.fn(), + PutItemCommand: PutItemCommandMock, + QueryCommand: QueryCommandMock, + __mockSend: mockSend + } +}) + +// Mock AWS credentials utility +jest.mock('../../../src/awsToolsUtils', () => ({ + AWS_REGIONS: [ + { label: 'US East (N. Virginia)', name: 'us-east-1' }, + { label: 'US West (Oregon)', name: 'us-west-2' } + ], + DEFAULT_AWS_REGION: 'us-east-1', + getAWSCredentials: jest.fn(() => + Promise.resolve({ + accessKeyId: 'test-access-key', + secretAccessKey: 'test-secret-key', + sessionToken: 'test-session-token' + }) + ) +})) + +// Mock getBaseClasses function +jest.mock('../../../src/utils', () => ({ + getBaseClasses: jest.fn(() => ['Tool', 'StructuredTool']) +})) + +describe('AWSDynamoDBKVStorage', () => { + let AWSDynamoDBKVStorage_Tools: any + let mockSend: jest.Mock + let PutItemCommandMock: jest.Mock + let QueryCommandMock: jest.Mock + + // Helper function to create a node instance + const createNode = () => new AWSDynamoDBKVStorage_Tools() + + // Helper function to create nodeData + const createNodeData = (overrides = {}) => ({ + inputs: { + region: 'us-east-1', + tableName: 'test-table', + keyPrefix: '', + operation: 'store', + ...overrides + } + }) + + beforeEach(async () => { + // Clear all mocks before each test + jest.clearAllMocks() + + // Get the mock functions + const dynamoDBModule = require('@aws-sdk/client-dynamodb') + mockSend = dynamoDBModule.__mockSend + PutItemCommandMock = dynamoDBModule.PutItemCommand + QueryCommandMock = dynamoDBModule.QueryCommand + + mockSend.mockReset() + PutItemCommandMock.mockClear() + QueryCommandMock.mockClear() + + // Dynamic import to get fresh module instance + const module = (await import('./AWSDynamoDBKVStorage')) as any + AWSDynamoDBKVStorage_Tools = module.nodeClass + }) + + describe('AWSDynamoDBKVStorage_Tools Node', () => { + it('should have correct input parameters', () => { + const node = createNode() + const inputNames = node.inputs.map((input: any) => input.name) + + expect(inputNames).toEqual(['region', 'tableName', 'keyPrefix', 'operation']) + }) + }) + + describe('loadMethods - listTables', () => { + it('should list valid DynamoDB tables with correct schema', async () => { + const node = createNode() + + // Mock responses for list and describe commands + mockSend + .mockResolvedValueOnce({ + TableNames: ['table1', 'table2', 'invalid-table'] + }) + .mockResolvedValueOnce({ + Table: { + KeySchema: [ + { AttributeName: 'pk', KeyType: 'HASH' }, + { AttributeName: 'sk', KeyType: 'RANGE' } + ] + } + }) + .mockResolvedValueOnce({ + Table: { + KeySchema: [ + { AttributeName: 'pk', KeyType: 'HASH' }, + { AttributeName: 'sk', KeyType: 'RANGE' } + ] + } + }) + .mockResolvedValueOnce({ + Table: { + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }] + } + }) + + const nodeData = { inputs: { region: 'us-east-1' } } + + const result = await node.loadMethods.listTables(nodeData, {}) + + expect(result).toEqual([ + { + label: 'table1', + name: 'table1', + description: 'Table with pk (partition) and sk (sort) keys' + }, + { + label: 'table2', + name: 'table2', + description: 'Table with pk (partition) and sk (sort) keys' + } + ]) + }) + + it('should return error when no tables found', async () => { + const node = createNode() + + mockSend.mockResolvedValueOnce({ + TableNames: [] + }) + + const nodeData = { inputs: { region: 'us-east-1' } } + + const result = await node.loadMethods.listTables(nodeData, {}) + + expect(result).toEqual([ + { + label: 'No tables found', + name: 'error', + description: 'No DynamoDB tables found in this region' + } + ]) + }) + + it('should return error when no compatible tables found', async () => { + const node = createNode() + + mockSend + .mockResolvedValueOnce({ + TableNames: ['invalid-table'] + }) + .mockResolvedValueOnce({ + Table: { + KeySchema: [{ AttributeName: 'id', KeyType: 'HASH' }] + } + }) + + const nodeData = { inputs: { region: 'us-east-1' } } + + const result = await node.loadMethods.listTables(nodeData, {}) + + expect(result).toHaveLength(1) + expect(result[0]).toMatchObject({ + label: 'No compatible tables found', + name: 'error' + }) + expect(result[0].description).toContain('Found 1 table(s) with different schema') + }) + + it('should handle AWS credentials error', async () => { + const node = createNode() + const { getAWSCredentials } = require('../../../src/awsToolsUtils') + + getAWSCredentials.mockRejectedValueOnce(new Error('AWS Access Key not found')) + + const nodeData = { inputs: { region: 'us-east-1' } } + + const result = await node.loadMethods.listTables(nodeData, {}) + + expect(result).toEqual([ + { + label: 'AWS Credentials Required', + name: 'error', + description: 'Enter AWS Access Key ID and Secret Access Key' + } + ]) + }) + }) + + describe('init method', () => { + it.each([ + ['store', 'test-prefix', 'dynamodb_kv_store', 'Store a text value with a key in DynamoDB'], + ['retrieve', '', 'dynamodb_kv_retrieve', 'Retrieve a value by key from DynamoDB'] + ])('should create correct tool for %s operation', async (operation, keyPrefix, expectedName, expectedDescription) => { + const node = createNode() + const nodeData = createNodeData({ keyPrefix, operation }) + + const tool = await node.init(nodeData, '', {}) + + expect(tool.name).toBe(expectedName) + expect(tool.description).toContain(expectedDescription) + }) + + it.each([ + ['error', '', 'Valid DynamoDB Table selection is required'], + ['test-table', 'prefix#invalid', 'Key prefix cannot contain "#" character'] + ])('should throw error for invalid config (table: %s, prefix: %s)', async (tableName, keyPrefix, expectedError) => { + const node = createNode() + const nodeData = createNodeData({ tableName, keyPrefix }) + + await expect(node.init(nodeData, '', {})).rejects.toThrow(expectedError) + }) + }) + + describe('DynamoDBStoreTool', () => { + it('should store value successfully', async () => { + const node = createNode() + + mockSend.mockResolvedValueOnce({}) + + const nodeData = createNodeData({ keyPrefix: 'test' }) + + const tool = await node.init(nodeData, '', {}) + const result = await tool._call({ key: 'mykey', value: 'myvalue' }) + + expect(result).toContain('Successfully stored value with key "mykey"') + expect(mockSend).toHaveBeenCalledTimes(1) + + // Verify PutItemCommand was called with correct parameters + expect(PutItemCommandMock).toHaveBeenCalledTimes(1) + const putCommandInput = PutItemCommandMock.mock.calls[0][0] + + expect(putCommandInput).toMatchObject({ + TableName: 'test-table', + Item: { + pk: { S: 'test#mykey' }, + value: { S: 'myvalue' } + } + }) + + // Verify timestamp fields exist + expect(putCommandInput.Item.sk).toBeDefined() + expect(putCommandInput.Item.timestamp).toBeDefined() + }) + + it.each([ + ['', 'Key must be a non-empty string'], + [' ', 'Key must be a non-empty string'], + ['a'.repeat(2049), 'Key too long'] + ])('should handle invalid key: "%s"', async (key, expectedError) => { + const node = createNode() + + const nodeData = createNodeData() + + const tool = await node.init(nodeData, '', {}) + await expect(tool._call({ key, value: 'myvalue' })).rejects.toThrow(expectedError) + }) + + it.each([ + ['store', { key: 'mykey', value: 'myvalue' }, 'Failed to store value: DynamoDB error'], + ['retrieve', { key: 'mykey' }, 'Failed to retrieve value: DynamoDB error'] + ])('should handle DynamoDB error for %s', async (operation, callParams, expectedError) => { + const node = createNode() + mockSend.mockRejectedValueOnce(new Error('DynamoDB error')) + + const nodeData = createNodeData({ operation }) + const tool = await node.init(nodeData, '', {}) + + await expect(tool._call(callParams)).rejects.toThrow(expectedError) + }) + }) + + describe('DynamoDBRetrieveTool', () => { + it('should retrieve latest value successfully', async () => { + const node = createNode() + + mockSend.mockResolvedValueOnce({ + Items: [ + { + pk: { S: 'test#mykey' }, + sk: { S: '1234567890' }, + value: { S: 'myvalue' }, + timestamp: { S: '2024-01-01T00:00:00.000Z' } + } + ] + }) + + const nodeData = createNodeData({ keyPrefix: 'test', operation: 'retrieve' }) + + const tool = await node.init(nodeData, '', {}) + const result = await tool._call({ key: 'mykey' }) + const parsed = JSON.parse(result) + + expect(parsed).toEqual({ + value: 'myvalue', + timestamp: '2024-01-01T00:00:00.000Z' + }) + expect(mockSend).toHaveBeenCalledTimes(1) + + // Verify QueryCommand was called with correct parameters + expect(QueryCommandMock).toHaveBeenCalledTimes(1) + const queryCommandInput = QueryCommandMock.mock.calls[0][0] + + expect(queryCommandInput).toMatchObject({ + TableName: 'test-table', + KeyConditionExpression: 'pk = :pk', + ExpressionAttributeValues: { + ':pk': { S: 'test#mykey' } + }, + ScanIndexForward: false, + Limit: 1 + }) + }) + + it('should retrieve nth latest value', async () => { + const node = createNode() + + mockSend.mockResolvedValueOnce({ + Items: [ + { + pk: { S: 'mykey' }, + sk: { S: '1234567892' }, + value: { S: 'newest' }, + timestamp: { S: '2024-01-03T00:00:00.000Z' } + }, + { + pk: { S: 'mykey' }, + sk: { S: '1234567891' }, + value: { S: 'second' }, + timestamp: { S: '2024-01-02T00:00:00.000Z' } + }, + { + pk: { S: 'mykey' }, + sk: { S: '1234567890' }, + value: { S: 'oldest' }, + timestamp: { S: '2024-01-01T00:00:00.000Z' } + } + ] + }) + + const nodeData = createNodeData({ operation: 'retrieve' }) + + const tool = await node.init(nodeData, '', {}) + const result = await tool._call({ key: 'mykey', nthLatest: '2' }) + const parsed = JSON.parse(result) + + expect(parsed).toEqual({ + value: 'second', + timestamp: '2024-01-02T00:00:00.000Z' + }) + + // Verify QueryCommand was called with Limit: 2 + expect(QueryCommandMock).toHaveBeenCalledTimes(1) + const queryCommandInput = QueryCommandMock.mock.calls[0][0] + expect(queryCommandInput.Limit).toBe(2) + }) + + it('should return null when key not found', async () => { + const node = createNode() + + mockSend.mockResolvedValueOnce({ + Items: [] + }) + + const nodeData = createNodeData({ operation: 'retrieve' }) + + const tool = await node.init(nodeData, '', {}) + const result = await tool._call({ key: 'nonexistent' }) + const parsed = JSON.parse(result) + + expect(parsed).toEqual({ + value: null, + timestamp: null + }) + }) + + it('should return null when nth version does not exist', async () => { + const node = createNode() + + mockSend.mockResolvedValueOnce({ + Items: [ + { + pk: { S: 'mykey' }, + sk: { S: '1234567890' }, + value: { S: 'only-one' }, + timestamp: { S: '2024-01-01T00:00:00.000Z' } + } + ] + }) + + const nodeData = createNodeData({ operation: 'retrieve' }) + + const tool = await node.init(nodeData, '', {}) + const result = await tool._call({ key: 'mykey', nthLatest: '3' }) + const parsed = JSON.parse(result) + + expect(parsed).toEqual({ + value: null, + timestamp: null + }) + }) + + it.each([ + ['0', 'nthLatest must be a positive number'], + ['-1', 'nthLatest must be a positive number'] + ])('should reject invalid nthLatest value "%s"', async (nthLatest, expectedError) => { + const node = createNode() + + const nodeData = createNodeData({ operation: 'retrieve' }) + + const tool = await node.init(nodeData, '', {}) + await expect(tool._call({ key: 'mykey', nthLatest })).rejects.toThrow(expectedError) + }) + + it.each([ + ['', 'Key must be a non-empty string'], + [' ', 'Key must be a non-empty string'] + ])('should handle invalid key for retrieve: "%s"', async (key, expectedError) => { + const node = createNode() + + const nodeData = createNodeData({ operation: 'retrieve' }) + + const tool = await node.init(nodeData, '', {}) + await expect(tool._call({ key })).rejects.toThrow(expectedError) + }) + }) + + describe('Helper Functions', () => { + it.each([ + ['myapp', 'userdata', 'myapp#userdata'], + ['', 'userdata', 'userdata'] + ])('should build full key correctly (prefix: "%s", key: "%s", expected: "%s")', async (keyPrefix, key, expectedFullKey) => { + const node = createNode() + mockSend.mockResolvedValueOnce({}) + const nodeData = createNodeData({ keyPrefix }) + + const tool = await node.init(nodeData, '', {}) + await tool._call({ key, value: 'test' }) + + // Verify the put command was called with the correct full key + expect(mockSend).toHaveBeenCalledTimes(1) + expect(PutItemCommandMock).toHaveBeenCalledTimes(1) + + const putCommandInput = PutItemCommandMock.mock.calls[0][0] + expect(putCommandInput.Item.pk.S).toBe(expectedFullKey) + }) + + it.each([ + [{ accessKeyId: 'test-key', secretAccessKey: 'test-secret', sessionToken: 'test-token' }, 'with session token'], + [{ accessKeyId: 'test-key', secretAccessKey: 'test-secret' }, 'without session token'] + ])('should work %s', async (credentials, _description) => { + const node = createNode() + const { getAWSCredentials } = require('../../../src/awsToolsUtils') + + getAWSCredentials.mockResolvedValueOnce(credentials) + mockSend.mockResolvedValueOnce({}) + + const nodeData = createNodeData() + + const tool = await node.init(nodeData, '', {}) + await tool._call({ key: 'test', value: 'value' }) + expect(getAWSCredentials).toHaveBeenCalled() + }) + }) +}) diff --git a/packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.ts b/packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.ts new file mode 100644 index 000000000..6d1141592 --- /dev/null +++ b/packages/components/nodes/tools/AWSDynamoDBKVStorage/AWSDynamoDBKVStorage.ts @@ -0,0 +1,375 @@ +import { z } from 'zod' +import { StructuredTool } from '@langchain/core/tools' +import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { AWS_REGIONS, DEFAULT_AWS_REGION, AWSCredentials, getAWSCredentials } from '../../../src/awsToolsUtils' +import { DynamoDBClient, DescribeTableCommand, ListTablesCommand, PutItemCommand, QueryCommand } from '@aws-sdk/client-dynamodb' + +// Operation enum +enum Operation { + STORE = 'store', + RETRIEVE = 'retrieve' +} + +// Constants +const ERROR_PLACEHOLDER = 'error' +const KEY_SEPARATOR = '#' +const MAX_KEY_LENGTH = 2048 // DynamoDB limit for partition key + +// Helper function to create DynamoDB client +function createDynamoDBClient(credentials: AWSCredentials, region: string): DynamoDBClient { + return new DynamoDBClient({ + region, + credentials: { + accessKeyId: credentials.accessKeyId, + secretAccessKey: credentials.secretAccessKey, + ...(credentials.sessionToken && { sessionToken: credentials.sessionToken }) + } + }) +} + +// Helper function to build full key with optional prefix +function buildFullKey(key: string, keyPrefix: string): string { + const fullKey = keyPrefix ? `${keyPrefix}${KEY_SEPARATOR}${key}` : key + + // Validate key length (DynamoDB limit) + if (fullKey.length > MAX_KEY_LENGTH) { + throw new Error(`Key too long. Maximum length is ${MAX_KEY_LENGTH} characters, got ${fullKey.length}`) + } + + return fullKey +} + +// Helper function to validate and sanitize input +function validateKey(key: string): void { + if (!key || key.trim().length === 0) { + throw new Error('Key must be a non-empty string') + } +} + +/** + * Tool for storing key-value pairs in DynamoDB with automatic versioning + */ +class DynamoDBStoreTool extends StructuredTool { + name = 'dynamodb_kv_store' + description = 'Store a text value with a key in DynamoDB. Input must be an object with "key" and "value" properties.' + schema = z.object({ + key: z.string().min(1).describe('The key to store the value under'), + value: z.string().describe('The text value to store') + }) + private readonly dynamoClient: DynamoDBClient + private readonly tableName: string + private readonly keyPrefix: string + + constructor(dynamoClient: DynamoDBClient, tableName: string, keyPrefix: string = '') { + super() + this.dynamoClient = dynamoClient + this.tableName = tableName + this.keyPrefix = keyPrefix + } + + async _call({ key, value }: z.infer): Promise { + try { + validateKey(key) + const fullKey = buildFullKey(key, this.keyPrefix) + const timestamp = Date.now() + const isoTimestamp = new Date(timestamp).toISOString() + + const putCommand = new PutItemCommand({ + TableName: this.tableName, + Item: { + pk: { S: fullKey }, + sk: { S: timestamp.toString() }, + value: { S: value }, + timestamp: { S: isoTimestamp } + } + }) + + await this.dynamoClient.send(putCommand) + return `Successfully stored value with key "${key}" at ${isoTimestamp}` + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + throw new Error(`Failed to store value: ${errorMessage}`) + } + } +} + +/** + * Tool for retrieving key-value pairs from DynamoDB with version control + */ +class DynamoDBRetrieveTool extends StructuredTool { + name = 'dynamodb_kv_retrieve' + description = + 'Retrieve a value by key from DynamoDB. Returns JSON with value and timestamp. Specify which version to get (1=latest, 2=2nd latest, etc).' + schema = z.object({ + key: z.string().min(1).describe('The key to retrieve the value for'), + nthLatest: z + .string() + .regex(/^\d+$/, 'Must be a positive number') + .describe( + 'Which version to retrieve: "1" for latest, "2" for 2nd latest, "3" for 3rd latest, etc. Use "1" to get the most recent value.' + ) + .optional() + .default('1') + }) + private readonly dynamoClient: DynamoDBClient + private readonly tableName: string + private readonly keyPrefix: string + + constructor(dynamoClient: DynamoDBClient, tableName: string, keyPrefix: string = '') { + super() + this.dynamoClient = dynamoClient + this.tableName = tableName + this.keyPrefix = keyPrefix + } + + async _call(input: z.infer): Promise { + try { + const { key, nthLatest = '1' } = input + validateKey(key) + const fullKey = buildFullKey(key, this.keyPrefix) + + // Convert string to number and validate + const nthLatestNum = parseInt(nthLatest, 10) + if (isNaN(nthLatestNum) || nthLatestNum < 1) { + throw new Error('nthLatest must be a positive number (1 or greater)') + } + + const queryCommand = new QueryCommand({ + TableName: this.tableName, + KeyConditionExpression: 'pk = :pk', + ExpressionAttributeValues: { + ':pk': { S: fullKey } + }, + ScanIndexForward: false, // Sort descending (newest first) + Limit: nthLatestNum + }) + + const result = await this.dynamoClient.send(queryCommand) + + if (!result.Items || result.Items.length === 0) { + return JSON.stringify({ + value: null, + timestamp: null + }) + } + + if (result.Items.length < nthLatestNum) { + return JSON.stringify({ + value: null, + timestamp: null + }) + } + + const item = result.Items[nthLatestNum - 1] + const value = item.value?.S || null + const timestamp = item.timestamp?.S || item.sk?.S || null + + // Return JSON with value and timestamp + return JSON.stringify({ + value: value, + timestamp: timestamp + }) + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + throw new Error(`Failed to retrieve value: ${errorMessage}`) + } + } +} + +/** + * Node implementation for AWS DynamoDB KV Storage tools + */ +class AWSDynamoDBKVStorage_Tools implements INode { + label: string + name: string + version: number + type: string + icon: string + category: string + description: string + baseClasses: string[] + credential: INodeParams + inputs: INodeParams[] + + constructor() { + this.label = 'AWS DynamoDB KV Storage' + this.name = 'awsDynamoDBKVStorage' + this.version = 1.0 + this.type = 'AWSDynamoDBKVStorage' + this.icon = 'dynamodbkvstorage.svg' + this.category = 'Tools' + this.description = 'Store and retrieve versioned text values in AWS DynamoDB' + this.baseClasses = [this.type, ...getBaseClasses(DynamoDBStoreTool)] + this.credential = { + label: 'AWS Credentials', + name: 'credential', + type: 'credential', + credentialNames: ['awsApi'] + } + this.inputs = [ + { + label: 'AWS Region', + name: 'region', + type: 'options', + options: AWS_REGIONS, + default: DEFAULT_AWS_REGION, + description: 'AWS Region where your DynamoDB tables are located' + }, + { + label: 'DynamoDB Table', + name: 'tableName', + type: 'asyncOptions', + loadMethod: 'listTables', + description: 'Select a DynamoDB table with partition key "pk" and sort key "sk"', + refresh: true + }, + { + label: 'Key Prefix', + name: 'keyPrefix', + type: 'string', + description: 'Optional prefix to add to all keys (e.g., "myapp" would make keys like "myapp#userdata")', + optional: true, + additionalParams: true + }, + { + label: 'Operation', + name: 'operation', + type: 'options', + options: [ + { label: 'Store', name: Operation.STORE }, + { label: 'Retrieve', name: Operation.RETRIEVE } + ], + default: Operation.STORE, + description: 'Choose whether to store or retrieve data' + } + ] + } + + loadMethods: Record Promise> = { + listTables: async (nodeData: INodeData, options?: ICommonObject): Promise => { + try { + const credentials = await getAWSCredentials(nodeData, options ?? {}) + const region = (nodeData.inputs?.region as string) || DEFAULT_AWS_REGION + const dynamoClient = createDynamoDBClient(credentials, region) + + const listCommand = new ListTablesCommand({}) + const listResponse = await dynamoClient.send(listCommand) + + if (!listResponse.TableNames || listResponse.TableNames.length === 0) { + return [ + { + label: 'No tables found', + name: ERROR_PLACEHOLDER, + description: 'No DynamoDB tables found in this region' + } + ] + } + + const validTables: INodeOptionsValue[] = [] + const invalidTables: string[] = [] + + // Check tables in parallel for better performance + const tableChecks = await Promise.allSettled( + listResponse.TableNames.map(async (tableName) => { + const describeCommand = new DescribeTableCommand({ + TableName: tableName + }) + const describeResponse = await dynamoClient.send(describeCommand) + + const keySchema = describeResponse.Table?.KeySchema + if (keySchema) { + const hasPk = keySchema.some((key) => key.AttributeName === 'pk' && key.KeyType === 'HASH') + const hasSk = keySchema.some((key) => key.AttributeName === 'sk' && key.KeyType === 'RANGE') + + if (hasPk && hasSk) { + return { + valid: true, + table: { + label: tableName, + name: tableName, + description: `Table with pk (partition) and sk (sort) keys` + } + } + } + } + return { valid: false, tableName } + }) + ) + + tableChecks.forEach((result) => { + if (result.status === 'fulfilled') { + if (result.value.valid) { + validTables.push(result.value.table!) + } else if (result.value.tableName) { + invalidTables.push(result.value.tableName) + } + } + }) + + if (validTables.length === 0) { + return [ + { + label: 'No compatible tables found', + name: ERROR_PLACEHOLDER, + description: `No tables with partition key "pk" and sort key "sk" found. ${ + invalidTables.length > 0 ? `Found ${invalidTables.length} table(s) with different schema.` : '' + } Please create a table with these keys.` + } + ] + } + + // Sort tables alphabetically + validTables.sort((a, b) => a.label.localeCompare(b.label)) + + return validTables + } catch (error) { + if (error instanceof Error && error.message.includes('AWS Access Key')) { + return [ + { + label: 'AWS Credentials Required', + name: ERROR_PLACEHOLDER, + description: 'Enter AWS Access Key ID and Secret Access Key' + } + ] + } + console.error('Error loading DynamoDB tables:', error) + return [ + { + label: 'Error Loading Tables', + name: ERROR_PLACEHOLDER, + description: `Failed to load tables: ${error instanceof Error ? error.message : String(error)}` + } + ] + } + } + } + + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentials = await getAWSCredentials(nodeData, options) + + const region = (nodeData.inputs?.region as string) || DEFAULT_AWS_REGION + const tableName = nodeData.inputs?.tableName as string + const keyPrefix = (nodeData.inputs?.keyPrefix as string) || '' + const operation = (nodeData.inputs?.operation as string) || Operation.STORE + + if (!tableName || tableName === ERROR_PLACEHOLDER) { + throw new Error('Valid DynamoDB Table selection is required') + } + + // Validate key prefix doesn't contain separator + if (keyPrefix && keyPrefix.includes(KEY_SEPARATOR)) { + throw new Error(`Key prefix cannot contain "${KEY_SEPARATOR}" character`) + } + + const dynamoClient = createDynamoDBClient(credentials, region) + + if (operation === Operation.STORE) { + return new DynamoDBStoreTool(dynamoClient, tableName, keyPrefix) + } else { + return new DynamoDBRetrieveTool(dynamoClient, tableName, keyPrefix) + } + } +} + +module.exports = { nodeClass: AWSDynamoDBKVStorage_Tools } diff --git a/packages/components/nodes/tools/AWSDynamoDBKVStorage/dynamodbkvstorage.svg b/packages/components/nodes/tools/AWSDynamoDBKVStorage/dynamodbkvstorage.svg new file mode 100644 index 000000000..3912d7a8f --- /dev/null +++ b/packages/components/nodes/tools/AWSDynamoDBKVStorage/dynamodbkvstorage.svg @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + K:V + \ No newline at end of file diff --git a/packages/components/nodes/tools/AWSSNS/AWSSNS.ts b/packages/components/nodes/tools/AWSSNS/AWSSNS.ts index a25d04640..8dc09d8ee 100644 --- a/packages/components/nodes/tools/AWSSNS/AWSSNS.ts +++ b/packages/components/nodes/tools/AWSSNS/AWSSNS.ts @@ -1,6 +1,7 @@ import { Tool } from '@langchain/core/tools' import { ICommonObject, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface' -import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils' +import { getBaseClasses } from '../../../src/utils' +import { AWS_REGIONS, DEFAULT_AWS_REGION, getAWSCredentials } from '../../../src/awsToolsUtils' import { SNSClient, ListTopicsCommand, PublishCommand } from '@aws-sdk/client-sns' class AWSSNSTool extends Tool { @@ -62,30 +63,8 @@ class AWSSNS_Tools implements INode { label: 'AWS Region', name: 'region', type: 'options', - options: [ - { label: 'US East (N. Virginia) - us-east-1', name: 'us-east-1' }, - { label: 'US East (Ohio) - us-east-2', name: 'us-east-2' }, - { label: 'US West (N. California) - us-west-1', name: 'us-west-1' }, - { label: 'US West (Oregon) - us-west-2', name: 'us-west-2' }, - { label: 'Africa (Cape Town) - af-south-1', name: 'af-south-1' }, - { label: 'Asia Pacific (Hong Kong) - ap-east-1', name: 'ap-east-1' }, - { label: 'Asia Pacific (Mumbai) - ap-south-1', name: 'ap-south-1' }, - { label: 'Asia Pacific (Osaka) - ap-northeast-3', name: 'ap-northeast-3' }, - { label: 'Asia Pacific (Seoul) - ap-northeast-2', name: 'ap-northeast-2' }, - { label: 'Asia Pacific (Singapore) - ap-southeast-1', name: 'ap-southeast-1' }, - { label: 'Asia Pacific (Sydney) - ap-southeast-2', name: 'ap-southeast-2' }, - { label: 'Asia Pacific (Tokyo) - ap-northeast-1', name: 'ap-northeast-1' }, - { label: 'Canada (Central) - ca-central-1', name: 'ca-central-1' }, - { label: 'Europe (Frankfurt) - eu-central-1', name: 'eu-central-1' }, - { label: 'Europe (Ireland) - eu-west-1', name: 'eu-west-1' }, - { label: 'Europe (London) - eu-west-2', name: 'eu-west-2' }, - { label: 'Europe (Milan) - eu-south-1', name: 'eu-south-1' }, - { label: 'Europe (Paris) - eu-west-3', name: 'eu-west-3' }, - { label: 'Europe (Stockholm) - eu-north-1', name: 'eu-north-1' }, - { label: 'Middle East (Bahrain) - me-south-1', name: 'me-south-1' }, - { label: 'South America (São Paulo) - sa-east-1', name: 'sa-east-1' } - ], - default: 'us-east-1', + options: AWS_REGIONS, + default: DEFAULT_AWS_REGION, description: 'AWS Region where your SNS topics are located' }, { @@ -103,32 +82,8 @@ class AWSSNS_Tools implements INode { loadMethods = { listTopics: async (nodeData: INodeData, options?: ICommonObject): Promise => { try { - const credentialData = await getCredentialData(nodeData.credential ?? '', options ?? {}) - - const accessKeyId = getCredentialParam('awsKey', credentialData, nodeData) - const secretAccessKey = getCredentialParam('awsSecret', credentialData, nodeData) - const sessionToken = getCredentialParam('awsSession', credentialData, nodeData) - - const region = (nodeData.inputs?.region as string) || 'us-east-1' - - if (!accessKeyId || !secretAccessKey) { - return [ - { - label: 'AWS Credentials Required', - name: 'placeholder', - description: 'Enter AWS Access Key ID and Secret Access Key' - } - ] - } - - const credentials: any = { - accessKeyId: accessKeyId, - secretAccessKey: secretAccessKey - } - - if (sessionToken) { - credentials.sessionToken = sessionToken - } + const credentials = await getAWSCredentials(nodeData, options ?? {}) + const region = (nodeData.inputs?.region as string) || DEFAULT_AWS_REGION const snsClient = new SNSClient({ region: region, @@ -161,9 +116,9 @@ class AWSSNS_Tools implements INode { console.error('Error loading SNS topics:', error) return [ { - label: 'Error Loading Topics', - name: 'error', - description: `Failed to load topics: ${error}` + label: 'AWS Credentials Required', + name: 'placeholder', + description: 'Enter AWS Access Key ID and Secret Access Key' } ] } @@ -171,32 +126,14 @@ class AWSSNS_Tools implements INode { } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const credentialData = await getCredentialData(nodeData.credential ?? '', options) - - const accessKeyId = getCredentialParam('awsKey', credentialData, nodeData) - const secretAccessKey = getCredentialParam('awsSecret', credentialData, nodeData) - const sessionToken = getCredentialParam('awsSession', credentialData, nodeData) - - const region = (nodeData.inputs?.region as string) || 'us-east-1' + const credentials = await getAWSCredentials(nodeData, options) + const region = (nodeData.inputs?.region as string) || DEFAULT_AWS_REGION const topicArn = nodeData.inputs?.topicArn as string - if (!accessKeyId || !secretAccessKey) { - throw new Error('AWS Access Key ID and Secret Access Key are required') - } - if (!topicArn) { throw new Error('SNS Topic ARN is required') } - const credentials: any = { - accessKeyId: accessKeyId, - secretAccessKey: secretAccessKey - } - - if (sessionToken) { - credentials.sessionToken = sessionToken - } - const snsClient = new SNSClient({ region: region, credentials: credentials diff --git a/packages/components/src/awsToolsUtils.ts b/packages/components/src/awsToolsUtils.ts new file mode 100644 index 000000000..46edafeff --- /dev/null +++ b/packages/components/src/awsToolsUtils.ts @@ -0,0 +1,65 @@ +import { ICommonObject, INodeData } from './Interface' +import { getCredentialData, getCredentialParam } from './utils' + +// AWS Regions constant +export const AWS_REGIONS = [ + { label: 'US East (N. Virginia) - us-east-1', name: 'us-east-1' }, + { label: 'US East (Ohio) - us-east-2', name: 'us-east-2' }, + { label: 'US West (N. California) - us-west-1', name: 'us-west-1' }, + { label: 'US West (Oregon) - us-west-2', name: 'us-west-2' }, + { label: 'Africa (Cape Town) - af-south-1', name: 'af-south-1' }, + { label: 'Asia Pacific (Hong Kong) - ap-east-1', name: 'ap-east-1' }, + { label: 'Asia Pacific (Mumbai) - ap-south-1', name: 'ap-south-1' }, + { label: 'Asia Pacific (Osaka) - ap-northeast-3', name: 'ap-northeast-3' }, + { label: 'Asia Pacific (Seoul) - ap-northeast-2', name: 'ap-northeast-2' }, + { label: 'Asia Pacific (Singapore) - ap-southeast-1', name: 'ap-southeast-1' }, + { label: 'Asia Pacific (Sydney) - ap-southeast-2', name: 'ap-southeast-2' }, + { label: 'Asia Pacific (Tokyo) - ap-northeast-1', name: 'ap-northeast-1' }, + { label: 'Canada (Central) - ca-central-1', name: 'ca-central-1' }, + { label: 'Europe (Frankfurt) - eu-central-1', name: 'eu-central-1' }, + { label: 'Europe (Ireland) - eu-west-1', name: 'eu-west-1' }, + { label: 'Europe (London) - eu-west-2', name: 'eu-west-2' }, + { label: 'Europe (Milan) - eu-south-1', name: 'eu-south-1' }, + { label: 'Europe (Paris) - eu-west-3', name: 'eu-west-3' }, + { label: 'Europe (Stockholm) - eu-north-1', name: 'eu-north-1' }, + { label: 'Middle East (Bahrain) - me-south-1', name: 'me-south-1' }, + { label: 'South America (São Paulo) - sa-east-1', name: 'sa-east-1' } +] + +export const DEFAULT_AWS_REGION = 'us-east-1' + +// AWS Credentials interface +export interface AWSCredentials { + accessKeyId: string + secretAccessKey: string + sessionToken?: string +} + +/** + * Get AWS credentials from node data + * @param {INodeData} nodeData - Node data containing credential information + * @param {ICommonObject} options - Options containing appDataSource and databaseEntities + * @returns {Promise} - AWS credentials object + */ +export async function getAWSCredentials(nodeData: INodeData, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + + const accessKeyId = getCredentialParam('awsKey', credentialData, nodeData) + const secretAccessKey = getCredentialParam('awsSecret', credentialData, nodeData) + const sessionToken = getCredentialParam('awsSession', credentialData, nodeData) + + if (!accessKeyId || !secretAccessKey) { + throw new Error('AWS Access Key ID and Secret Access Key are required') + } + + const credentials: AWSCredentials = { + accessKeyId, + secretAccessKey + } + + if (sessionToken) { + credentials.sessionToken = sessionToken + } + + return credentials +}