diff --git a/packages/components/nodes/documentloaders/Airtable/Airtable.ts b/packages/components/nodes/documentloaders/Airtable/Airtable.ts new file mode 100644 index 000000000..9607693a1 --- /dev/null +++ b/packages/components/nodes/documentloaders/Airtable/Airtable.ts @@ -0,0 +1,226 @@ +import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { BaseDocumentLoader } from 'langchain/document_loaders/base' +import { Document } from 'langchain/document' +import axios from 'axios' + +class Airtable_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs?: INodeParams[] + + constructor() { + this.label = 'Airtable' + this.name = 'airtable' + this.type = 'Document' + this.icon = 'airtable.svg' + this.category = 'Document Loaders' + this.description = `Load data from Airtable table` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + }, + { + label: 'Personal Access Token', + name: 'accessToken', + type: 'password', + description: + 'Get personal access token from official guide' + }, + { + label: 'Base Id', + name: 'baseId', + type: 'string', + placeholder: 'app11RobdGoX0YNsC', + description: + 'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id' + }, + { + label: 'Table Id', + name: 'tableId', + type: 'string', + placeholder: 'tblJdmvbrgizbYICO', + description: + 'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id' + }, + { + label: 'Return All', + name: 'returnAll', + type: 'boolean', + default: true, + additionalParams: true, + description: 'If all results should be returned or only up to a given limit' + }, + { + label: 'Limit', + name: 'limit', + type: 'number', + default: 100, + step: 1, + additionalParams: true, + description: 'Number of results to return' + }, + { + label: 'Metadata', + name: 'metadata', + type: 'json', + optional: true, + additionalParams: true + } + ] + } + async init(nodeData: INodeData): Promise { + const accessToken = nodeData.inputs?.accessToken as string + const baseId = nodeData.inputs?.baseId as string + const tableId = nodeData.inputs?.tableId as string + const returnAll = nodeData.inputs?.returnAll as boolean + const limit = nodeData.inputs?.limit as string + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const metadata = nodeData.inputs?.metadata + + const options: AirtableLoaderParams = { + baseId, + tableId, + returnAll, + accessToken, + limit: limit ? parseInt(limit, 10) : 100 + } + + const loader = new AirtableLoader(options) + + let docs = [] + + if (textSplitter) { + docs = await loader.loadAndSplit(textSplitter) + } else { + docs = await loader.load() + } + + if (metadata) { + const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) + let finaldocs = [] + for (const doc of docs) { + const newdoc = { + ...doc, + metadata: { + ...doc.metadata, + ...parsedMetadata + } + } + finaldocs.push(newdoc) + } + return finaldocs + } + + return docs + } +} + +interface AirtableLoaderParams { + baseId: string + tableId: string + accessToken: string + limit?: number + returnAll?: boolean +} + +interface AirtableLoaderResponse { + records: AirtableLoaderPage[] + offset?: string +} + +interface AirtableLoaderPage { + id: string + createdTime: string + fields: ICommonObject +} + +class AirtableLoader extends BaseDocumentLoader { + public readonly baseId: string + + public readonly tableId: string + + public readonly accessToken: string + + public readonly limit: number + + public readonly returnAll: boolean + + constructor({ baseId, tableId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { + super() + this.baseId = baseId + this.tableId = tableId + this.accessToken = accessToken + this.limit = limit + this.returnAll = returnAll + } + + public async load(): Promise { + if (this.returnAll) { + return this.loadAll() + } + return this.loadLimit() + } + + protected async fetchAirtableData(url: string, params: ICommonObject): Promise { + try { + const headers = { + Authorization: `Bearer ${this.accessToken}`, + 'Content-Type': 'application/json', + Accept: 'application/json' + } + const response = await axios.get(url, { params, headers }) + return response.data + } catch (error) { + throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) + } + } + + private createDocumentFromPage(page: AirtableLoaderPage): Document { + // Generate the URL + const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}` + + // Return a langchain document + return new Document({ + pageContent: JSON.stringify(page.fields, null, 2), + metadata: { + url: pageUrl + } + }) + } + + private async loadLimit(): Promise { + const params = { maxRecords: this.limit } + const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params) + if (data.records.length === 0) { + return [] + } + return data.records.map((page) => this.createDocumentFromPage(page)) + } + + private async loadAll(): Promise { + const params: ICommonObject = { pageSize: 100 } + let data: AirtableLoaderResponse + let returnPages: AirtableLoaderPage[] = [] + + do { + data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params) + returnPages.push.apply(returnPages, data.records) + params.offset = data.offset + } while (data.offset !== undefined) + return returnPages.map((page) => this.createDocumentFromPage(page)) + } +} + +module.exports = { + nodeClass: Airtable_DocumentLoaders +} diff --git a/packages/components/nodes/documentloaders/Airtable/airtable.svg b/packages/components/nodes/documentloaders/Airtable/airtable.svg new file mode 100644 index 000000000..867c3b5ae --- /dev/null +++ b/packages/components/nodes/documentloaders/Airtable/airtable.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index 862b81ac8..47b5aba29 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -60,6 +60,7 @@ export interface INodeParams { warning?: string options?: Array optional?: boolean | INodeDisplay + step?: number rows?: number list?: boolean acceptVariable?: boolean diff --git a/packages/ui/src/ui-component/input/Input.js b/packages/ui/src/ui-component/input/Input.js index 5a7f45b7b..b7e161db5 100644 --- a/packages/ui/src/ui-component/input/Input.js +++ b/packages/ui/src/ui-component/input/Input.js @@ -37,7 +37,7 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo onChange(e.target.value) }} inputProps={{ - step: 0.1, + step: inputParam.step ?? 0.1, style: { height: inputParam.rows ? '90px' : 'inherit' }