import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' import { BaseDocumentLoader } from 'langchain/document_loaders/base' import { Document } from 'langchain/document' import axios from 'axios' import { getCredentialData, getCredentialParam } from '../../../src/utils' class Airtable_DocumentLoaders implements INode { label: string name: string version: number description: string type: string icon: string category: string baseClasses: string[] credential: INodeParams inputs?: INodeParams[] constructor() { this.label = 'Airtable' this.name = 'airtable' this.version = 2.0 this.type = 'Document' this.icon = 'airtable.svg' this.category = 'Document Loaders' this.description = `Load data from Airtable table` this.baseClasses = [this.type] this.credential = { label: 'Connect Credential', name: 'credential', type: 'credential', credentialNames: ['airtableApi'] } this.inputs = [ { label: 'Text Splitter', name: 'textSplitter', type: 'TextSplitter', optional: true }, { label: 'Base Id', name: 'baseId', type: 'string', placeholder: 'app11RobdGoX0YNsC', description: 'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id' }, { label: 'Table Id', name: 'tableId', type: 'string', placeholder: 'tblJdmvbrgizbYICO', description: 'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id' }, { label: 'View Id', name: 'viewId', type: 'string', placeholder: 'viw9UrP77Id0CE4ee', description: 'If your view URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, viw9UrP77Id0CE4ee is the view id', optional: true }, { label: 'Return All', name: 'returnAll', type: 'boolean', default: true, additionalParams: true, description: 'If all results should be returned or only up to a given limit' }, { label: 'Limit', name: 'limit', type: 'number', default: 100, additionalParams: true, description: 'Number of results to return' }, { label: 'Metadata', name: 'metadata', type: 'json', optional: true, additionalParams: true } ] } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { const baseId = nodeData.inputs?.baseId as string const tableId = nodeData.inputs?.tableId as string const viewId = nodeData.inputs?.viewId as string const returnAll = nodeData.inputs?.returnAll as boolean const limit = nodeData.inputs?.limit as string const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const metadata = nodeData.inputs?.metadata const credentialData = await getCredentialData(nodeData.credential ?? '', options) const accessToken = getCredentialParam('accessToken', credentialData, nodeData) const airtableOptions: AirtableLoaderParams = { baseId, tableId, viewId, returnAll, accessToken, limit: limit ? parseInt(limit, 10) : 100 } const loader = new AirtableLoader(airtableOptions) let docs = [] if (textSplitter) { docs = await loader.loadAndSplit(textSplitter) } else { docs = await loader.load() } if (metadata) { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) let finaldocs = [] for (const doc of docs) { const newdoc = { ...doc, metadata: { ...doc.metadata, ...parsedMetadata } } finaldocs.push(newdoc) } return finaldocs } return docs } } interface AirtableLoaderParams { baseId: string tableId: string accessToken: string viewId?: string limit?: number returnAll?: boolean } interface AirtableLoaderResponse { records: AirtableLoaderPage[] offset?: string } interface AirtableLoaderPage { id: string createdTime: string fields: ICommonObject } class AirtableLoader extends BaseDocumentLoader { public readonly baseId: string public readonly tableId: string public readonly viewId?: string public readonly accessToken: string public readonly limit: number public readonly returnAll: boolean constructor({ baseId, tableId, viewId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) { super() this.baseId = baseId this.tableId = tableId this.viewId = viewId this.accessToken = accessToken this.limit = limit this.returnAll = returnAll } public async load(): Promise { if (this.returnAll) { return this.loadAll() } return this.loadLimit() } protected async fetchAirtableData(url: string, params: ICommonObject): Promise { try { const headers = { Authorization: `Bearer ${this.accessToken}`, 'Content-Type': 'application/json', Accept: 'application/json' } const response = await axios.get(url, { params, headers }) return response.data } catch (error) { throw new Error(`Failed to fetch ${url} from Airtable: ${error}`) } } private createDocumentFromPage(page: AirtableLoaderPage): Document { // Generate the URL const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}` // Return a langchain document return new Document({ pageContent: JSON.stringify(page.fields, null, 2), metadata: { url: pageUrl } }) } private async loadLimit(): Promise { const params = { maxRecords: this.limit, view: this.viewId } const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params) if (data.records.length === 0) { return [] } return data.records.map((page) => this.createDocumentFromPage(page)) } private async loadAll(): Promise { const params: ICommonObject = { pageSize: 100, view: this.viewId } let data: AirtableLoaderResponse let returnPages: AirtableLoaderPage[] = [] do { data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params) returnPages.push.apply(returnPages, data.records) params.offset = data.offset } while (data.offset !== undefined) return returnPages.map((page) => this.createDocumentFromPage(page)) } } module.exports = { nodeClass: Airtable_DocumentLoaders }