Merge pull request #590 from FlowiseAI/feature/Airtable

Feature/AirtableLoader

packages/components/nodes/documentloaders/Airtable/Airtable.ts

@@ -0,0 +1,226 @@
+import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
+import { TextSplitter } from 'langchain/text_splitter'
+import { BaseDocumentLoader } from 'langchain/document_loaders/base'
+import { Document } from 'langchain/document'
+import axios from 'axios'
+
+class Airtable_DocumentLoaders implements INode {
+    label: string
+    name: string
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs?: INodeParams[]
+
+    constructor() {
+        this.label = 'Airtable'
+        this.name = 'airtable'
+        this.type = 'Document'
+        this.icon = 'airtable.svg'
+        this.category = 'Document Loaders'
+        this.description = `Load data from Airtable table`
+        this.baseClasses = [this.type]
+        this.inputs = [
+            {
+                label: 'Text Splitter',
+                name: 'textSplitter',
+                type: 'TextSplitter',
+                optional: true
+            },
+            {
+                label: 'Personal Access Token',
+                name: 'accessToken',
+                type: 'password',
+                description:
+                    'Get personal access token from <a target="_blank" href="https://airtable.com/developers/web/guides/personal-access-tokens">official guide</a>'
+            },
+            {
+                label: 'Base Id',
+                name: 'baseId',
+                type: 'string',
+                placeholder: 'app11RobdGoX0YNsC',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id'
+            },
+            {
+                label: 'Table Id',
+                name: 'tableId',
+                type: 'string',
+                placeholder: 'tblJdmvbrgizbYICO',
+                description:
+                    'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id'
+            },
+            {
+                label: 'Return All',
+                name: 'returnAll',
+                type: 'boolean',
+                default: true,
+                additionalParams: true,
+                description: 'If all results should be returned or only up to a given limit'
+            },
+            {
+                label: 'Limit',
+                name: 'limit',
+                type: 'number',
+                default: 100,
+                step: 1,
+                additionalParams: true,
+                description: 'Number of results to return'
+            },
+            {
+                label: 'Metadata',
+                name: 'metadata',
+                type: 'json',
+                optional: true,
+                additionalParams: true
+            }
+        ]
+    }
+    async init(nodeData: INodeData): Promise<any> {
+        const accessToken = nodeData.inputs?.accessToken as string
+        const baseId = nodeData.inputs?.baseId as string
+        const tableId = nodeData.inputs?.tableId as string
+        const returnAll = nodeData.inputs?.returnAll as boolean
+        const limit = nodeData.inputs?.limit as string
+        const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
+        const metadata = nodeData.inputs?.metadata
+
+        const options: AirtableLoaderParams = {
+            baseId,
+            tableId,
+            returnAll,
+            accessToken,
+            limit: limit ? parseInt(limit, 10) : 100
+        }
+
+        const loader = new AirtableLoader(options)
+
+        let docs = []
+
+        if (textSplitter) {
+            docs = await loader.loadAndSplit(textSplitter)
+        } else {
+            docs = await loader.load()
+        }
+
+        if (metadata) {
+            const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
+            let finaldocs = []
+            for (const doc of docs) {
+                const newdoc = {
+                    ...doc,
+                    metadata: {
+                        ...doc.metadata,
+                        ...parsedMetadata
+                    }
+                }
+                finaldocs.push(newdoc)
+            }
+            return finaldocs
+        }
+
+        return docs
+    }
+}
+
+interface AirtableLoaderParams {
+    baseId: string
+    tableId: string
+    accessToken: string
+    limit?: number
+    returnAll?: boolean
+}
+
+interface AirtableLoaderResponse {
+    records: AirtableLoaderPage[]
+    offset?: string
+}
+
+interface AirtableLoaderPage {
+    id: string
+    createdTime: string
+    fields: ICommonObject
+}
+
+class AirtableLoader extends BaseDocumentLoader {
+    public readonly baseId: string
+
+    public readonly tableId: string
+
+    public readonly accessToken: string
+
+    public readonly limit: number
+
+    public readonly returnAll: boolean
+
+    constructor({ baseId, tableId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) {
+        super()
+        this.baseId = baseId
+        this.tableId = tableId
+        this.accessToken = accessToken
+        this.limit = limit
+        this.returnAll = returnAll
+    }
+
+    public async load(): Promise<Document[]> {
+        if (this.returnAll) {
+            return this.loadAll()
+        }
+        return this.loadLimit()
+    }
+
+    protected async fetchAirtableData(url: string, params: ICommonObject): Promise<AirtableLoaderResponse> {
+        try {
+            const headers = {
+                Authorization: `Bearer ${this.accessToken}`,
+                'Content-Type': 'application/json',
+                Accept: 'application/json'
+            }
+            const response = await axios.get(url, { params, headers })
+            return response.data
+        } catch (error) {
+            throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
+        }
+    }
+
+    private createDocumentFromPage(page: AirtableLoaderPage): Document {
+        // Generate the URL
+        const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}`
+
+        // Return a langchain document
+        return new Document({
+            pageContent: JSON.stringify(page.fields, null, 2),
+            metadata: {
+                url: pageUrl
+            }
+        })
+    }
+
+    private async loadLimit(): Promise<Document[]> {
+        const params = { maxRecords: this.limit }
+        const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+        if (data.records.length === 0) {
+            return []
+        }
+        return data.records.map((page) => this.createDocumentFromPage(page))
+    }
+
+    private async loadAll(): Promise<Document[]> {
+        const params: ICommonObject = { pageSize: 100 }
+        let data: AirtableLoaderResponse
+        let returnPages: AirtableLoaderPage[] = []
+
+        do {
+            data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
+            returnPages.push.apply(returnPages, data.records)
+            params.offset = data.offset
+        } while (data.offset !== undefined)
+        return returnPages.map((page) => this.createDocumentFromPage(page))
+    }
+}
+
+module.exports = {
+    nodeClass: Airtable_DocumentLoaders
+}

packages/components/nodes/documentloaders/Airtable/airtable.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="256px" height="215px" viewBox="0 0 256 215" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
+	<g>
+		<path d="M114.25873,2.70101695 L18.8604023,42.1756384 C13.5552723,44.3711638 13.6102328,51.9065311 18.9486282,54.0225085 L114.746142,92.0117514 C123.163769,95.3498757 132.537419,95.3498757 140.9536,92.0117514 L236.75256,54.0225085 C242.08951,51.9065311 242.145916,44.3711638 236.83934,42.1756384 L141.442459,2.70101695 C132.738459,-0.900338983 122.961284,-0.900338983 114.25873,2.70101695" fill="#FFBF00"></path>
+		<path d="M136.349071,112.756863 L136.349071,207.659101 C136.349071,212.173089 140.900664,215.263892 145.096461,213.600615 L251.844122,172.166219 C254.281184,171.200072 255.879376,168.845451 255.879376,166.224705 L255.879376,71.3224678 C255.879376,66.8084791 251.327783,63.7176768 247.131986,65.3809537 L140.384325,106.815349 C137.94871,107.781496 136.349071,110.136118 136.349071,112.756863" fill="#26B5F8"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L76.5257763,134.504714 L9.65047684,166.548104 C5.4112904,168.593211 0.000578531073,165.503855 0.000578531073,160.794612 L0.000578531073,71.7210757 C0.000578531073,70.0173017 0.874160452,68.5463864 2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill="#ED3049"></path>
+		<path d="M111.422771,117.65355 L79.742409,132.949912 L2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill-opacity="0.25" fill="#000000"></path>
+	</g>
+</svg>

packages/components/src/Interface.ts

@@ -60,6 +60,7 @@ export interface INodeParams {
     warning?: string
     options?: Array<INodeOptionsValue>
     optional?: boolean | INodeDisplay
+    step?: number
     rows?: number
     list?: boolean
     acceptVariable?: boolean

packages/ui/src/ui-component/input/Input.js

@@ -37,7 +37,7 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo
                         onChange(e.target.value)
                     }}
                     inputProps={{
-                        step: 0.1,
+                        step: inputParam.step ?? 0.1,
                         style: {
                             height: inputParam.rows ? '90px' : 'inherit'
                         }