Merge pull request #590 from FlowiseAI/feature/Airtable
Feature/AirtableLoader
This commit is contained in:
Henry Heng
GitHub
commit
c8223d5273
|
|
@ -0,0 +1,226 @@
|
|||
import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
|
||||
import { Document } from 'langchain/document'
|
||||
import axios from 'axios'
|
||||
|
||||
class Airtable_DocumentLoaders implements INode {
|
||||
label: string
|
||||
name: string
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs?: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Airtable'
|
||||
this.name = 'airtable'
|
||||
this.type = 'Document'
|
||||
this.icon = 'airtable.svg'
|
||||
this.category = 'Document Loaders'
|
||||
this.description = `Load data from Airtable table`
|
||||
this.baseClasses = [this.type]
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Text Splitter',
|
||||
name: 'textSplitter',
|
||||
type: 'TextSplitter',
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Personal Access Token',
|
||||
name: 'accessToken',
|
||||
type: 'password',
|
||||
description:
|
||||
'Get personal access token from <a target="_blank" href="https://airtable.com/developers/web/guides/personal-access-tokens">official guide</a>'
|
||||
},
|
||||
{
|
||||
label: 'Base Id',
|
||||
name: 'baseId',
|
||||
type: 'string',
|
||||
placeholder: 'app11RobdGoX0YNsC',
|
||||
description:
|
||||
'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, app11RovdGoX0YNsC is the base id'
|
||||
},
|
||||
{
|
||||
label: 'Table Id',
|
||||
name: 'tableId',
|
||||
type: 'string',
|
||||
placeholder: 'tblJdmvbrgizbYICO',
|
||||
description:
|
||||
'If your table URL looks like: https://airtable.com/app11RobdGoX0YNsC/tblJdmvbrgizbYICO/viw9UrP77Id0CE4ee, tblJdmvbrgizbYICO is the table id'
|
||||
},
|
||||
{
|
||||
label: 'Return All',
|
||||
name: 'returnAll',
|
||||
type: 'boolean',
|
||||
default: true,
|
||||
additionalParams: true,
|
||||
description: 'If all results should be returned or only up to a given limit'
|
||||
},
|
||||
{
|
||||
label: 'Limit',
|
||||
name: 'limit',
|
||||
type: 'number',
|
||||
default: 100,
|
||||
step: 1,
|
||||
additionalParams: true,
|
||||
description: 'Number of results to return'
|
||||
},
|
||||
{
|
||||
label: 'Metadata',
|
||||
name: 'metadata',
|
||||
type: 'json',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
}
|
||||
]
|
||||
}
|
||||
async init(nodeData: INodeData): Promise<any> {
|
||||
const accessToken = nodeData.inputs?.accessToken as string
|
||||
const baseId = nodeData.inputs?.baseId as string
|
||||
const tableId = nodeData.inputs?.tableId as string
|
||||
const returnAll = nodeData.inputs?.returnAll as boolean
|
||||
const limit = nodeData.inputs?.limit as string
|
||||
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
|
||||
const metadata = nodeData.inputs?.metadata
|
||||
|
||||
const options: AirtableLoaderParams = {
|
||||
baseId,
|
||||
tableId,
|
||||
returnAll,
|
||||
accessToken,
|
||||
limit: limit ? parseInt(limit, 10) : 100
|
||||
}
|
||||
|
||||
const loader = new AirtableLoader(options)
|
||||
|
||||
let docs = []
|
||||
|
||||
if (textSplitter) {
|
||||
docs = await loader.loadAndSplit(textSplitter)
|
||||
} else {
|
||||
docs = await loader.load()
|
||||
}
|
||||
|
||||
if (metadata) {
|
||||
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
|
||||
let finaldocs = []
|
||||
for (const doc of docs) {
|
||||
const newdoc = {
|
||||
...doc,
|
||||
metadata: {
|
||||
...doc.metadata,
|
||||
...parsedMetadata
|
||||
}
|
||||
}
|
||||
finaldocs.push(newdoc)
|
||||
}
|
||||
return finaldocs
|
||||
}
|
||||
|
||||
return docs
|
||||
}
|
||||
}
|
||||
|
||||
interface AirtableLoaderParams {
|
||||
baseId: string
|
||||
tableId: string
|
||||
accessToken: string
|
||||
limit?: number
|
||||
returnAll?: boolean
|
||||
}
|
||||
|
||||
interface AirtableLoaderResponse {
|
||||
records: AirtableLoaderPage[]
|
||||
offset?: string
|
||||
}
|
||||
|
||||
interface AirtableLoaderPage {
|
||||
id: string
|
||||
createdTime: string
|
||||
fields: ICommonObject
|
||||
}
|
||||
|
||||
class AirtableLoader extends BaseDocumentLoader {
|
||||
public readonly baseId: string
|
||||
|
||||
public readonly tableId: string
|
||||
|
||||
public readonly accessToken: string
|
||||
|
||||
public readonly limit: number
|
||||
|
||||
public readonly returnAll: boolean
|
||||
|
||||
constructor({ baseId, tableId, accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) {
|
||||
super()
|
||||
this.baseId = baseId
|
||||
this.tableId = tableId
|
||||
this.accessToken = accessToken
|
||||
this.limit = limit
|
||||
this.returnAll = returnAll
|
||||
}
|
||||
|
||||
public async load(): Promise<Document[]> {
|
||||
if (this.returnAll) {
|
||||
return this.loadAll()
|
||||
}
|
||||
return this.loadLimit()
|
||||
}
|
||||
|
||||
protected async fetchAirtableData(url: string, params: ICommonObject): Promise<AirtableLoaderResponse> {
|
||||
try {
|
||||
const headers = {
|
||||
Authorization: `Bearer ${this.accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json'
|
||||
}
|
||||
const response = await axios.get(url, { params, headers })
|
||||
return response.data
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to fetch ${url} from Airtable: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
private createDocumentFromPage(page: AirtableLoaderPage): Document {
|
||||
// Generate the URL
|
||||
const pageUrl = `https://api.airtable.com/v0/${this.baseId}/${this.tableId}/${page.id}`
|
||||
|
||||
// Return a langchain document
|
||||
return new Document({
|
||||
pageContent: JSON.stringify(page.fields, null, 2),
|
||||
metadata: {
|
||||
url: pageUrl
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private async loadLimit(): Promise<Document[]> {
|
||||
const params = { maxRecords: this.limit }
|
||||
const data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
|
||||
if (data.records.length === 0) {
|
||||
return []
|
||||
}
|
||||
return data.records.map((page) => this.createDocumentFromPage(page))
|
||||
}
|
||||
|
||||
private async loadAll(): Promise<Document[]> {
|
||||
const params: ICommonObject = { pageSize: 100 }
|
||||
let data: AirtableLoaderResponse
|
||||
let returnPages: AirtableLoaderPage[] = []
|
||||
|
||||
do {
|
||||
data = await this.fetchAirtableData(`https://api.airtable.com/v0/${this.baseId}/${this.tableId}`, params)
|
||||
returnPages.push.apply(returnPages, data.records)
|
||||
params.offset = data.offset
|
||||
} while (data.offset !== undefined)
|
||||
return returnPages.map((page) => this.createDocumentFromPage(page))
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
nodeClass: Airtable_DocumentLoaders
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg width="256px" height="215px" viewBox="0 0 256 215" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid">
|
||||
<g>
|
||||
<path d="M114.25873,2.70101695 L18.8604023,42.1756384 C13.5552723,44.3711638 13.6102328,51.9065311 18.9486282,54.0225085 L114.746142,92.0117514 C123.163769,95.3498757 132.537419,95.3498757 140.9536,92.0117514 L236.75256,54.0225085 C242.08951,51.9065311 242.145916,44.3711638 236.83934,42.1756384 L141.442459,2.70101695 C132.738459,-0.900338983 122.961284,-0.900338983 114.25873,2.70101695" fill="#FFBF00"></path>
|
||||
<path d="M136.349071,112.756863 L136.349071,207.659101 C136.349071,212.173089 140.900664,215.263892 145.096461,213.600615 L251.844122,172.166219 C254.281184,171.200072 255.879376,168.845451 255.879376,166.224705 L255.879376,71.3224678 C255.879376,66.8084791 251.327783,63.7176768 247.131986,65.3809537 L140.384325,106.815349 C137.94871,107.781496 136.349071,110.136118 136.349071,112.756863" fill="#26B5F8"></path>
|
||||
<path d="M111.422771,117.65355 L79.742409,132.949912 L76.5257763,134.504714 L9.65047684,166.548104 C5.4112904,168.593211 0.000578531073,165.503855 0.000578531073,160.794612 L0.000578531073,71.7210757 C0.000578531073,70.0173017 0.874160452,68.5463864 2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill="#ED3049"></path>
|
||||
<path d="M111.422771,117.65355 L79.742409,132.949912 L2.04568588,67.4384994 C2.53454463,66.9481944 3.08848814,66.5446689 3.66412655,66.2250305 C5.26231864,65.2661153 7.54173107,65.0101153 9.47981017,65.7766689 L110.890522,105.957098 C116.045234,108.002206 116.450206,115.225166 111.422771,117.65355" fill-opacity="0.25" fill="#000000"></path>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
|
|
@ -60,6 +60,7 @@ export interface INodeParams {
|
|||
warning?: string
|
||||
options?: Array<INodeOptionsValue>
|
||||
optional?: boolean | INodeDisplay
|
||||
step?: number
|
||||
rows?: number
|
||||
list?: boolean
|
||||
acceptVariable?: boolean
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ export const Input = ({ inputParam, value, onChange, disabled = false, showDialo
|
|||
onChange(e.target.value)
|
||||
}}
|
||||
inputProps={{
|
||||
step: 0.1,
|
||||
step: inputParam.step ?? 0.1,
|
||||
style: {
|
||||
height: inputParam.rows ? '90px' : 'inherit'
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue