Enhance Airtable Document Loader with Filter and Text Output (#3074)
This commit is contained in:
Rafael Reis
GitHub
parent
66acd0c000
commit
43f11f21f8
|
|
@ -4,7 +4,8 @@ import { Document } from '@langchain/core/documents'
|
|||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
|
||||
import { getCredentialData, getCredentialParam } from '../../../src/utils'
|
||||
import { IDocument, ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
|
||||
import { IDocument, ICommonObject, INode, INodeData, INodeParams, INodeOutputsValue } from '../../../src/Interface'
|
||||
import { handleEscapeCharacters } from '../../../src'
|
||||
|
||||
class Airtable_DocumentLoaders implements INode {
|
||||
label: string
|
||||
|
|
@ -17,11 +18,12 @@ class Airtable_DocumentLoaders implements INode {
|
|||
baseClasses: string[]
|
||||
credential: INodeParams
|
||||
inputs?: INodeParams[]
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Airtable'
|
||||
this.name = 'airtable'
|
||||
this.version = 3.0
|
||||
this.version = 3.02
|
||||
this.type = 'Document'
|
||||
this.icon = 'airtable.svg'
|
||||
this.category = 'Document Loaders'
|
||||
|
|
@ -111,6 +113,30 @@ class Airtable_DocumentLoaders implements INode {
|
|||
placeholder: 'key1, key2, key3.nestedKey1',
|
||||
optional: true,
|
||||
additionalParams: true
|
||||
},
|
||||
{
|
||||
label: 'Filter By Formula',
|
||||
name: 'filterByFormula',
|
||||
type: 'string',
|
||||
placeholder: 'NOT({Id} = "")',
|
||||
optional: true,
|
||||
additionalParams: true,
|
||||
description:
|
||||
'A formula used to filter records. The formula will be evaluated for each record, and if the result is not 0, false, "", NaN, [], or #Error! the record will be included in the response.'
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
description: 'Array of document objects containing metadata and pageContent',
|
||||
baseClasses: [...this.baseClasses, 'json']
|
||||
},
|
||||
{
|
||||
label: 'Text',
|
||||
name: 'text',
|
||||
description: 'Concatenated string from pageContent of documents',
|
||||
baseClasses: ['string', 'json']
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -125,6 +151,7 @@ class Airtable_DocumentLoaders implements INode {
|
|||
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
|
||||
const metadata = nodeData.inputs?.metadata
|
||||
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
|
||||
const filterByFormula = nodeData.inputs?.filterByFormula as string
|
||||
|
||||
let omitMetadataKeys: string[] = []
|
||||
if (_omitMetadataKeys) {
|
||||
|
|
@ -141,7 +168,8 @@ class Airtable_DocumentLoaders implements INode {
|
|||
fields,
|
||||
returnAll,
|
||||
accessToken,
|
||||
limit: limit ? parseInt(limit, 10) : 100
|
||||
limit: limit ? parseInt(limit, 10) : 100,
|
||||
filterByFormula
|
||||
}
|
||||
|
||||
const loader = new AirtableLoader(airtableOptions)
|
||||
|
|
@ -191,6 +219,16 @@ class Airtable_DocumentLoaders implements INode {
|
|||
}))
|
||||
}
|
||||
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
if (output === 'text') {
|
||||
let finalText = ''
|
||||
for (const doc of docs) {
|
||||
finalText += `${doc.pageContent}\n`
|
||||
}
|
||||
return handleEscapeCharacters(finalText, false)
|
||||
}
|
||||
|
||||
return docs
|
||||
}
|
||||
}
|
||||
|
|
@ -203,6 +241,7 @@ interface AirtableLoaderParams {
|
|||
fields?: string[]
|
||||
limit?: number
|
||||
returnAll?: boolean
|
||||
filterByFormula?: string
|
||||
}
|
||||
|
||||
interface AirtableLoaderRequest {
|
||||
|
|
@ -210,6 +249,7 @@ interface AirtableLoaderRequest {
|
|||
view: string | undefined
|
||||
fields?: string[]
|
||||
offset?: string
|
||||
filterByFormula?: string
|
||||
}
|
||||
|
||||
interface AirtableLoaderResponse {
|
||||
|
|
@ -238,7 +278,18 @@ class AirtableLoader extends BaseDocumentLoader {
|
|||
|
||||
public readonly returnAll: boolean
|
||||
|
||||
constructor({ baseId, tableId, viewId, fields = [], accessToken, limit = 100, returnAll = false }: AirtableLoaderParams) {
|
||||
public readonly filterByFormula?: string
|
||||
|
||||
constructor({
|
||||
baseId,
|
||||
tableId,
|
||||
viewId,
|
||||
fields = [],
|
||||
accessToken,
|
||||
limit = 100,
|
||||
returnAll = false,
|
||||
filterByFormula
|
||||
}: AirtableLoaderParams) {
|
||||
super()
|
||||
this.baseId = baseId
|
||||
this.tableId = tableId
|
||||
|
|
@ -247,6 +298,7 @@ class AirtableLoader extends BaseDocumentLoader {
|
|||
this.accessToken = accessToken
|
||||
this.limit = limit
|
||||
this.returnAll = returnAll
|
||||
this.filterByFormula = filterByFormula
|
||||
}
|
||||
|
||||
public async load(): Promise<IDocument[]> {
|
||||
|
|
@ -297,6 +349,10 @@ class AirtableLoader extends BaseDocumentLoader {
|
|||
data.fields = this.fields
|
||||
}
|
||||
|
||||
if (this.filterByFormula) {
|
||||
data.filterByFormula = this.filterByFormula
|
||||
}
|
||||
|
||||
let response: AirtableLoaderResponse
|
||||
let returnPages: AirtableLoaderPage[] = []
|
||||
|
||||
|
|
@ -327,6 +383,10 @@ class AirtableLoader extends BaseDocumentLoader {
|
|||
data.fields = this.fields
|
||||
}
|
||||
|
||||
if (this.filterByFormula) {
|
||||
data.filterByFormula = this.filterByFormula
|
||||
}
|
||||
|
||||
let response: AirtableLoaderResponse
|
||||
let returnPages: AirtableLoaderPage[] = []
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue