Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Add Custom Api Url Param for Self-hosted FireCrawl endpoint (#3480) 

* feat: Add Custom api url for Self-hoseted FireCrawl endpoint

* Update FireCrawlApi.credential.ts

---------

Co-authored-by: Henry Heng <henryheng@flowiseai.com>
This commit is contained in:
ZacharyQin 2024-11-09 08:44:17 +08:00 committed by GitHub
parent 835b1511d7
commit b01772302e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/components/credentials/FireCrawlApi.credential.ts
View File

@ -10,7 +10,7 @@ class FireCrawlApiCredential implements INodeCredential {
constructor() {
this.label = 'FireCrawl API'
this.name = 'fireCrawlApi'
this.version = 1.0
this.version = 2.0
this.description =
'You can find the FireCrawl API token on your <a target="_blank" href="https://www.firecrawl.dev/">FireCrawl account</a> page.'
this.inputs = [
@ -18,6 +18,12 @@ class FireCrawlApiCredential implements INodeCredential {
label: 'FireCrawl API',
name: 'firecrawlApiToken',
type: 'password'
},
{
label: 'FireCrawl API URL',
name: 'firecrawlApiUrl',
type: 'string',
default: 'https://api.firecrawl.dev'
}
]
}

10
packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts
View File

@ -194,19 +194,21 @@ class FirecrawlApp {
interface FirecrawlLoaderParameters {
url: string
apiKey?: string
apiUrl?: string
mode?: 'crawl' | 'scrape'
params?: Record<string, unknown>
}
class FireCrawlLoader extends BaseDocumentLoader {
private apiKey: string
private apiUrl: string
private url: string
private mode: 'crawl' | 'scrape'
private params?: Record<string, unknown>
constructor(loaderParams: FirecrawlLoaderParameters) {
super()
const { apiKey, url, mode = 'crawl', params } = loaderParams
const { apiKey, apiUrl, url, mode = 'crawl', params } = loaderParams
if (!apiKey) {
throw new Error('Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.')
}
@ -215,10 +217,11 @@ class FireCrawlLoader extends BaseDocumentLoader {
this.url = url
this.mode = mode
this.params = params
this.apiUrl = apiUrl || 'https://api.firecrawl.dev'
}
public async load(): Promise<DocumentInterface[]> {
const app = new FirecrawlApp({ apiKey: this.apiKey })
const app = new FirecrawlApp({ apiKey: this.apiKey, apiUrl: this.apiUrl })
let firecrawlDocs: FirecrawlDocument[]
if (this.mode === 'scrape') {
@ -319,6 +322,7 @@ class FireCrawl_DocumentLoaders implements INode {
const onlyMainContent = nodeData.inputs?.onlyMainContent as boolean
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const firecrawlApiToken = getCredentialParam('firecrawlApiToken', credentialData, nodeData)
const firecrawlApiUrl = getCredentialParam('firecrawlApiUrl', credentialData, nodeData, 'https://api.firecrawl.dev')
const urlPatternsExcludes = nodeData.inputs?.urlPatternsExcludes
? (nodeData.inputs.urlPatternsExcludes.split(',') as string[])
@ -331,6 +335,7 @@ class FireCrawl_DocumentLoaders implements INode {
url,
mode: crawlerType as 'crawl' | 'scrape',
apiKey: firecrawlApiToken,
apiUrl: firecrawlApiUrl,
params: {
crawlerOptions: {
includes: urlPatternsIncludes,
@ -344,7 +349,6 @@ class FireCrawl_DocumentLoaders implements INode {
}
}
}
const loader = new FireCrawlLoader(input)
let docs = []