Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update console statements to use logger

This commit is contained in:
Ilango 2024-01-22 08:49:04 +05:30
parent c24708f53b
commit 193e5c4640
3 changed files with 24 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
View File

@ -1,4 +1,4 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter' import { TextSplitter } from 'langchain/text_splitter'
import { CheerioWebBaseLoader, WebBaseLoaderParams } from 'langchain/document_loaders/web/cheerio' import { CheerioWebBaseLoader, WebBaseLoaderParams } from 'langchain/document_loaders/web/cheerio'
import { test } from 'linkifyjs' import { test } from 'linkifyjs'
@ -87,7 +87,7 @@ class Cheerio_DocumentLoaders implements INode {
] ]
} }
async init(nodeData: INodeData): Promise<any> { async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata const metadata = nodeData.inputs?.metadata
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
@ -119,13 +119,13 @@ class Cheerio_DocumentLoaders implements INode {
} }
return docs return docs
} catch (err) { } catch (err) {
if (process.env.DEBUG === 'true') console.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`) if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
} }
} }
let docs = [] let docs = []
if (relativeLinksMethod) { if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10 if (!limit) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0') else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] = const pages: string[] =
@ -134,14 +134,15 @@ class Cheerio_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl' : relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit) ? await webCrawl(url, limit)
: await xmlScrape(url, limit) : await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found') if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) { for (const page of pages) {
docs.push(...(await cheerioLoader(page))) docs.push(...(await cheerioLoader(page)))
} }
if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`) if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) { } else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`) if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) { for (const page of selectedLinks) {
docs.push(...(await cheerioLoader(page))) docs.push(...(await cheerioLoader(page)))
} }

15
packages/components/nodes/documentloaders/Playwright/Playwright.ts
View File

@ -1,4 +1,4 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter' import { TextSplitter } from 'langchain/text_splitter'
import { Browser, Page, PlaywrightWebBaseLoader, PlaywrightWebBaseLoaderOptions } from 'langchain/document_loaders/web/playwright' import { Browser, Page, PlaywrightWebBaseLoader, PlaywrightWebBaseLoaderOptions } from 'langchain/document_loaders/web/playwright'
import { test } from 'linkifyjs' import { test } from 'linkifyjs'
@ -115,7 +115,7 @@ class Playwright_DocumentLoaders implements INode {
] ]
} }
async init(nodeData: INodeData): Promise<any> { async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata const metadata = nodeData.inputs?.metadata
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
@ -160,13 +160,13 @@ class Playwright_DocumentLoaders implements INode {
} }
return docs return docs
} catch (err) { } catch (err) {
if (process.env.DEBUG === 'true') console.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`) if (process.env.DEBUG === 'true') options.logger.error(`error in PlaywrightWebBaseLoader: ${err.message}, on page: ${url}`)
} }
} }
let docs = [] let docs = []
if (relativeLinksMethod) { if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10 if (!limit) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0') else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] = const pages: string[] =
@ -175,14 +175,15 @@ class Playwright_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl' : relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit) ? await webCrawl(url, limit)
: await xmlScrape(url, limit) : await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found') if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) { for (const page of pages) {
docs.push(...(await playwrightLoader(page))) docs.push(...(await playwrightLoader(page)))
} }
if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`) if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) { } else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`) if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) { for (const page of selectedLinks) {
docs.push(...(await playwrightLoader(page))) docs.push(...(await playwrightLoader(page)))
} }

15
packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts
View File

@ -1,4 +1,4 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface' import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter' import { TextSplitter } from 'langchain/text_splitter'
import { Browser, Page, PuppeteerWebBaseLoader, PuppeteerWebBaseLoaderOptions } from 'langchain/document_loaders/web/puppeteer' import { Browser, Page, PuppeteerWebBaseLoader, PuppeteerWebBaseLoaderOptions } from 'langchain/document_loaders/web/puppeteer'
import { test } from 'linkifyjs' import { test } from 'linkifyjs'
@ -116,7 +116,7 @@ class Puppeteer_DocumentLoaders implements INode {
] ]
} }
async init(nodeData: INodeData): Promise<any> { async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const metadata = nodeData.inputs?.metadata const metadata = nodeData.inputs?.metadata
const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string const relativeLinksMethod = nodeData.inputs?.relativeLinksMethod as string
@ -161,13 +161,13 @@ class Puppeteer_DocumentLoaders implements INode {
} }
return docs return docs
} catch (err) { } catch (err) {
if (process.env.DEBUG === 'true') console.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`) if (process.env.DEBUG === 'true') options.logger.error(`error in PuppeteerWebBaseLoader: ${err.message}, on page: ${url}`)
} }
} }
let docs = [] let docs = []
if (relativeLinksMethod) { if (relativeLinksMethod) {
if (process.env.DEBUG === 'true') console.info(`Start ${relativeLinksMethod}`) if (process.env.DEBUG === 'true') options.logger.info(`Start ${relativeLinksMethod}`)
if (!limit) limit = 10 if (!limit) limit = 10
else if (limit < 0) throw new Error('Limit cannot be less than 0') else if (limit < 0) throw new Error('Limit cannot be less than 0')
const pages: string[] = const pages: string[] =
@ -176,14 +176,15 @@ class Puppeteer_DocumentLoaders implements INode {
: relativeLinksMethod === 'webCrawl' : relativeLinksMethod === 'webCrawl'
? await webCrawl(url, limit) ? await webCrawl(url, limit)
: await xmlScrape(url, limit) : await xmlScrape(url, limit)
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`) if (process.env.DEBUG === 'true') options.logger.info(`pages: ${JSON.stringify(pages)}, length: ${pages.length}`)
if (!pages || pages.length === 0) throw new Error('No relative links found') if (!pages || pages.length === 0) throw new Error('No relative links found')
for (const page of pages) { for (const page of pages) {
docs.push(...(await puppeteerLoader(page))) docs.push(...(await puppeteerLoader(page)))
} }
if (process.env.DEBUG === 'true') console.info(`Finish ${relativeLinksMethod}`) if (process.env.DEBUG === 'true') options.logger.info(`Finish ${relativeLinksMethod}`)
} else if (selectedLinks && selectedLinks.length > 0) { } else if (selectedLinks && selectedLinks.length > 0) {
if (process.env.DEBUG === 'true') console.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`) if (process.env.DEBUG === 'true')
options.logger.info(`pages: ${JSON.stringify(selectedLinks)}, length: ${selectedLinks.length}`)
for (const page of selectedLinks) { for (const page of selectedLinks) {
docs.push(...(await puppeteerLoader(page))) docs.push(...(await puppeteerLoader(page)))
} }