Merge branch 'main' into fix/stripe-issues
This commit is contained in:
Ong Chung Yau
GitHub
commit
c74e1750f3
|
|
@ -0,0 +1,30 @@
|
|||
import { INodeParams, INodeCredential } from '../src/Interface'
|
||||
|
||||
class OxylabsApiCredential implements INodeCredential {
|
||||
label: string
|
||||
name: string
|
||||
version: number
|
||||
description: string
|
||||
inputs: INodeParams[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Oxylabs API'
|
||||
this.name = 'oxylabsApi'
|
||||
this.version = 1.0
|
||||
this.description = 'Oxylabs API credentials description, to add more info'
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Oxylabs Username',
|
||||
name: 'username',
|
||||
type: 'string'
|
||||
},
|
||||
{
|
||||
label: 'Oxylabs Password',
|
||||
name: 'password',
|
||||
type: 'password'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { credClass: OxylabsApiCredential }
|
||||
|
|
@ -0,0 +1,327 @@
|
|||
import { TextSplitter } from 'langchain/text_splitter'
|
||||
import { DocumentInterface } from '@langchain/core/documents'
|
||||
import { BaseDocumentLoader } from 'langchain/document_loaders/base'
|
||||
import { INode, INodeData, INodeParams, ICommonObject, INodeOutputsValue } from '../../../src/Interface'
|
||||
import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
|
||||
import axios, { AxiosResponse } from 'axios'
|
||||
|
||||
interface OxylabsDocument extends DocumentInterface {}
|
||||
|
||||
interface OxylabsResponse {
|
||||
results: Result[]
|
||||
job: Job
|
||||
}
|
||||
|
||||
interface Result {
|
||||
content: any
|
||||
created_at: string
|
||||
updated_at: string
|
||||
page: number
|
||||
url: string
|
||||
job_id: string
|
||||
is_render_forced: boolean
|
||||
status_code: number
|
||||
parser_type: string
|
||||
}
|
||||
|
||||
interface Job {
|
||||
callback_url: string
|
||||
client_id: number
|
||||
context: any
|
||||
created_at: string
|
||||
domain: string
|
||||
geo_location: any
|
||||
id: string
|
||||
limit: number
|
||||
locale: any
|
||||
pages: number
|
||||
parse: boolean
|
||||
parser_type: any
|
||||
parser_preset: any
|
||||
parsing_instructions: any
|
||||
browser_instructions: any
|
||||
render: any
|
||||
url: any
|
||||
query: string
|
||||
source: string
|
||||
start_page: number
|
||||
status: string
|
||||
storage_type: any
|
||||
storage_url: any
|
||||
subdomain: string
|
||||
content_encoding: string
|
||||
updated_at: string
|
||||
user_agent_type: string
|
||||
is_premium_domain: boolean
|
||||
}
|
||||
|
||||
interface OxylabsLoaderParameters {
|
||||
username: string
|
||||
password: string
|
||||
query: string
|
||||
source: string
|
||||
geo_location: string
|
||||
render: boolean
|
||||
parse: boolean
|
||||
user_agent_type: string
|
||||
}
|
||||
|
||||
export class OxylabsLoader extends BaseDocumentLoader {
|
||||
private params: OxylabsLoaderParameters
|
||||
|
||||
constructor(loaderParams: OxylabsLoaderParameters) {
|
||||
super()
|
||||
this.params = loaderParams
|
||||
}
|
||||
|
||||
private async sendAPIRequest<R>(params: any): Promise<AxiosResponse<R, any>> {
|
||||
params = Object.fromEntries(Object.entries(params).filter(([_, value]) => value !== null && value !== '' && value !== undefined))
|
||||
|
||||
const auth = Buffer.from(`${this.params.username}:${this.params.password}`).toString('base64')
|
||||
|
||||
const response = await axios.post<R>('https://realtime.oxylabs.io/v1/queries', params, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-oxylabs-sdk': 'oxylabs-integration-flowise/1.0.0 (1.0.0; 64bit)',
|
||||
Authorization: `Basic ${auth}`
|
||||
}
|
||||
})
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new Error(`Oxylabs: Failed to call Oxylabs API: ${response.status}`)
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
public async load(): Promise<DocumentInterface[]> {
|
||||
const response = await this.sendAPIRequest<OxylabsResponse>({
|
||||
url: this.params.query,
|
||||
source: this.params.source,
|
||||
geo_location: this.params.geo_location,
|
||||
render: this.params.render,
|
||||
parse: this.params.parse,
|
||||
user_agent_type: this.params.user_agent_type
|
||||
})
|
||||
|
||||
const docs: OxylabsDocument[] = response.data.results.map((result, index) => ({
|
||||
id: `${response.data.job.id.toString()}-${index}`,
|
||||
pageContent: result.content,
|
||||
metadata: {}
|
||||
}))
|
||||
|
||||
return docs
|
||||
}
|
||||
}
|
||||
|
||||
class Oxylabs_DocumentLoaders implements INode {
|
||||
label: string
|
||||
name: string
|
||||
description: string
|
||||
type: string
|
||||
icon: string
|
||||
version: number
|
||||
category: string
|
||||
baseClasses: string[]
|
||||
inputs: INodeParams[]
|
||||
credential: INodeParams
|
||||
outputs: INodeOutputsValue[]
|
||||
|
||||
constructor() {
|
||||
this.label = 'Oxylabs'
|
||||
this.name = 'oxylabs'
|
||||
this.type = 'Document'
|
||||
this.icon = 'oxylabs.svg'
|
||||
this.version = 1.0
|
||||
this.category = 'Document Loaders'
|
||||
this.description = 'Extract data from URLs using Oxylabs'
|
||||
this.baseClasses = [this.type]
|
||||
this.credential = {
|
||||
label: 'Oxylabs API',
|
||||
name: 'credential',
|
||||
type: 'credential',
|
||||
credentialNames: ['oxylabsApi']
|
||||
}
|
||||
this.inputs = [
|
||||
{
|
||||
label: 'Text Splitter',
|
||||
name: 'textSplitter',
|
||||
type: 'TextSplitter',
|
||||
optional: false
|
||||
},
|
||||
{
|
||||
label: 'Query',
|
||||
name: 'query',
|
||||
type: 'string',
|
||||
description: 'Website URL of query keyword.'
|
||||
},
|
||||
{
|
||||
label: 'Source',
|
||||
name: 'source',
|
||||
type: 'options',
|
||||
description: 'Target website to scrape.',
|
||||
options: [
|
||||
{
|
||||
label: 'Universal',
|
||||
name: 'universal'
|
||||
},
|
||||
{
|
||||
label: 'Google Search',
|
||||
name: 'google_search'
|
||||
},
|
||||
{
|
||||
label: 'Amazon Product',
|
||||
name: 'amazon_product'
|
||||
},
|
||||
{
|
||||
label: 'Amazon Search',
|
||||
name: 'amazon_search'
|
||||
}
|
||||
],
|
||||
default: 'universal'
|
||||
},
|
||||
{
|
||||
label: 'Geolocation',
|
||||
name: 'geo_location',
|
||||
type: 'string',
|
||||
description: "Sets the proxy's geo location to retrieve data. Check Oxylabs documentation for more details.",
|
||||
optional: true
|
||||
},
|
||||
{
|
||||
label: 'Render',
|
||||
name: 'render',
|
||||
type: 'boolean',
|
||||
description: 'Enables JavaScript rendering when set to true.',
|
||||
optional: true,
|
||||
default: false
|
||||
},
|
||||
{
|
||||
label: 'Parse',
|
||||
name: 'parse',
|
||||
type: 'boolean',
|
||||
description:
|
||||
"Returns parsed data when set to true, as long as a dedicated parser exists for the submitted URL's page type.",
|
||||
optional: true,
|
||||
default: false
|
||||
},
|
||||
{
|
||||
label: 'User Agent Type',
|
||||
name: 'user_agent_type',
|
||||
type: 'options',
|
||||
description: 'Device type and browser.',
|
||||
options: [
|
||||
{
|
||||
label: 'Desktop',
|
||||
name: 'desktop'
|
||||
},
|
||||
{
|
||||
label: 'Desktop Chrome',
|
||||
name: 'desktop_chrome'
|
||||
},
|
||||
{
|
||||
label: 'Desktop Edge',
|
||||
name: 'desktop_edge'
|
||||
},
|
||||
{
|
||||
label: 'Desktop Firefox',
|
||||
name: 'desktop_firefox'
|
||||
},
|
||||
{
|
||||
label: 'Desktop Opera',
|
||||
name: 'desktop_opera'
|
||||
},
|
||||
{
|
||||
label: 'Desktop Safari',
|
||||
name: 'desktop_safari'
|
||||
},
|
||||
{
|
||||
label: 'Mobile',
|
||||
name: 'mobile'
|
||||
},
|
||||
{
|
||||
label: 'Mobile Android',
|
||||
name: 'mobile_android'
|
||||
},
|
||||
{
|
||||
label: 'Mobile iOS',
|
||||
name: 'mobile_ios'
|
||||
},
|
||||
{
|
||||
label: 'Tablet',
|
||||
name: 'tablet'
|
||||
},
|
||||
{
|
||||
label: 'Tablet Android',
|
||||
name: 'tablet_android'
|
||||
},
|
||||
{
|
||||
label: 'Tablet iOS',
|
||||
name: 'tablet_ios'
|
||||
}
|
||||
],
|
||||
optional: true
|
||||
}
|
||||
]
|
||||
this.outputs = [
|
||||
{
|
||||
label: 'Document',
|
||||
name: 'document',
|
||||
description: 'Array of document objects containing metadata and pageContent',
|
||||
baseClasses: [...this.baseClasses, 'json']
|
||||
},
|
||||
{
|
||||
label: 'Text',
|
||||
name: 'text',
|
||||
description: 'Concatenated string from pageContent of documents',
|
||||
baseClasses: ['string', 'json']
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
|
||||
const query = nodeData.inputs?.query as string
|
||||
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
|
||||
const source = nodeData.inputs?.source as string
|
||||
const geo_location = nodeData.inputs?.geo_location as string
|
||||
const render = nodeData.inputs?.render as boolean
|
||||
const parse = nodeData.inputs?.parse as boolean
|
||||
const user_agent_type = nodeData.inputs?.user_agent_type as string
|
||||
|
||||
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
|
||||
const username = getCredentialParam('username', credentialData, nodeData)
|
||||
const password = getCredentialParam('password', credentialData, nodeData)
|
||||
|
||||
const output = nodeData.outputs?.output as string
|
||||
|
||||
const input: OxylabsLoaderParameters = {
|
||||
username,
|
||||
password,
|
||||
query,
|
||||
source,
|
||||
geo_location,
|
||||
render,
|
||||
parse,
|
||||
user_agent_type
|
||||
}
|
||||
|
||||
const loader = new OxylabsLoader(input)
|
||||
|
||||
let docs: OxylabsDocument[] = await loader.load()
|
||||
|
||||
if (textSplitter && docs.length > 0) {
|
||||
docs = await textSplitter.splitDocuments(docs)
|
||||
}
|
||||
|
||||
if (output === 'document') {
|
||||
return docs
|
||||
} else {
|
||||
let finaltext = ''
|
||||
for (const doc of docs) {
|
||||
finaltext += `${doc.pageContent}\n`
|
||||
}
|
||||
return handleEscapeCharacters(finaltext, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { nodeClass: Oxylabs_DocumentLoaders }
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
<svg width="120" height="120" viewBox="0 0 120 120" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path opacity="0.4" fill-rule="evenodd" clip-rule="evenodd" d="M75.136 59.9998L93.145 40.7848C97.129 36.5408 96.918 29.8708 92.674 25.8868C92.658 25.8728 92.644 25.8578 92.628 25.8438C88.342 21.8598 81.624 22.0898 77.623 26.3578L52.855 52.7878C49.049 56.8438 49.049 63.1578 52.855 67.2128L77.623 93.6418C81.623 97.9088 88.343 98.1418 92.628 94.1568C96.896 90.1978 97.147 83.5288 93.188 79.2618C93.173 79.2468 93.159 79.2308 93.145 79.2158L75.136 59.9998Z" fill="#23E6A8"/>
|
||||
<path fill-rule="evenodd" clip-rule="evenodd" d="M43.8646 59.9998L25.8546 40.7848C21.8716 36.5408 22.0826 29.8708 26.3266 25.8868C26.3416 25.8728 26.3566 25.8578 26.3726 25.8438C30.6586 21.8598 37.3766 22.0898 41.3776 26.3578L66.1446 52.7878C69.9506 56.8438 69.9506 63.1578 66.1446 67.2128L41.3776 93.6418C37.3776 97.9088 30.6576 98.1418 26.3726 94.1568C22.1046 90.1978 21.8536 83.5288 25.8126 79.2618C25.8266 79.2468 25.8416 79.2308 25.8546 79.2158L43.8646 59.9998Z" fill="#23E6A8"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.0 KiB |
Loading…
Reference in New Issue