Sultan
/
Flowise
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use URL constructor to handle all types of links 

This should enable to handle all types of relative URLs defined in RFC: https://www.rfc-editor.org/rfc/rfc3986#section-5.4
This commit is contained in:
Andres Pegam 2024-02-16 11:43:32 +01:00
parent e4cc333374
commit d67c265c0a
1 changed files with 7 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
packages/components/src/utils.ts
View File

@ -290,23 +290,13 @@ function getURLsFromHTML(htmlBody: string, baseURL: string): string[] {
const linkElements = dom.window.document.querySelectorAll('a') const linkElements = dom.window.document.querySelectorAll('a')
const urls: string[] = [] const urls: string[] = []
for (const linkElement of linkElements) { for (const linkElement of linkElements) {
if (linkElement.href.slice(0, 1) === '/') {
try { try {
const urlObj = new URL(baseURL + linkElement.href) const urlObj = new URL(linkElement.href, baseURL)
urls.push(urlObj.href) //relative urls.push(urlObj.href)
} catch (err) { } catch (err) {
if (process.env.DEBUG === 'true') console.error(`error with relative url: ${err.message}`) if (process.env.DEBUG === 'true') console.error(`error with scraped URL: ${err.message}`)
continue continue
} }
} else {
try {
const urlObj = new URL(linkElement.href)
urls.push(urlObj.href) //absolute
} catch (err) {
if (process.env.DEBUG === 'true') console.error(`error with absolute url: ${err.message}`)
continue
}
}
} }
return urls return urls
} }
@ -365,7 +355,7 @@ async function crawl(baseURL: string, currentURL: string, pages: string[], limit
} }
const htmlBody = await resp.text() const htmlBody = await resp.text()
const nextURLs = getURLsFromHTML(htmlBody, baseURL) const nextURLs = getURLsFromHTML(htmlBody, currentURL)
for (const nextURL of nextURLs) { for (const nextURL of nextURLs) {
pages = await crawl(baseURL, nextURL, pages, limit) pages = await crawl(baseURL, nextURL, pages, limit)
} }