From db06f85c2a1b28a472132823b77681469b738ad1 Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 13 Oct 2023 01:28:48 +0100 Subject: [PATCH] add multi options --- .../Unstructured/UnstructuredFile.ts | 75 +++++++++++++++++- .../Unstructured/UnstructuredFolder.ts | 75 +++++++++++++++++- .../Elasticsearch/Elasticsearch_Existing.ts | 2 +- packages/components/src/Interface.ts | 1 + .../ui-component/dropdown/MultiDropdown.js | 79 +++++++++++++++++++ packages/ui/src/utils/genericHelper.js | 15 +++- .../ui/src/views/canvas/NodeInputHandler.js | 10 +++ 7 files changed, 249 insertions(+), 8 deletions(-) create mode 100644 packages/ui/src/ui-component/dropdown/MultiDropdown.js diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts index 820aaab55..3ee7ff730 100644 --- a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts +++ b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts @@ -45,6 +45,66 @@ class UnstructuredFile_DocumentLoaders implements INode { type: 'string', default: 'http://localhost:8000/general/v0/general' }, + { + label: 'Element Type', + name: 'elementType', + description: + 'Unstructured partition document into different types, select the types to return. If not selected, all types will be returned', + type: 'multiOptions', + options: [ + { + label: 'FigureCaption', + name: 'FigureCaption' + }, + { + label: 'NarrativeText', + name: 'NarrativeText' + }, + { + label: 'ListItem', + name: 'ListItem' + }, + { + label: 'Title', + name: 'Title' + }, + { + label: 'Address', + name: 'Address' + }, + { + label: 'Table', + name: 'Table' + }, + { + label: 'PageBreak', + name: 'PageBreak' + }, + { + label: 'Header', + name: 'Header' + }, + { + label: 'Footer', + name: 'Footer' + }, + { + label: 'UncategorizedText', + name: 'UncategorizedText' + }, + { + label: 'Image', + name: 'Image' + }, + { + label: 'Formula', + name: 'Formula' + } + ], + default: [], + optional: true, + additionalParams: true + }, { label: 'Metadata', name: 'metadata', @@ -52,13 +112,13 @@ class UnstructuredFile_DocumentLoaders implements INode { optional: true, additionalParams: true } - /*TODO Add Filter Options*/ ] } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { const filePath = nodeData.inputs?.filePath as string const unstructuredAPIUrl = nodeData.inputs?.unstructuredAPIUrl as string + const elementType = nodeData.inputs?.elementType as string const metadata = nodeData.inputs?.metadata const obj: UnstructuredLoaderOptions = { apiUrl: unstructuredAPIUrl } @@ -70,6 +130,15 @@ class UnstructuredFile_DocumentLoaders implements INode { const loader = new UnstructuredLoader(filePath, obj) const docs = await loader.load() + let elementTypes: string[] = [] + if (elementType) { + try { + elementTypes = JSON.parse(elementType) + } catch (e) { + elementTypes = [] + } + } + if (metadata) { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) let finaldocs = [] @@ -83,10 +152,10 @@ class UnstructuredFile_DocumentLoaders implements INode { } finaldocs.push(newdoc) } - return finaldocs + return elementTypes.length ? finaldocs.filter((doc) => elementTypes.includes(doc.metadata.category)) : finaldocs } - return docs + return elementTypes.length ? docs.filter((doc) => elementTypes.includes(doc.metadata.category)) : docs } } diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts index 4a52fb5a6..c56ff0233 100644 --- a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts +++ b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts @@ -45,6 +45,66 @@ class UnstructuredFolder_DocumentLoaders implements INode { type: 'string', default: 'http://localhost:8000/general/v0/general' }, + { + label: 'Element Type', + name: 'elementType', + description: + 'Unstructured partition document into different types, select the types to return. If not selected, all types will be returned', + type: 'multiOptions', + options: [ + { + label: 'FigureCaption', + name: 'FigureCaption' + }, + { + label: 'NarrativeText', + name: 'NarrativeText' + }, + { + label: 'ListItem', + name: 'ListItem' + }, + { + label: 'Title', + name: 'Title' + }, + { + label: 'Address', + name: 'Address' + }, + { + label: 'Table', + name: 'Table' + }, + { + label: 'PageBreak', + name: 'PageBreak' + }, + { + label: 'Header', + name: 'Header' + }, + { + label: 'Footer', + name: 'Footer' + }, + { + label: 'UncategorizedText', + name: 'UncategorizedText' + }, + { + label: 'Image', + name: 'Image' + }, + { + label: 'Formula', + name: 'Formula' + } + ], + default: [], + optional: true, + additionalParams: true + }, { label: 'Metadata', name: 'metadata', @@ -52,7 +112,6 @@ class UnstructuredFolder_DocumentLoaders implements INode { optional: true, additionalParams: true } - /*TODO Add Filter Options*/ ] } @@ -60,6 +119,7 @@ class UnstructuredFolder_DocumentLoaders implements INode { const folderPath = nodeData.inputs?.folderPath as string const unstructuredAPIUrl = nodeData.inputs?.unstructuredAPIUrl as string const metadata = nodeData.inputs?.metadata + const elementType = nodeData.inputs?.elementType as string const obj: UnstructuredLoaderOptions = { apiUrl: unstructuredAPIUrl } @@ -70,6 +130,15 @@ class UnstructuredFolder_DocumentLoaders implements INode { const loader = new UnstructuredDirectoryLoader(folderPath, obj) const docs = await loader.load() + let elementTypes: string[] = [] + if (elementType) { + try { + elementTypes = JSON.parse(elementType) + } catch (e) { + elementTypes = [] + } + } + if (metadata) { const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata) let finaldocs = [] @@ -83,10 +152,10 @@ class UnstructuredFolder_DocumentLoaders implements INode { } finaldocs.push(newdoc) } - return finaldocs + return elementTypes.length ? finaldocs.filter((doc) => elementTypes.includes(doc.metadata.category)) : finaldocs } - return docs + return elementTypes.length ? docs.filter((doc) => elementTypes.includes(doc.metadata.category)) : docs } } diff --git a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts index 94e45d742..fcb1b2a52 100644 --- a/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts +++ b/packages/components/nodes/vectorstores/Elasticsearch/Elasticsearch_Existing.ts @@ -18,7 +18,7 @@ class ElasicsearchExisting_VectorStores extends ElasticSearchBase implements INo async constructVectorStore( embeddings: Embeddings, elasticSearchClientArgs: ElasticClientArgs, - docs: Document>[] | undefined + _: Document>[] | undefined ): Promise { return await ElasticVectorSearch.fromExistingIndex(embeddings, elasticSearchClientArgs) } diff --git a/packages/components/src/Interface.ts b/packages/components/src/Interface.ts index e883d056e..d0694d6f7 100644 --- a/packages/components/src/Interface.ts +++ b/packages/components/src/Interface.ts @@ -5,6 +5,7 @@ export type NodeParamsType = | 'asyncOptions' | 'options' + | 'multiOptions' | 'string' | 'number' | 'boolean' diff --git a/packages/ui/src/ui-component/dropdown/MultiDropdown.js b/packages/ui/src/ui-component/dropdown/MultiDropdown.js new file mode 100644 index 000000000..ff07b89f5 --- /dev/null +++ b/packages/ui/src/ui-component/dropdown/MultiDropdown.js @@ -0,0 +1,79 @@ +import { useState } from 'react' +import { useSelector } from 'react-redux' + +import { Popper, FormControl, TextField, Box, Typography } from '@mui/material' +import Autocomplete, { autocompleteClasses } from '@mui/material/Autocomplete' +import { styled } from '@mui/material/styles' +import PropTypes from 'prop-types' + +const StyledPopper = styled(Popper)({ + boxShadow: '0px 8px 10px -5px rgb(0 0 0 / 20%), 0px 16px 24px 2px rgb(0 0 0 / 14%), 0px 6px 30px 5px rgb(0 0 0 / 12%)', + borderRadius: '10px', + [`& .${autocompleteClasses.listbox}`]: { + boxSizing: 'border-box', + '& ul': { + padding: 10, + margin: 10 + } + } +}) + +export const MultiDropdown = ({ name, value, options, onSelect, disabled = false, disableClearable = false }) => { + const customization = useSelector((state) => state.customization) + const findMatchingOptions = (options = [], internalValue) => { + let values = [] + if (internalValue && typeof internalValue === 'string') values = JSON.parse(internalValue) + else values = internalValue + return options.filter((option) => values.includes(option.name)) + } + const getDefaultOptionValue = () => [] + let [internalValue, setInternalValue] = useState(value ?? []) + + return ( + + { + let value = '' + if (selections.length) { + const selectionNames = [] + for (let i = 0; i < selections.length; i += 1) { + selectionNames.push(selections[i].name) + } + value = JSON.stringify(selectionNames) + } + setInternalValue(value) + onSelect(value) + }} + PopperComponent={StyledPopper} + renderInput={(params) => } + renderOption={(props, option) => ( + +
+ {option.label} + {option.description && ( + {option.description} + )} +
+ + )} + /> + + ) +} + +MultiDropdown.propTypes = { + name: PropTypes.string, + value: PropTypes.string, + options: PropTypes.array, + onSelect: PropTypes.func, + disabled: PropTypes.bool, + disableClearable: PropTypes.bool +} diff --git a/packages/ui/src/utils/genericHelper.js b/packages/ui/src/utils/genericHelper.js index 324cc1121..af7f43533 100644 --- a/packages/ui/src/utils/genericHelper.js +++ b/packages/ui/src/utils/genericHelper.js @@ -39,7 +39,20 @@ export const initNode = (nodeData, newNodeId) => { const incoming = nodeData.inputs ? nodeData.inputs.length : 0 const outgoing = 1 - const whitelistTypes = ['asyncOptions', 'options', 'string', 'number', 'boolean', 'password', 'json', 'code', 'date', 'file', 'folder'] + const whitelistTypes = [ + 'asyncOptions', + 'options', + 'multiOptions', + 'string', + 'number', + 'boolean', + 'password', + 'json', + 'code', + 'date', + 'file', + 'folder' + ] // Inputs for (let i = 0; i < incoming; i += 1) { diff --git a/packages/ui/src/views/canvas/NodeInputHandler.js b/packages/ui/src/views/canvas/NodeInputHandler.js index 176df52f8..3a13a3b5f 100644 --- a/packages/ui/src/views/canvas/NodeInputHandler.js +++ b/packages/ui/src/views/canvas/NodeInputHandler.js @@ -11,6 +11,7 @@ import { IconArrowsMaximize, IconEdit, IconAlertTriangle } from '@tabler/icons' // project import import { Dropdown } from 'ui-component/dropdown/Dropdown' +import { MultiDropdown } from 'ui-component/dropdown/MultiDropdown' import { AsyncDropdown } from 'ui-component/dropdown/AsyncDropdown' import { Input } from 'ui-component/input/Input' import { File } from 'ui-component/file/File' @@ -308,6 +309,15 @@ const NodeInputHandler = ({ inputAnchor, inputParam, data, disabled = false, isA value={data.inputs[inputParam.name] ?? inputParam.default ?? 'choose an option'} /> )} + {inputParam.type === 'multiOptions' && ( + (data.inputs[inputParam.name] = newValue)} + value={data.inputs[inputParam.name] ?? inputParam.default ?? 'choose an option'} + /> + )} {inputParam.type === 'asyncOptions' && ( <> {data.inputParams.length === 1 &&
}