From 1e18c289cb06b8cc4401f53d99b31f689b059b13 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Sun, 16 Apr 2023 00:15:12 +0700 Subject: [PATCH 1/9] add CSV loader --- .../nodes/documentloaders/Csv/Csv.png | Bin 0 -> 8498 bytes .../nodes/documentloaders/Csv/Csv.ts | 59 ++++++++++++++++++ packages/components/package.json | 1 + 3 files changed, 60 insertions(+) create mode 100644 packages/components/nodes/documentloaders/Csv/Csv.png create mode 100644 packages/components/nodes/documentloaders/Csv/Csv.ts diff --git a/packages/components/nodes/documentloaders/Csv/Csv.png b/packages/components/nodes/documentloaders/Csv/Csv.png new file mode 100644 index 0000000000000000000000000000000000000000..41b84e16a04dd8b6b6eb1606f4fa9f5317bffc96 GIT binary patch literal 8498 zcmeHsWmJ?=+w~bb1Zf26lt!eHR7y%gLRz}JL&iWtL|Oqsx?$*!A*DgOLApV@d1rXm zdVjq?zMtQ>zIWD|b)9qOy7s>IzRx|g?lU1TRps$;C~*J)c!~Az|Vh$*&{!W&lr?(HDo5c5%>h*+d>lhtw4aI|pqeCuKkJUu$7iZeD&tVG*pjq_nKOqOz*GrnauWp|J_x+|t_C-toJ$tGlPSuYX|h z&(QG5=-Bwg)PYk$Q}khV|hU8fOlF2*Cz@#@wOC)kgY^@7nfmZ=c=j<+YIKz@+$EG?!(6uTC5a zX@lofUIwECpX%@Exjf7GqF`>0;V$Ey97m5Ey=|t}*z4ft4OnR;oln=q)wq)M$?b^7 z=hY@{7MB3k#uK^KCStzZsv$=q-rEvug)2!>t)A+u?{nYEe`OnW|G&Ez?F47@j1uCZ86`;a29UziY zNe^a%^f4b{#t5K7-z&HSe``4rz?dJzyt)q_-~nnF9tgvLp2VR8IaH}>98{1FNr@!{a4TTN4tO5| zN)*tG0UyUkCYz67BC~#P0c2Lg3V{9rgOv<-$(l@nml7mkAhMy!7{CH`q3({roFClP zz|fH4vKxXd-H}C>zLfb-4L-8wGYTk!r1S&09pFQ7B9e%;@51#81WBVUdlxnpkj?X> zf**JQ4;4WpWdNX)z)TPY3C~)D3=u*I7Iq+C&;bSB8x=WiJ*LK+UKNu#iqSvSA>DL@y>XSfCAiGn7j_i&G1>r!1J1de!)SU|%?#Cd=bghjLNWo!VXtFd=U#sA}AdR(o^NTDScSdOWU1-8!<|w4sevBvf7)^5IBc zMc%j}edAT>VZbED^HHTKr}&vGqcZZx$8jfyjK}|eg$DbmT$Ps5=UY2yBHrY@q{1Sr#fBt+ZKY@wyR_KC20oLuAjs}Pb<@ebC_}}cwkTew3 z>E8!CQUnbJ)PK1l>!A;P^b8D8ipl7YD#vPj0^C?+*6q#7qUXNmNM$z}2$*+X+b`}_ zr+TcY2b!{vhS*LVG#aW&91e{z*YPdai5(BejUKX!4>!49sf;R-6u#srdbL&L9~9eFI>f0+GH>=X8>0YJ70?LiF1QzRrX!A%jOtabwXN?pm62HHo zJ)B*ZoW~BdjJxQn_K2O#R$s4$^caLICd~?0iol#Jo$Ll=e}9;|@Zf#6H2c{Kx3x>= z{nsUj&1P-3o||NC$_dGm&cvW1v8^LttA)OkS~CU*nt#?>3u5GJhwKN{J&)W?h2y#S zTrv|S0CU&CB!B0*g86&7vLQR)UR|AmNinDVuJ>~|y9-oJg9a&nnG}USEs5oc7|oa@ z8Il`st7cz3{pe%tG9`h}0+P-}5js0u6Be)HZw zTTS58fYz&tHNURsRPKc3x4}E)Pj|3ExI}`{!BbMX#HlMHqHFu3CF389>YMQ>CWogH z?p*hg*&jd(7cvo6~<6*@XzT&{qwo!Qvm<#alpzRM>cldt#mv-1dHwwvE00R zRxFk-k(?(9a9?xk{V`l{(+JZKa6CIc5MCQB4DW%iv~^5J?gA)gSS1w(DJsO^E< z-XCfrK@wa1Zbp1zP1#}~v75cC>G;`8SUv-DT|Swy@<&&9A(5}M3LPot39FztOg;KG zRJy@|hhf@i4rb+^u% zdC3@{vM*r29V?)7P%~hgH84x@@g-X`IFzvbo5)hBe^k&9{Ipfx+!XqA`#!SUtc3Op zi;rjiEX*6%x2qwxz6Za*GP2T?kQVt(%7;6piuq@&keH1T$MRKI1}gAn&*^Cz6<4@6 zk{$i~1#ZpWEA00v8Wfquunr%iFRZ=5Ii$5iE2u#6Wd1PII?Wf})U1d`LHoc!K@Xg0 zzKRdv5OJE74y^P2I4VN%JBZqYpe59#uBQThK}s|897a5GXwQDtch4o4RU|gNYCCA3 zqhxfEq825>CwAyc&mYCdu&8Yw^4;`AHA`EZwX)k`0#5|&jIPf6NT#f9Z71Mx=%!AY ze`s;uNg)^ect{j!FZS8lBtPE`eZf4tKW}a;*PMbkj2hTbU?#g($x2uYOmGmX3z5-y$1wGHX6c7s)^ej`$EnXyet|TK*D3> zQZs$+!?PPq+0#Or`NR3O+i1hBMhv%=A0RX1`b!XhNXh9htzUl~b7tup*(%TJHn8E4 z%QA^P3I4ql_-3y8we9Wo;cwLIbu?Fkvt%Y|yrsgpH38EVN#d(ucXH-fI}M-)PZb%@Z0Hb4je5ClSGPF{<-<+T(<~6gIIKq>HjpH`|{cP&ynxcXLeGy2hI* zIG|j_W4C$r{%@6699Pw%%%=IIE(&k!=Z&CY#UhMQ{|@s=6z5ID%F5}9=3tr)EI8rK z>#&x?%r)tH6sem%&E;QmyhWbH)E`aTP9(;Np#64r6`tJ#AHC?R8=|f>h?YKVO+EJd zGN>Osl8PQ4}ZEvZHLP+Zyt~WW78T z`0pRVI$`1-tvXX>wANT~t>kt;4pMXJr*HITYsbIwazA}#B5s-<4tQB1+%IE4-N=mN z#X{h@(s;|159#b}&Q$4d?V7&*V^Q zL6up1MNzXpQ+jf!yhEmUq5KS}7LP&M4nCHW3@+&VT@CVew5!D)^XYXVAnRX2tv3$_ z-qailn)g$np3QtC_(xR!E&4NXqD(KCpb7-_J;LvKV!WHaJ71cLzYh-)+Z43L0@++0 zIG4(AZ#Ibgv5p>V&dtB|vMhAqFr01TLx{trGBtR;`b&V*u|2;QGs&VJwZvg|@%zSm zgewBAS|ie#;HQPb%TrF-T?*Twu~om9q1sdvc?gA0cDZxVBHpb~N-E8zq(7bZ*9@i# z`RoWxr|?1;2`Zc_tf0~$tKAP%EbP<-*s9T;jjr(#xPAeB_s~JE>KF^>dE05~5|2dAd%L$riuOZ< zP?~$s>SQ25#nUbFEgty`Z>h%?Zlw1v{?Qx0yP@EMSkYB#JmsHuwiGj9t?nJZWU7YI zdyu#}bkhmk%6CJIIjzTIWYS~9Vf^*pA8ZxcvG2_hv$c&+8qVcG6ii=B$$iEGs?vh{ zkD8UZaX*Ds{WboYD~bD!CX*S=^KI8ZWCu|=)z!NBa8x~g{~*5IKxqlI_Iy15Y-T{q zyKRS03it*7#nIW=z$p>e2~P=((|rDTVi_lu7jN&rq@(F0MBD zpbSPk+a3qv10#+qG#&dSEsWB3TM`mn;YTdVhDYhifR`n3EQNr*qR1A7!I#=)^^(41 z2YPt%Nohu3-Ur1ge?mF^C5qJLi#irKJO9ideufT9E4WQ#G0deP3-I3j)eRy6Q!J*i z#adO2(4lsXI@BG;v;6(eU;OAPwG+x~gD6tITMBe3@aLGx?FO@dA*(FCMATlW);Pcv zEz2?(@jUjGiNIK(?DpS5@`Md^$51Rd-1n6>X!8``DtU=26|e5`jiX}Z+E~Ttt^^fTyHUn{zpK{R7I4KnM-ji82&vNCC zlUVz0_b>c}lt)66%hl`B7mh=u(D?@zx+_G$tu*_S(EB>OLK}%URwXq=gIy&rLpjrOX)AsrnKwtoPChz9ow4B;ML@F(ln(nkN$e{M|dYciDqF?O(!m z7=2K#vYr602OeaRGzW!SKjz;)vMX@mG+bRuqjseszp969+>;9UoOF}%H9Q=)@`wce zTd_g#rTT9pFQ}DVCrKgVE+%^?u#xd?dc~Ch_a^z{Vj=OOz2-1uxD_7!@}AgRHeEug zuJj%4AQRP+KHRTH)w&T@sVf~B0$^JB^BiMeOc|%dj0JB;$yZ7C@>s8%0og<}mEnh= z->bO6zh5Tf7v{7rE%$%}9sx|xubJw;^nKgqJs(u5FjAABg?1_Lej2;H5&Lr%l}(j}70O%KiU!oV8uedIm&P4*i8#2qu`{_mmbjNI@eGkL~j zd;VrsqaVhiuNqBjQTNGytE&xZ_3)U=rDN6c&@t+o*^<>ckf9D_IC1b4P3&{IIYph8Ezlu%{@|!u?)2j5cAfy;wPn>81zPjeHoqLhICo~OGM7P5 zOxSS{UDX=}YFh75w^eaN(D{hIqGCtrkMWX`FSaEnk8F>Q9y7@$1|RwQ_Rg^-9OvPx z1Nnr{!VIi3Kfn-D?Mh-C zUAv?dVnV&Ne&ZXJ%Yst) zA$kw(cl=L{-d9Mj1!Me@*E$Qc1jhEX{;C7ExX|jR**T{1SrprV-Aqx|b)KncPw=Bd zfTBb}8+=;vt?O|g$9E`nibHI=Q*ktS`~yMAlcQcq(~4V*6V6bN(iu4tH`yKrYZ?@2 z^0J=4v=Vq=+u&Jd`D%psmXaDz6#R*-N0V}S+c^17G!Cuq%fR$*N5h9eHqk$&DFH@{ zY``i5udYNgPjOLw4c3SGV5ixmt_&?Rvu7jMIDk8DmkX%x98HiXWi#Ab9)5ptkl!i< zWeKYyuX8;T#JR^ZPqYmyjhjGx@h|>^;f|bX2z>jOQ>3Vh@1Nw{eY{QCN7#G5K|s0< z>#4_(s_c`UO0TG{PAc%8 zYj<8ZIQy^pk=tEapDLWF6RLVR+`ygJ z>mtb^Daoqhn%jc6hvMJMr7rOuehAbL_mR6%OOg+Ir|{IQ%RQ6WuNvFURT-F`Fxkan zZC2x;DjnQ{NZH`^d`;^fx1TAQ&rw|ZeBZ0spiIw(w0M~1I|ctlJNGyHsgi+U>_aX8 zf9F%DmTHuLYV#Um+gOD|-#rY9d=$T;0<2s!GIe56Yi{&&r(g7Mx*Ss1Mpw7XSV2-Y z@*u@1_6HJLx#PM0X#srQ@pTo+=ui=j!9WR|1!2R?;XGBxuKxbK*Oi_Sr+*u*8qiqX z(l2>#wKLXdNwb8$kwK77yBsnoXkFIRyl^v&6C?WDHZRJ9JZDtLk>a{F{LSX}rn#LR z@0jY0BWcGPtdDbOq7du`RJP6H7}Um(3OB!(%+<(ofdEmD{1+%!_)Kdg+G|eLiD!GB z3r-E=kQDyd+i%fV-ZMr0MtYvEh~2PLY!|);Nh!b_VQzDb_p9ggEaA?!3@h0w;qj&g zICFceFWzYukuV5OmEci}y3oT=V^&Kl&wyrhHh;eQ8$TAN6+J5M)_ToWfJp^34 zd`Zi&s~Z^?3e`7oo$$x_JbiTjXoV0w^7X2YDmZ)N>=2c3HSeY@gV9*J;ii!cfv!it zU>{msds0&vb#)pwoZ#p<(DI`3DH@>OgBkmsttwr|XhqyG95QDvVCy;1rq`o^KMv>V zZh_pnF}vy!xA~quFPd5O-U+Qe2EkgRQAvqOyq`4Y^)Ai_xw!79;?3(FFo6K8 zoq(1M=A6c+KPk+|%x{`ZE4Hc3VvWU`ejc!c=kup*49wDzT*Oym3Y%Jk75|F8(95(o z-c{a8dp`haWlFUj)qR7cJZO_jV)o|qHV$Fh0VkH~shQhPCgQRoXV2noeME09?l)Eb zI=ej#!tTQnE;URWXHm~6oAEUg=-vWX=jAtM%Sk7WXy=Z!rJ;4bW>b0Z+_yxH%F=<8m%ccr^|_g~cZmBg-r(HzJQ0TX(Jv)kKi=o7m@-lz4;nz ztTz!akowQFAIpiJoEa*%bB@{Lcr@Qi}kf0XhI60P8C*oU`Ty1`(f~7Fp!8cN zs(d6GNWDP+@BXb)(U_P7OkYFkx_#-1s0% zi{7c=mPN)5T33EH;K)2>vxy>-bD`m~;IH?tRkL)so8`yJ2Y u(oR+NV%^$;V!ERC6o&GwPsubR^KVgWi2vFX5{O|V#VX3GJ};6n{_tOAR;!Kx literal 0 HcmV?d00001 diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts new file mode 100644 index 000000000..db1a3bac6 --- /dev/null +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -0,0 +1,59 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { CSVLoader } from "langchain/document_loaders/fs/csv"; + +class Csv_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Csv File' + this.name = 'csvFile' + this.type = 'Document' + this.icon = 'Csv.png' + this.category = 'Document Loaders' + this.description = `Load data from CSV files` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'Csv File', + name: 'csvFile', + type: 'file', + fileType: '.csv' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + const csvFileBase64 = nodeData.inputs?.csvFile as string + const splitDataURI = csvFileBase64.split(',') + splitDataURI.pop() + const bf = Buffer.from(splitDataURI.pop() || '', 'base64') + + const blob = new Blob([bf]) + const loader = new CSVLoader(blob) + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + return docs + } else { + const docs = await loader.load() + return docs + } + } +} + +module.exports = { nodeClass: Csv_DocumentLoaders } diff --git a/packages/components/package.json b/packages/components/package.json index d2d680376..d75cda9d5 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -21,6 +21,7 @@ "@pinecone-database/pinecone": "^0.0.12", "axios": "^0.27.2", "chromadb": "^1.3.1", + "d3-dsv": "2", "dotenv": "^16.0.0", "express": "^4.17.3", "form-data": "^4.0.0", From 256550f953ac7f58c7a80d193f817505b86fbb5c Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Sun, 16 Apr 2023 00:56:16 +0700 Subject: [PATCH 2/9] add single column extraction --- packages/components/nodes/documentloaders/Csv/Csv.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts index db1a3bac6..819bfeafc 100644 --- a/packages/components/nodes/documentloaders/Csv/Csv.ts +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -32,6 +32,14 @@ class Csv_DocumentLoaders implements INode { name: 'textSplitter', type: 'TextSplitter', optional: true + }, + { + label: 'Single Column Extraction', + name: 'columnName', + type: 'string', + description: 'Extracting a single column', + placeholder: "Enter column name", + optional: true } ] } @@ -39,12 +47,13 @@ class Csv_DocumentLoaders implements INode { async init(nodeData: INodeData): Promise { const textSplitter = nodeData.inputs?.textSplitter as TextSplitter const csvFileBase64 = nodeData.inputs?.csvFile as string + const columnName = nodeData.inputs?.columnName as string const splitDataURI = csvFileBase64.split(',') splitDataURI.pop() const bf = Buffer.from(splitDataURI.pop() || '', 'base64') const blob = new Blob([bf]) - const loader = new CSVLoader(blob) + const loader = new CSVLoader(blob, columnName.trim().length === 0 ? undefined : columnName.trim()) if (textSplitter) { const docs = await loader.loadAndSplit(textSplitter) From da56cd5f689fdc06fe88911bfc6234d8f2947d43 Mon Sep 17 00:00:00 2001 From: Miles Zimmerman Date: Sun, 16 Apr 2023 13:48:41 +0000 Subject: [PATCH 3/9] feat(components): cohere.ai embeddings node --- .../CohereEmbedding/CohereEmbedding.ts | 40 ++++++++++++++++++ .../embeddings/CohereEmbedding/cohere.png | Bin 0 -> 7847 bytes packages/components/package.json | 3 +- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 packages/components/nodes/embeddings/CohereEmbedding/CohereEmbedding.ts create mode 100644 packages/components/nodes/embeddings/CohereEmbedding/cohere.png diff --git a/packages/components/nodes/embeddings/CohereEmbedding/CohereEmbedding.ts b/packages/components/nodes/embeddings/CohereEmbedding/CohereEmbedding.ts new file mode 100644 index 000000000..923cf6c64 --- /dev/null +++ b/packages/components/nodes/embeddings/CohereEmbedding/CohereEmbedding.ts @@ -0,0 +1,40 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { getBaseClasses } from '../../../src/utils' +import { CohereEmbeddings } from 'langchain/embeddings/cohere' + +class CohereEmbedding_Embeddings implements INode { + label: string + name: string + type: string + icon: string + category: string + description: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Cohere Embeddings' + this.name = 'cohereEmbeddings' + this.type = 'CohereEmbeddings' + this.icon = 'cohere.png' + this.category = 'Embeddings' + this.description = 'Cohere API to generate embeddings for a given text' + this.baseClasses = [this.type, ...getBaseClasses(CohereEmbeddings)] + this.inputs = [ + { + label: 'Cohere API Key', + name: 'cohereApiKey', + type: 'password' + } + ] + } + + async init(nodeData: INodeData): Promise { + const apiKey = nodeData.inputs?.cohereApiKey as string + + const model = new CohereEmbeddings({ apiKey }) + return model + } +} + +module.exports = { nodeClass: CohereEmbedding_Embeddings } diff --git a/packages/components/nodes/embeddings/CohereEmbedding/cohere.png b/packages/components/nodes/embeddings/CohereEmbedding/cohere.png new file mode 100644 index 0000000000000000000000000000000000000000..266adeac2214b8627504eaf299c990b28f0448e0 GIT binary patch literal 7847 zcmeG>`6JWu`#W6CvB^lT4H4yxOwPHYgBVkhE6LGZ$x)b*F)1cWB}XXCO-AGh(MhSf zhULnA-$Q-heLnxg=cn%v`{DI^&e!vP?j=}Tne)TMU?32P|BQtx4g>;I|NVHlfE!Y? z))x>+%I}P+kzJVc!tjBi0{6zwmCu!Ox@K`omyh4OdUhhmD%FCI93@9S;5(RPSen#h zdU@9vp4(zG}qg2@2yixX43b9YYb8&>ND?U#rDed|)5&D?RiP~NBB zt=GL3#oDS-QVaSG2yEG7!&xAZt0oTuggpr108wK>V9*h1EC_Vw|F8bnmbe!ERKt@k zrFPN1GuiQcI^J%MW{)c^ba1Y@?{kJRMl-Y94CXD`KG#dQK+D4In{5=%f*E`aCtAh= ztHA2n-d0rPsrneuyDN%HN=Pl6Mu&}?j6vzRPEkhg-_4iPuR8Iev{yxbIH#g|hv=)( zcOT1Q)moQ_-`}Av;4|Vy#f~R$%eXEsT00f%mp0a`c-Kc(W>q<9BY!FGup&*k69o4YkX`g)tK_0D{Ik zSiP%!rh>9xD0LX*hvKVUuty6GiqW_G;khF`)W^7hx0li{&^CDjGr6oiwfMI+U@1%6 z;Gc)UJttrS8TuZf;*kt7KheSlSv+@TQvP8cs)$WkfBl05+SfhNZ|J_lO>0WO(*d9R zAa5{dyf5k|#fOEcMS!0zUZ8CqGgVoE7Q!L;~Uq0?UKtJjZfxhC!}wK(~@VWF9K+!t|;)i(AdH9KSG?p}?*B{f6nN+flBp7gbv*(CH~7 z#_?Bx51rJ{MA|c={&(SIJE@Lx^U1>(HvW zA!#gt8>de0jFBF>jgZ3DzsIB2ERl(zqLC>(|1C?<+CpYq=$gU-Fz!X=3XcOX0(95W zs;0m{SNkorh|=nQ*y$BlNH6EjdfS@OpJ!508_*Ac5oRT%c;M(-hMoVaCGsw)h_q-x z@0r|hf};}`BBAep3EF>*^K=@{x$_7GrM{PyR#AG(f}>Rzv7bB}<_=0>10UmkXEQz| zgA`L14eWa+u`F(Crtel#j39clYC*()*%CC+cp23%vSvA9MoKe{dfM0XUh5o(_1j1I z;$8Vw8SJMZ(x+Y`NFggzuqw-6Mir9JQNJ5z_G=NUAfAecohGHZt1@^ags?A0ro}%b zf)t-D8q8l-O5}=OEv02Qn?(riAkgo_Lw^l(TXX7q`FOKT4LB^K;|c^9$9F+z-8^Ia zbB&R)AjQX}aZYU*a9XRBOCUE64avV0zRg9#DbI+1hzBWt=wBIs44;R=?7j*-H8$d* zju8&kMRL(PBrSko9DDm21In@Fp+fFU8g?z}X>CS=UOKj5sBkcjj~5;?%t_md^+%&1 z`J%czT%@yl^Z#MmA>RbfIhef5wJME0A+Adt@`?7%kJi~XM`K}Tq>0+6J_lGF*7XH; z$!vK5WtG{20ojdJVoAt*ppV@G$q?&CZb)?nfO}0dWKVl{?b@1YjucTpxM#vM^545} z59~mTx8+G^uZ;h*&BwV%(QDkK`t1k)XjRA&)r?l&oI6o?%l{aE;$+7^?jelEri8aV z)VKEQ{Gdm5UbFch=~M)#40g3DeHugyP4Gt>0;JA4P)37EaHhWua4PqRfMRF5GIYFG zQ=1a@0OC3*m=^lNKSz|q!dVqy^s@fSNe0G-51@o_6!r!ZTyRnV9cLsI>+NT42{>0g z=wspq18i{8*kaxlfQlXoA+0*S)cK6DQ39x52|y;Rg{QIsXaoR#PUK2k%77(j@5$1k+trg0Dk8r@^trEG*Pv$ce;NXXSP{8O2L%YcD7#kHp zhh0F25V~tX=yfN2GN6ReH`3Lw)(f=duy633f3lkG^(QkxRTazqr$^m78M$1*`xaqC zWRO{%c}-^PxQHQX7J)uIqq?Ob-F1$e+P#ig((_w~7K(r$vit$=RdaYpMLpsqx3xax zh+#M!;u^rZ*oYrhABN{h@lf~5cnNa?e6tn|0T^6ls<$7a;n}sW&*wV79hevwc4+m4 zA(v|wL4ENLED6XW>`{Ru8vE~~9MT;lQ7w9wHjOFEUIKt;&38lZXGykiDl)Xm+^N=L zvF4XJMFFRJCT-B+(4Gtyb-Op43KL*vaqzcETByG&_FJbHoSmAwkA6l5SRDpfotCzmar%}Fmb`|db>Stu?yYVDu9^#~ z{8e|RrU?br`7Oh^C@gW24b~I~x(XQi;^q=Ua|f zcy7I~p{Xir&v!wUJI<{`_?Je5Dwm+j!8JaPjO(%H7$9(UdoiTPmAXlpct-_oqM9DB zpk#||Nz4Nv%#7JL86reJue$6a0o%)7713VYf@nEw$~^7!KoeXu&6g3BYdj8=L_cZl z*&NVirFP~X|GzTDvfVF_89xC4WUyyjop{7PC3zd13w{A#Jsb92T5N8SdGeg-cMm z%A5-GB69^zB`oR4zk4+A+zEWm3ltP~EBRBe-}BbQzzHnL&Q!XpQsp9NQlr_5A3pPy z4*uN=RUGL$@x-asDc^^?QDpl~U?js!Xk1%#a`U!UiJM!7=mFu=I1bC>= zY$*L92X}W3bmr6YQC-V`N=9e1)z+?#HDXkhQ(Y$sSD+v_He9&c! z^i+I!o%HLBOgUooER%*hrcF4jV(rLbVX0^;@^dmHS~o?dLiwlfJJu=K1Mz*dHDJpJ z{HmI-GjWC!o!`Q`*s^)Prlb0lr(&LP)W`U89^c zJk&UA%3GIoce})0^`9dwJiYY8VcL>{_I%x?oE3o1hC+}U(;=N4M$FEPAYAS-o9vXf zP3912{yfBCp|Ph7CR_by%szJCZeth#ulYes%}APQF%xK~Kjt3QHP-sPungjjIjypl zyTO=+`NN5`SGh9>kgJ!OGySix7JK#%_b!l$Za85e^%J4Ja-`?EMFH=OBCEo$OgOv}&7KXo~O!Ni< z>SwhWmas!3Gq;f7o&Z>i+^rpB3N!l)`<;2El(i);Q@y6c^WJea`yP)}1S+(w+~Ih8 z%ZC}_qtdim*_*zfAI%{ZqQrLOypBWi<&a0*gLh~?=PTS zbx(#PUOYA|)J8r$E%Y3vfivoY+iEx_Nkkg!a4l?u?M#CjN0>&x8#p?D&P1vo(8nOPJOptC)1w2E^wD5+|r5!kA~ zyQIzy91F}i6Z{la0ws6!NE>9`srAtXTJ7fCO(}R_a&0Q|NG;n)lNa0rhCLJx9n5eY8I`Yk>U5NrG^u|kNDwWVi>~w>-;CLa zq+uFH%PP|wMZ~5QHRrFaH~>Aga{IS_^>#(Nulv#CgA>WJ!bj+nykfDS83l4eBVpK} z-Q~P$Xq&Ic)Qx`GW4Z%NHvwts<&xx%PtpdtRk{KhPJNv8?bRSy%3$27tYpxPHfMse z6!ZK^XOHZVi;_FS54I{k_4+(7Wp1E69zDhaHS8y!z>542cGH;H>wG47*^9#T>Ax?$(cDf|GBspexh1r4#+TtWre8ujG$zau}l`H+jFOBu`+FRe- zn>Ab1JpTmgl=^;YfKM&M>{{{U!l9Xx4c^V6-E7?^S{5uYvnAUkBGgUeCBa3G_W={X zrJz7yd*&%<+xL0IBV6uRuT1V$2g~lRIOhY_Nwa#fHZ)c%KXfV|7tlWuyA@Z58xjG@ zWG<94Y$^Tmygxa>kZYQ6aYK-CCt9iTn*F`L0VJafCg`XeBZkJ3_~Js})r5*K*_9gH zL>?XpFJGt=D&mH4SZFF|jn!IG-c@=~xKF$YGsGiyc~Pd<=FKWTLt$VHJUH(NoYSlvi}eZWHq! zh>K|rWWfY8ekZ~Jhy;+XsPei`K6ho*R!`X&(*3-G==w!m;GXOIS;~XehZ^Yt-G7E( zr48olGk=diRSTpnR8L_?-7?NeiHt$c{LILFXYiODHz|u9%waidx8KbrVETfM_29$y zv8v)@N2e|qw>%(smI$-xENW&eM!)^h$fn|=Jm)!2&ssn&;r%{iJq^yUfw7PYp#Qrn zh_+-b4UE{*trr$XUEZa*r<&-N4{knX8&%9a6pCLA94~#j+;5j|YaH5T?w#t1A4kg6 z=-sO;<)O;YY|Z<%_Dc2_xHH+|J1H*IVRzab-h{R$ET3Gr(rXmMXZmRfDme^*U8GBbbi@#!kqNevf0!8lNH~Jletj;0a7IS z!y{r&8B$71c3~ZX;?0emCp>*1ETeUpgY%R64emGV;hRMtL&Sa0hKcH~josY;OUywP zRtiB!zhB=3zc>?x5>M2n>P&CVw>RaP=%E@us#sfo8>VE+Tsl|3tF67<`j(f__~ZEP zLpY^HEUkB`RATH|;;Z=h+wrrVZ2voCR3Vhyxfs1X4a-1R766kIfQ5|u?kq~nc;<+7 z95!$cmASYZP2*e24}ahjXc*Uf$GV2TXiVtjM3Qu=wDoZDxG*&ZBU-KFVFRy;f+<&}yvGLS-iFyV+%_tP*IDOOmmqc>e9bCnP;^=m ztJ+*)Dt0k8zS1eWSzqG$%V#3pN`^8BDW~gGKi5Qy(Hs_U2-Qu>k=0N_@y@@5wPg2{ zApWF}Ew9rVQgRy1L#=}-?Og8wi_5Fzvq6CIcrgE*GPq(-r(Eqyzb`cCBn={K2?8EYZ2JXD7PzaNGS`q zsPTKEeXXzL)|T7uyDBX?FHakz99bB3QAoZvy`d~}_T9X)@Yvy{ z0lo{vk*Q12Hv0{JzBrJgakFQ3$mWKxUbl*L7WrXfsWv^&CwgEM>U5pMLfsivU*mM< zSa|HCwevUaCEWeavaAXgc&JyGC5Y=)63V(jze4YdD|+@s{8fnOY0`uU>LX#2`6RE@ zWN3tF9in~cc(*!6tCI7uAbO`!Zn-3w*l+cQKFQ9Zukg2w?q33g5a|NoHP0ZHg>|o# zqn8SOqx(MSWAZ}V6wAJC?)t;?y5Su$^(CXbq_(AoFWbe{1< z#$B(ql@_eBAo{j%i9;7g{n(A&j);mw({we#C*oRvJ_GC$8u!7mG{A3@AMiON@u4YkQc$M9YrsEDE#c&&YDYuj*+Q)LU%;#SZye41 z`tWuK#(YMF!N=l=8?NZ`bC+8F=dK^;Y*;OY19n6Cpzo^OyDariRM5?6e>5Bzo((4R ztBdOH5^BFnM1AQ>jRq<50dsJ61T^ouZ6Nn;4($O9=#5oq;nueNDjIm>sfhaB*7g^k zkwFG~G8ImktfliTEU-0k){DOoTI&DhDARib)0AKpZXI0aFM|SxXz!+I8}aG%zd#gM zu9c~-Mcx`zWQVVK{%6_Z+*d5MVtN15LQ{&XLiFccS04C)S_~DOs>F|WEIN!DI z-oS~v@al#qE-rBu_AuG?&v~0|fxSYbX0`dnaSoRGRDXSr39#M(oX%-f z{7J#_NOuqmPFNZVX8nv$Z~?YPve?z?4W9E;lKSEH`x>nCFI^-yJ)<5kh${tPTk$snuqTj#j`~w~LY;4aq>IeXso;qh>y(aLX6wr^V*Hs@iiI+pAw& zD6sW!Nuk(r(yb+!U(&7fKCus@v2$9K;IFKiXqlspMM-UErBgtPt?kmkbCSDmpStey zHHnxql>hnfj)BA6rO586i3|l1Fz&f6MD@p1ht>tXE{uk*>HE`n7PgKvjG7Y01kq** z*(LBbU%e?uw#Kn_j{<{~KQ`&Ay9N&Lt#XGeyJ@*9ojug*1)<+abjfa>QV*p!{ri$D#lX^Y-HyF}kg3@WZS5EIh5=ilL`->kY z&j*?KP0{j_LsQi!_YyWDqa{-_J~jC>XLr2{F8tf4+>F$_Hl$r)M8N;y4)VYJ(6~8* zP~9kLhXJOtK_p|7{b%5*ZearF^{$dV&w598<0Iptracd0;mi5qZhKg=_&)a0t4w|O z8brZX*<7c!?xErkae=6xJfUch4DDOlf8I3G(!5KAt5MsbKb7_F6Aup;eU2h)ALrZH z2xcV3n(G5O_z}^+`~S(!(worZxA$ zPu_2wA38Vsz2EyWEaj{5o-<*h(^`RPvy})-%I%6b0!p3E%+@AzXjQIqZlXM~E zk#nd00XoJm(dAycky}*&B}3DpqFi`v%l$lMsdC=wp8jUl(&gc$OY`Y>_nj(> z=$l*b<*oYt>O$kULw^e}RFR@i{eAq^MkEmEVTHIV2z>wy0a+OTJ0j2^V3Wl2|5yK8 cORWACc0UkdKO-p*`~eL*V`gPqX6zdCfAk*42LJ#7 literal 0 HcmV?d00001 diff --git a/packages/components/package.json b/packages/components/package.json index d2d680376..d71499a18 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -21,10 +21,11 @@ "@pinecone-database/pinecone": "^0.0.12", "axios": "^0.27.2", "chromadb": "^1.3.1", + "cohere-ai": "^6.2.0", "dotenv": "^16.0.0", "express": "^4.17.3", "form-data": "^4.0.0", - "langchain": "^0.0.53", + "langchain": "^0.0.56", "moment": "^2.29.3", "node-fetch": "2", "pdf-parse": "^1.1.1", From f5acca1aee8cd741e9d006f0d9c1d2e7799286c5 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Sun, 16 Apr 2023 20:56:35 +0700 Subject: [PATCH 4/9] fix yarn lint-fix --- packages/components/nodes/documentloaders/Csv/Csv.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/components/nodes/documentloaders/Csv/Csv.ts b/packages/components/nodes/documentloaders/Csv/Csv.ts index 819bfeafc..2ead7055c 100644 --- a/packages/components/nodes/documentloaders/Csv/Csv.ts +++ b/packages/components/nodes/documentloaders/Csv/Csv.ts @@ -1,6 +1,6 @@ import { INode, INodeData, INodeParams } from '../../../src/Interface' import { TextSplitter } from 'langchain/text_splitter' -import { CSVLoader } from "langchain/document_loaders/fs/csv"; +import { CSVLoader } from 'langchain/document_loaders/fs/csv' class Csv_DocumentLoaders implements INode { label: string @@ -38,7 +38,7 @@ class Csv_DocumentLoaders implements INode { name: 'columnName', type: 'string', description: 'Extracting a single column', - placeholder: "Enter column name", + placeholder: 'Enter column name', optional: true } ] From b28546c2ac6fc134e2bd4fce48f7fb98ff91d1c5 Mon Sep 17 00:00:00 2001 From: chungyau97 Date: Mon, 17 Apr 2023 21:53:47 +0700 Subject: [PATCH 5/9] add cheerio web scraper --- .../nodes/documentloaders/Cheerio/Cheerio.ts | 64 +++++++++++++++++++ .../nodes/documentloaders/Cheerio/cheerio.svg | 5 ++ packages/components/package.json | 1 + 3 files changed, 70 insertions(+) create mode 100644 packages/components/nodes/documentloaders/Cheerio/Cheerio.ts create mode 100644 packages/components/nodes/documentloaders/Cheerio/cheerio.svg diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts new file mode 100644 index 000000000..8be10f742 --- /dev/null +++ b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts @@ -0,0 +1,64 @@ +import { INode, INodeData, INodeParams } from '../../../src/Interface' +import { TextSplitter } from 'langchain/text_splitter' +import { CheerioWebBaseLoader } from 'langchain/document_loaders/web/cheerio' + +class Cheerio_DocumentLoaders implements INode { + label: string + name: string + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + + constructor() { + this.label = 'Cheerio Web Scraper' + this.name = 'cheerioWebScraper' + this.type = 'Document' + this.icon = 'cheerio.svg' + this.category = 'Document Loaders' + this.description = `Load data from webpages` + this.baseClasses = [this.type] + this.inputs = [ + { + label: 'URL', + name: 'url', + type: 'string' + }, + { + label: 'Text Splitter', + name: 'textSplitter', + type: 'TextSplitter', + optional: true + } + ] + } + + async init(nodeData: INodeData): Promise { + const textSplitter = nodeData.inputs?.textSplitter as TextSplitter + let url = nodeData.inputs?.url as string + + var urlPattern = new RegExp( + '^(https?:\\/\\/)?' + // validate protocol + '((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|' + // validate domain name + '((\\d{1,3}\\.){3}\\d{1,3}))' + // validate OR ip (v4) address + '(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*' + // validate port and path + '(\\?[;&a-z\\d%_.~+=-]*)?' + // validate query string + '(\\#[-a-z\\d_]*)?$', + 'i' + ) // validate fragment locator + + const loader = new CheerioWebBaseLoader(urlPattern.test(url.trim()) ? url.trim() : '') + + if (textSplitter) { + const docs = await loader.loadAndSplit(textSplitter) + return docs + } else { + const docs = await loader.load() + return docs + } + } +} + +module.exports = { nodeClass: Cheerio_DocumentLoaders } diff --git a/packages/components/nodes/documentloaders/Cheerio/cheerio.svg b/packages/components/nodes/documentloaders/Cheerio/cheerio.svg new file mode 100644 index 000000000..8e3334b9f --- /dev/null +++ b/packages/components/nodes/documentloaders/Cheerio/cheerio.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/components/package.json b/packages/components/package.json index d75cda9d5..1775a8cfd 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -20,6 +20,7 @@ "@huggingface/inference": "^1.6.3", "@pinecone-database/pinecone": "^0.0.12", "axios": "^0.27.2", + "cheerio": "^1.0.0-rc.12", "chromadb": "^1.3.1", "d3-dsv": "2", "dotenv": "^16.0.0", From 180990b0258a34640c589c1708296da1241bcb8a Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Apr 2023 20:33:32 +0100 Subject: [PATCH 6/9] clear connected input if node/edge is removed --- .../ui/src/store/context/ReactFlowContext.js | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/packages/ui/src/store/context/ReactFlowContext.js b/packages/ui/src/store/context/ReactFlowContext.js index b8b32606b..1a6b979f1 100644 --- a/packages/ui/src/store/context/ReactFlowContext.js +++ b/packages/ui/src/store/context/ReactFlowContext.js @@ -13,13 +13,51 @@ export const flowContext = createContext(initialValue) export const ReactFlowContext = ({ children }) => { const [reactFlowInstance, setReactFlowInstance] = useState(null) - const deleteNode = (id) => { - reactFlowInstance.setNodes(reactFlowInstance.getNodes().filter((n) => n.id !== id)) - reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((ns) => ns.source !== id && ns.target !== id)) + const deleteNode = (nodeid) => { + deleteConnectedInput(nodeid, 'node') + reactFlowInstance.setNodes(reactFlowInstance.getNodes().filter((n) => n.id !== nodeid)) + reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((ns) => ns.source !== nodeid && ns.target !== nodeid)) } - const deleteEdge = (id) => { - reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((edge) => edge.id !== id)) + const deleteEdge = (edgeid) => { + deleteConnectedInput(edgeid, 'edge') + reactFlowInstance.setEdges(reactFlowInstance.getEdges().filter((edge) => edge.id !== edgeid)) + } + + const deleteConnectedInput = (id, type) => { + const connectedEdges = + type === 'node' + ? reactFlowInstance.getEdges().filter((edge) => edge.source === id) + : reactFlowInstance.getEdges().filter((edge) => edge.id === id) + + for (const edge of connectedEdges) { + const targetNodeId = edge.target + const sourceNodeId = edge.source + const targetInput = edge.targetHandle.split('-')[2] + + reactFlowInstance.setNodes((nds) => + nds.map((node) => { + if (node.id === targetNodeId) { + let value + const inputAnchor = node.data.inputAnchors.find((ancr) => ancr.name === targetInput) + if (inputAnchor && inputAnchor.list) { + const values = node.data.inputs[targetInput] || [] + value = values.filter((item) => !item.includes(sourceNodeId)) + } else { + value = '' + } + node.data = { + ...node.data, + inputs: { + ...node.data.inputs, + [targetInput]: value + } + } + } + return node + }) + ) + } } return ( From 4a9d91dd34a177f0118f0f5b1b3552c8bc5811c5 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 17 Apr 2023 20:50:57 +0100 Subject: [PATCH 7/9] Update langchain version to ^0.0.53 --- packages/components/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/package.json b/packages/components/package.json index edf0336f3..83e3e0237 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -26,7 +26,7 @@ "dotenv": "^16.0.0", "express": "^4.17.3", "form-data": "^4.0.0", - "langchain": "^0.0.56", + "langchain": "^0.0.53", "moment": "^2.29.3", "node-fetch": "2", "pdf-parse": "^1.1.1", From eb3897b042b973f35f53a893e51c25b0fe782365 Mon Sep 17 00:00:00 2001 From: Henry Date: Mon, 17 Apr 2023 22:12:12 +0100 Subject: [PATCH 8/9] add fix to chat message scroll --- packages/ui/src/views/chatmessage/ChatMessage.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/ui/src/views/chatmessage/ChatMessage.js b/packages/ui/src/views/chatmessage/ChatMessage.js index 4fbb54e68..4dbdc8498 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.js +++ b/packages/ui/src/views/chatmessage/ChatMessage.js @@ -39,6 +39,7 @@ export const ChatMessage = ({ chatflowid }) => { const customization = useSelector((state) => state.customization) const { confirm } = useConfirm() const dispatch = useDispatch() + const ps = useRef() useNotifier() const enqueueSnackbar = (...args) => dispatch(enqueueSnackbarAction(...args)) @@ -115,7 +116,9 @@ export const ChatMessage = ({ chatflowid }) => { } const scrollToBottom = () => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }) + if (ps.current) { + ps.current.scrollTo({ top: Number.MAX_SAFE_INTEGER, behavior: 'smooth' }) + } } const addChatMessage = async (message, type) => { @@ -286,7 +289,7 @@ export const ChatMessage = ({ chatflowid }) => {
-
+
{messages.map((message, index) => { return ( // The latest message sent by the user will be animated while waiting for a response @@ -331,7 +334,6 @@ export const ChatMessage = ({ chatflowid }) => { ) })} -
From f938ee7b4b412be2050dabcbaaad1012d05e8b00 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Tue, 18 Apr 2023 00:03:45 +0100 Subject: [PATCH 9/9] Fix yarn lint --- packages/ui/src/views/chatmessage/ChatMessage.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/ui/src/views/chatmessage/ChatMessage.js b/packages/ui/src/views/chatmessage/ChatMessage.js index 4dbdc8498..b6632b1ea 100644 --- a/packages/ui/src/views/chatmessage/ChatMessage.js +++ b/packages/ui/src/views/chatmessage/ChatMessage.js @@ -55,7 +55,6 @@ export const ChatMessage = ({ chatflowid }) => { } ]) - const messagesEndRef = useRef(null) const inputRef = useRef(null) const anchorRef = useRef(null) const prevOpen = useRef(open)