From d5153c384076d889ed6b70a0d1db6b51a2d472b2 Mon Sep 17 00:00:00 2001 From: Mohamed Yasser Oaf <49288970+MohamedYasserOaf@users.noreply.github.com> Date: Sun, 18 Aug 2024 14:23:45 +0300 Subject: [PATCH] Added meilisearch retriever component (#2824) * added meilisearch retriever and credentials.ts * added semantic ratio * removed a TODO implementatio * meilisearch component implemented with searching and upsert functionality (#3) meilisearch retriever component created , searching for an existing index and upserting a new or existing index has been implemented , component utilizes langchain and meilisearch vector search Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/3 Reviewed-by: mohamed1999akram * added CI/CD for ntg branch, added proper dockerfile for flowise-ntg (#4) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/4 Reviewed-by: mohammad * modified os version , removed linting errors , removed cypress github actions (#5) added --no-lock-file flag to pass CICD , made the runner run on debian and not ubuntu , removed code that caused warnings to pass linting Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/5 Reviewed-by: omaryassery * removed unnecessary QEMU install action (#6) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/6 Reviewed-by: omaryassery * removed cypress installation and linting from dockerfile (#7) Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/7 Reviewed-by: isameh * dockerfile-ntg-modification (#9) dockerfile-ntg modified to copy all working directory before calling pnpm install Reviewed-on: https://git.beyond.cc/ntg/flowise/pulls/9 Reviewed-by: isameh * resolved comments, reverted CI/CD * add test docker build yml back * moved meilisearch to vector store folder * Update Meilisearch.ts --------- Co-authored-by: Henry Co-authored-by: Henry Heng --- .../credentials/MeilisearchApi.credential.ts | 32 ++++ .../vectorstores/Meilisearch/Meilisearch.png | Bin 0 -> 8843 bytes .../vectorstores/Meilisearch/Meilisearch.ts | 174 ++++++++++++++++++ .../nodes/vectorstores/Meilisearch/core.ts | 92 +++++++++ packages/components/package.json | 1 + pnpm-lock.yaml | 12 ++ 6 files changed, 311 insertions(+) create mode 100644 packages/components/credentials/MeilisearchApi.credential.ts create mode 100644 packages/components/nodes/vectorstores/Meilisearch/Meilisearch.png create mode 100644 packages/components/nodes/vectorstores/Meilisearch/Meilisearch.ts create mode 100644 packages/components/nodes/vectorstores/Meilisearch/core.ts diff --git a/packages/components/credentials/MeilisearchApi.credential.ts b/packages/components/credentials/MeilisearchApi.credential.ts new file mode 100644 index 000000000..64d8367b0 --- /dev/null +++ b/packages/components/credentials/MeilisearchApi.credential.ts @@ -0,0 +1,32 @@ +import { INodeParams, INodeCredential } from '../src/Interface' + +class MeilisearchApi implements INodeCredential { + label: string + name: string + version: number + description: string + inputs: INodeParams[] + + constructor() { + this.label = 'Meilisearch API' + this.name = 'meilisearchApi' + this.version = 1.0 + this.description = + 'Refer to official guide on how to get an API Key, you need a search API KEY for basic searching functionality, admin API KEY is optional but needed for upsert functionality ' + this.inputs = [ + { + label: 'Meilisearch Search API Key', + name: 'meilisearchSearchApiKey', + type: 'password' + }, + { + label: 'Meilisearch Admin API Key', + name: 'meilisearchAdminApiKey', + type: 'password', + optional: true + } + ] + } +} + +module.exports = { credClass: MeilisearchApi } diff --git a/packages/components/nodes/vectorstores/Meilisearch/Meilisearch.png b/packages/components/nodes/vectorstores/Meilisearch/Meilisearch.png new file mode 100644 index 0000000000000000000000000000000000000000..7bbb458fa99d39484d105145f114940571ee19cc GIT binary patch literal 8843 zcmd^_2-&hPRq+#V3x_C?5&G*tL6} z88zY8Gg)1LKQq4539Uk24iV{@huKq3LX08;rU%+DC^TulRk>i6NSyy)H>z+CZ0qoj zhHthGA#Sv+wiv$aPiC-0f+Eg*ZZb0O+=i;EA?d>t0|n(s5{@vAQ!w|py2S-h(eOsV zUE4CiGOa(|vD?{#&eek1S+X+=QJfLT_oBKO!luiD@my5pJh4J@e{!7R_R2qf$Ns6c zZFn=?SdisXyobe+KP+LSU$WYm#W+rWSfj4^)vHgtQW64QeXiE=3D%jKuqRRDMZ26U z>uH@$pdk@rj?p&*wYHa+L;IwemsjakXF^f(Z-2W9^*xp>X)g{ci&X2h>tRiKd~nBF zwK?JW2+e%3ZeoYDU9l~+*M!K{SRwSTA(G!Lsx@O~MO73ic_z}&Y=^_XntxoNa}iHrQ4e%GNyjHhO5{#(sF<-%B8}p}qeXuM+*wc!+e?`mFaO#IXPgd30z^v8XA;izB|0nz!l5>H*-5J45Bc=Wa@N~N>lHF1S_4!_v1d@#;^Tm6*Pd!rNr=AgU+n9 zn~NdYmV|8K2Q`Xoiv^`rp9{O|2;4f)~0|!#XN1G*| zy7gOyehK&z1GN9@-F(`1-Zn0)zhs#9h97*vfbwz3lh${AYGtOy`}-8L>A5h2=^N1$ zgukiN#?@ZY$6BJVwrl1>w<{}ztlFLKIoxKnYR^~SG>o=xsSH14j~X~y3?4|j>`66e z(Y~!)R+dd5Te!uzb{$7&*wW>xEB>m&i| zusls{Wm4Z&*f)-%c@t#k4Lm<(jy?`z&Qpirjj1oz+R$x*JMUO0A%^Dc-WVb~)7Hgj z!T2=u3|04%{+$*ifN(jYX(5&Qu^XheLiIxn)pf1XDC9xHH1AY`#$CYU`rt7V$v;Lb z(q=zV#~7AjW_}t%Ij4v_ceG+`-fZle;{v}H5z1at;BF1~cxB*y$>!#G^I__=Ys}+a z6Do_SGrCFuBtWjB@NPQASm%VGVXR=G5ra`EP`mH{-%CU_ru4CMHLKmf16Jb4kwDR1Xo&szk5nzhY@!6WuQ!-#}Q2FbY ziu8K8v_l>ryK82iX2N)~5yAkW)5quit80Te315V)qh=}H66&`}4qt0vGaunlASiV# zp+ve{h=BOWQE8^J1R{6P z95ZUmFSnK*8)X*~vqpU@YlgpsIx4`Ow3WoCiH3cz6!IvVxBcGm#NFFrx0ejigBx@; zP$|bi7)4lq5_n2eQ(eAdlG&t>G?{f;;we32GG3V8J+W`LU1UJksa-rM8!=m9AF8fO zQp#la^TNw!ydU>YS_8wI1w;xzZRdMhzN-AdM(LS!=ey_$p8?!S12cjru3os~p6mEZ zpFdS_JdxIEd#H7_(8o}_GCetufXbnqul`o2K2=C+9C$t@@m|0Qp*(`>J03l1&$%=H zBZH*9=1+*keG;8o95^W%RM@yUe8&ncfFDA%qgI0a@2KuH60AHBuibU6>(ts~S((6{UHr%QQq2J&eLN zTQdc!TSTkWoes~vK}ZPfv5`U@uPl4CCN9}!-N~|6xh)8KF@CaqFAbEgfB+I(VDfvU zjMCzMzU~|{Z*oiw@_q~Zo9MWP4Co05ZeNz0KMDFaW*EnaH@co3R&Ran&rQDn(Sl;c-)T`eK~9H@muD>-;9*!}`|Euw?(fdh-t*-1+GFtc24i1wthdAqGwZ&Iww@|8{l}L6o zwpgh8za-qH`5yfG(^May+irr+BVJuk7WkyzxtX9Y_;B${Otb(Li#Zg=OXdb7VSF!W zwf=FD!3TG8nVq&)9v1hK`l)r)>mZ(;4FB!OW#n~=B5Lw1dCia96e74=upo1TQ)0pLY>+< zoAL&{kOz`eY^0PrSUk_Gi6&UT+&7rfV&bvxl6ZO^Gx?sV`+sAh6!4TM)N6OPyQ;a} zp6CqX^nGHd#KWD`PyTrmTKvJV()FZWZ^t~r4^)dT7ZD!4(Hgy9{WaGlMcTE9P$S!Y z1oNYV`XT@BV+L=SK{~tQ=sp~%EPMP!I}NXume*HXg!JeA+64XMp|%5Jv_?FsLpjJh zihSqUrsNytgE(KuxHF?Jd?%5IouDUH2$;@x9zG9~_YilxtDg_xE7zpsmATGBXZaiepw*rqq9S+uDTlx?foks zL`fs9dXYcs+X|0s56Q6&uCy;7#4QRKf*??jVyiPg(cse(|A#V(RA~_lW9CKnnp?WD z#^Q~bzaIR)A`O7%8l<<>zhzd35NVab3hj99FYD}yHUz_;Z^JLSf?)phT@M_4;P{Zs zpXvPI5#W}HEAh+oGtM3SWG!HssiH@*2Y@3kPYeT`UpQ!~`Y1lqq zp{(P1rEf?40kwHs&3e>7P!t#Z&-iYv@LAuUXmYmoRz!d~pCFfPuyO<>H~8!8>r>D$ z^~7Sz;)d`rb5O;gdE|S-Vif@CIo$eCNEI|`*SCcF>jDg;jT*VIZqgCsSIF%#EnX$v zF`QJAvN(%nDFVlD>m@M}PJjyc?R&^1Fpt068B+^Cedm4CBk+?460Jf*h5I5#w@{T` ztyn?2@V7Kfi;3ZJN49>OU=d(H*c<}Ib^Jji^tQlf1O@c6HC4Kx-cYqRVd2~im-6WvNrsov`I~lY2uliG&ZtbY%nV=G+>kK9`zN6uZVUx|~ ztY;#IaD-ZZS=Se(-yw+giMY;euc6lBb0WOhmT@~@?YiXrd=2L~0V=HABCi+RD=SM6 z{{7+l_kx8`(l;;Tr_c1enZJ?Q(vVnTt#m2J+NUc2DKlh3-ibI`tdCJgW2M`bS>?0Y zHZV}$>DGD>|B(6}bfA1}R8?#c_0BHx_b2_2Hj}q{Ak+59G?MAVKI+PW*LY6&-+sO? zs?g!kIiRS?ep!Vw*Kj{P_ac>noPQ?{qjMSokxWTkfO0DncmMbb2mGM_u#{`8aR4y9 zC4BQP%)1QEUiO7?8Tzt(^k2>-Cu+Zflh?WqpNCG==k0+Vz=RUoQ@o#&ZZz0nwBu$* z7G;Bmcrr@$>AqW+OLw@0)zDAJVXfN_S?)&IQTbx~*+T@ZJkt`rR4q=%;@SfoOkXxi zk7l!jV(mfkM%1c@*QMnZE}!ZIAT+_p1Fj*0n(nIp{U5GNd+*W(oR|74ZMqe4bdc!+ z=j*?s@3BzMeTsCw;_-hm%65h%aj=T+TvK+z0Oj4qujy$|RU`PsTUj8Eo=r*@J#^Qm zyq|5IM)8IHi*jccApbsjU;Lb00@l65E7~n?W!i(zQN&=nMtjyF*1fxqs|JsRZeilL z42OqHalNRJ-ODvlu5@#?6CDR5=hs13?lW^|StFP<#)Kh**1!+BRhwh}rm8ppCPy@> z>w$~s6tyh@<>gX!#DRv{;8p2K(5~*=fomwdp`-R@8pp<4f3FaR?|Bk@IHo%ka?B(fQ6bQ-p%E^a| zqqRl9TpEjHf#y&@tjgD14INkSI^>kcL&tY=oA{a^E^Sxqf##g-2V0U)t6D^*JUkAM zHj8XyMXk&5(E(;O&2;PvRZq|Lbv-W8JDl&8+Lr(o;F_sQVhKvTp z+i|TC7j8$=7{UV~GK_)>;!l-rt~5~iy|16{3b^*hd->6!Yha;2T5`=lO(sm5p?#qiw zqlg5L0d-uK%I9Lzf)u$`1n95{aP~K2kZU%@()mqLEYWBz`(Vr7spHo!^wOuvGXKc# zbi7ab%2nzFGOwQ!#Uze|ipbJX(S9G$j9O&8jb~dNP}z1GXDANv9r#(UR!ckAT%Ya=?cx& ztdzvmynXktq{C&0SWoD`(l8ZDMcV4tFq)nCVjY`Kwgs;xWzEDgzZ*doFSbWIKJ3NC zW#@vsNfCR8?G7!VDe3myQ&eT}@)*y8YaUR4JztGXdTKpJ66>d;MG)Zf4G1lAJHSh2YhO_sd)4^!kqNf_ZCw8|4)bim$~4hJ!5a9HVu& zanLZS~C;I*d4r6@{e0d;RHt>7JuV=9LtNQ+EHzg5umSaXF60y9GYxF|$S%zka z5nRfl!c;U(7Jo`>7RB-_pNqe)t0UX(5Qh%`_+AeW9tNSYYEfJD%)<H@QiYUi2qo2EE8q@5Q&zA-JC-~Iuv7$&Jw`GYxf^k$``gSb-4h?onImsLWes&-7( z`z@+$3@Top*G8L0+q(%lJck5f{8o*8v!k4*$g0+JhjJ5 z%?_>PKKA?ixFmId&#V@E7U>+E94CO0Jz|q|tn?6b)haz=$floLvKg)i@_Z4va(xZ9 z*gV1%U@>;l@YjEJtG3#=Zo`WV!de$KPxQ#}FiZ^mLhSHbljN<tDSshr5@7tOz%I zS?H%WPk%|)x6Y^fI5u>9lPSJePJ0)(5BS()?)#F}OZU=icl!P@>Y6&c2?T?}98E{d zG+pxbGp%PsJe$diG3F$bhEKj1UMb}|ufFwX5}@;t8R7(3hnFDapFw{^;&l`0%-I99 zdmDgMQgpfj`t1Ba!wj-yI`Ak}&Ps-fLv(gAos3SeaFpQd!L^Cj=}1m8WEZBkc(Txy zn#;$~-f90bGri{@B>@bNQqIMfQ>@Oo?=nou1CL^(sH5x(rM#@=Z@@95rDE4SQtVdi zoryeOZSgLx+7!lTqKG_faXy%PKa#pqw;BqJ)@!%<9Km166m?Yb7P<3TMX3kd78am; z07;wd_3&lrJ3aDOH^CK^ZWv35MNQ+b?9te{ag`Pt#C{>^@Nb_dX_yOS7Qc_amt{vl zQ#*iN5z8rq`L4sbhy5PjOIg11jV2=t*>|zSAD!?H(F7nMwI2t%pdO;stf24^tNt+m0(ErD1Omy6Q^>rze=$TJO(tQD-uW z`x@e*H(3#CX1VMA{tZ1E$iAa{#6pf6sjS}sZXrci3dUj)jj9F6?Z#9^V?p_$v?K4m z8@l;N!`|ceYB%B>h6w{0Y4z5U8MKDcccC#4l9*wMnilf$p- zj0uPzmG5^+H52%Y;*#Je5T}#XkW_@E}A|_NRUrybN_yb|R=yEd@;{jsf zGhFus!EDCwvWVC1DC;peY7NlZl5s*1+csmMN;0S{)u>UNmlReC6R>B=}R7`vV^)ZTdVYnjSx5xIBtxUer?TL`$A!OBNR zSyzZF#d|E!z9#Q8du@esHqUY;ZZ5eJsW$o_)5W%Tm%9O-`1RV3T$ezt@+Vkb2F2SL zffhzi>+q9NSg7t=0=a*Wx&J{@%Yn3N1MVRsRN71UdNbiIIa7;Y7$%#3-KT;dD}eSg z!OFvJ+y~`b;%wPnP0f)oSmNw+g4uEX>&Up!M8ZJpzN=#$eUrAk)liko;kx=T+E7L1 zVEt3MeF_O?I$ye;D>IN=pm=NH=ume&2VPVLlWmpDz(B58q}U@bCS-Nxc1*NEF0X0u-w^Rq zIYJ(nMXWe@>fWR|?!3JBkQEE2%V*gnVx!uVoiwns?pp(bfrR+9*C+D{8d%IdsYW9v z_IZJ7#DV0&A!EcY(1oBLrUWYuwT$#P+fhCEp;*dJ=GV!M?-Lw1&E%Hb&TQ|Q(U#3H zZm4^v2Tyr5Eqh;Ux281}bN}MRWsM=S420R?`6IaQq^B9ai^3hafGVdcYYfsuAMY_2X{izHjc&)JUgB$G@pv ze4x(|Z4F!%+ccD{ZEl10TKpDJT?_z9O`_l1Y(eAx^LxWLmMB$K$OBz+W-OPgVhc<8 zfHLmt^F~D*Xnh#0SZCH$zReP{U#VQlWcMyNL*9WaTj-HOt6 z(!7Uy$qC=AG|6qJfnkd)Sy4TKzy8wg5b8P8Nv$TYiUqYvtPjZN$+K{_`r>+2*39Q(crvSCx|ZZnDt!|6vY`rh^WMvbnLDI0pE6TJx8=^I51u9V}SB01^0 zaF44t+Xtw3-`->^saM!Ap4*tjUR6KYLTIR;Qs3LrunzJ~pE#x;^v17>5zba_DcrOP zfb{_t3?s`lWS1^vb+!X7xbh%!m%Z>BVQn#LUyTfOojhS4g0eE5FW$^Dw;LGALMO6vWTo$u;Te#Pd-?+vFR zh_2&NKE;hk*v~&%wc{6t^X>2nDR0KH)|u?~w58NrAQm!9ZLNkksgck8pqYt7z}xbz zF2fT84%6So4!(N0@Id3250_`Y=nxi=b@FkO>iKKx6wS?IG;w1c?su2mh;qdBu^Q+1 zZ<8=NRbUp z;O6sFB$-{|pv3*2^%U-81BLYy9IZF_Yi56#hK;b3AITVJvVr%5r$sS{Ro4T|+vsc* zz5`#Bj*d11(XlAqB zwwWT2`lXMvzt6K#zoDcTxSA_Z8rya)vMt?TAe(DCt?wLQ2bi%K^GwX%HC%oz?3CLe z>Ta0Q(P+W= zgT8)7g7h<%Sp}0O)T_1oEt~Hus1Kv?myELB;RR0?KTP{>VIpbSOj|$l3#>IBZjI#U z1ifk$=>EfZ*M(<^rh*fz-$B^0H$aW1$D@Zvt(+jXdV%$pEi^&o>tunrxJY)j;&CD) z9NSrZCvm=a9UtpMUBPH43qGvKdwZbZ>#_;l-aDfmi34%N2^nTyd}^#j{|G}4b9XC{ z?53Gm_1o<0TBj~+>oREUf-trZKVjd=KCCP7)4a-~?tzme$jr{71EdI!&l{`YnHW%t zHKj~@rn#t~P?_Tmo=Y!!d>oh<*FYoYDwX&TNdMp}O~8~4^@TEM3Dm+T$z00IR4fA=otxftfcnp*sdW%$s6Z*rEib-R-T#U| z5^w(xbO>(~!mQ;UcWENR=s`QM;txXUXOpEE0AfOcDQL?jJ@tyVNX}$0Le-4ho#k~) z-^$@5JPHA$S;&?ylD<`OGT$22HU3flBI}X3hT0QI583yTX}lQ!)yStphUQ@C>g!(Vu;w8Q?C{j`FyS3URT*oDU5UD zLoyerPeD$PEi8}&SDlsTP_fNazOOPCY+*lKIu_RrHa<5^%=p-=T&4GC!>laUzkE}e zY)xV+B#HQq?AgzSbT#E)A=cZ_NR2?RuJ1<08Ey!RMdIh6JaYNG=l{P382)c& bSMvDi%M&@`T!a4;{eYpStgTd~U>Wj1P*^$} literal 0 HcmV?d00001 diff --git a/packages/components/nodes/vectorstores/Meilisearch/Meilisearch.ts b/packages/components/nodes/vectorstores/Meilisearch/Meilisearch.ts new file mode 100644 index 000000000..eecc5e5eb --- /dev/null +++ b/packages/components/nodes/vectorstores/Meilisearch/Meilisearch.ts @@ -0,0 +1,174 @@ +import { getCredentialData, getCredentialParam } from '../../../src' +import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' +import { Meilisearch } from 'meilisearch' +import { MeilisearchRetriever } from './core' +import { flatten } from 'lodash' +import { Document } from '@langchain/core/documents' +import { v4 as uuidv4 } from 'uuid' +import { Embeddings } from '@langchain/core/embeddings' + +class MeilisearchRetriever_node implements INode { + label: string + name: string + version: number + description: string + type: string + icon: string + category: string + baseClasses: string[] + inputs: INodeParams[] + credential: INodeParams + badge: string + outputs: INodeOutputsValue[] + author?: string + + constructor() { + this.label = 'Meilisearch' + this.name = 'meilisearch' + this.version = 1.0 + this.type = 'Meilisearch' + this.icon = 'Meilisearch.png' + this.category = 'Vector Stores' + this.badge = 'NEW' + this.description = `Upsert embedded data and perform similarity search upon query using Meilisearch hybrid search functionality` + this.baseClasses = ['BaseRetriever'] + this.credential = { + label: 'Connect Credential', + name: 'credential', + type: 'credential', + credentialNames: ['meilisearchApi'] + } + this.inputs = [ + { + label: 'Document', + name: 'document', + type: 'Document', + list: true, + optional: true + }, + { + label: 'Embeddings', + name: 'embeddings', + type: 'Embeddings' + }, + { + label: 'Host', + name: 'host', + type: 'string', + description: 'This is the URL for the desired Meilisearch instance' + }, + { + label: 'Index Uid', + name: 'indexUid', + type: 'string', + description: 'UID for the index to answer from' + }, + { + label: 'Top K', + name: 'K', + type: 'number', + description: 'number of top searches to return as context', + additionalParams: true, + optional: true + }, + { + label: 'Semantic Ratio', + name: 'semanticRatio', + type: 'number', + description: 'percentage of sematic reasoning in meilisearch hybrid search', + additionalParams: true, + optional: true + } + ] + this.outputs = [ + { + label: 'Meilisearch Retriever', + name: 'MeilisearchRetriever', + description: 'retrieve answers', + baseClasses: this.baseClasses + } + ] + this.outputs = [ + { + label: 'Meilisearch Retriever', + name: 'retriever', + baseClasses: this.baseClasses + } + ] + } + //@ts-ignore + vectorStoreMethods = { + async upsert(nodeData: INodeData, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const meilisearchAdminApiKey = getCredentialParam('meilisearchAdminApiKey', credentialData, nodeData) + const docs = nodeData.inputs?.document as Document[] + const host = nodeData.inputs?.host as string + const indexUid = nodeData.inputs?.indexUid as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + let embeddingDimension: number = 384 + const client = new Meilisearch({ + host: host, + apiKey: meilisearchAdminApiKey + }) + const flattenDocs = docs && docs.length ? flatten(docs) : [] + const finalDocs = [] + for (let i = 0; i < flattenDocs.length; i += 1) { + if (flattenDocs[i] && flattenDocs[i].pageContent) { + const uniqueId = uuidv4() + const { pageContent, metadata } = flattenDocs[i] + const docEmbedding = await embeddings.embedQuery(pageContent) + embeddingDimension = docEmbedding.length + const documentForIndexing = { + pageContent, + metadata, + objectID: uniqueId, + _vectors: { + ollama: { + embeddings: docEmbedding, + regenerate: false + } + } + } + finalDocs.push(documentForIndexing) + } + } + let index: any + try { + index = await client.getIndex(indexUid) + } catch (error) { + console.error('Error fetching index:', error) + await client.createIndex(indexUid, { primaryKey: 'objectID' }) + } finally { + index = await client.getIndex(indexUid) + } + + try { + await index.updateSettings({ + embedders: { + ollama: { + source: 'userProvided', + dimensions: embeddingDimension + } + } + }) + await index.addDocuments(finalDocs) + } catch (error) { + console.error('Error occurred while adding documents:', error) + } + return + } + } + async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { + const credentialData = await getCredentialData(nodeData.credential ?? '', options) + const meilisearchSearchApiKey = getCredentialParam('meilisearchSearchApiKey', credentialData, nodeData) + const host = nodeData.inputs?.host as string + const indexUid = nodeData.inputs?.indexUid as string + const K = nodeData.inputs?.K as string + const semanticRatio = nodeData.inputs?.semanticRatio as string + const embeddings = nodeData.inputs?.embeddings as Embeddings + + const hybridsearchretriever = new MeilisearchRetriever(host, meilisearchSearchApiKey, indexUid, K, semanticRatio, embeddings) + return hybridsearchretriever + } +} +module.exports = { nodeClass: MeilisearchRetriever_node } diff --git a/packages/components/nodes/vectorstores/Meilisearch/core.ts b/packages/components/nodes/vectorstores/Meilisearch/core.ts new file mode 100644 index 000000000..7c1063a29 --- /dev/null +++ b/packages/components/nodes/vectorstores/Meilisearch/core.ts @@ -0,0 +1,92 @@ +import { BaseRetriever, type BaseRetrieverInput } from '@langchain/core/retrievers' +import { Document } from '@langchain/core/documents' +import { Meilisearch } from 'meilisearch' +import { Embeddings } from '@langchain/core/embeddings' + +export interface CustomRetrieverInput extends BaseRetrieverInput {} + +export class MeilisearchRetriever extends BaseRetriever { + lc_namespace = ['langchain', 'retrievers'] + private readonly meilisearchSearchApiKey: any + private readonly host: any + private indexUid: string + private K: string + private semanticRatio: string + private embeddings: Embeddings + constructor( + host: string, + meilisearchSearchApiKey: any, + indexUid: string, + K: string, + semanticRatio: string, + embeddings: Embeddings, + fields?: CustomRetrieverInput + ) { + super(fields) + this.meilisearchSearchApiKey = meilisearchSearchApiKey + this.host = host + this.indexUid = indexUid + this.embeddings = embeddings + + if (semanticRatio == '') { + this.semanticRatio = '0.5' + } else { + let semanticRatio_Float = parseFloat(semanticRatio) + if (semanticRatio_Float > 1.0) { + this.semanticRatio = '1.0' + } else if (semanticRatio_Float < 0.0) { + this.semanticRatio = '0.0' + } else { + this.semanticRatio = semanticRatio + } + } + + if (K == '') { + K = '4' + } + this.K = K + } + + async _getRelevantDocuments(query: string): Promise { + // Pass `runManager?.getChild()` when invoking internal runnables to enable tracing + // const additionalDocs = await someOtherRunnable.invoke(params, runManager?.getChild()) + const client = new Meilisearch({ + host: this.host, + apiKey: this.meilisearchSearchApiKey + }) + + const index = await client.index(this.indexUid) + const questionEmbedding = await this.embeddings.embedQuery(query) + // Perform the search + const searchResults = await index.search(query, { + vector: questionEmbedding, + limit: parseInt(this.K), // Optional: Limit the number of results + attributesToRetrieve: ['*'], // Optional: Specify which fields to retrieve + hybrid: { + semanticRatio: parseFloat(this.semanticRatio), + embedder: 'ollama' + } + }) + const hits = searchResults.hits + let documents: Document[] = [ + new Document({ + pageContent: 'mock page', + metadata: {} + }) + ] + try { + documents = hits.map( + (hit: any) => + new Document({ + pageContent: hit.pageContent, + metadata: { + objectID: hit.objectID + } + }) + ) + } catch (e) { + console.error('Error occurred while adding documents:', e) + } + return documents + } +} diff --git a/packages/components/package.json b/packages/components/package.json index e447cdf24..a548490ed 100644 --- a/packages/components/package.json +++ b/packages/components/package.json @@ -94,6 +94,7 @@ "lodash": "^4.17.21", "lunary": "^0.6.16", "mammoth": "^1.5.1", + "meilisearch": "^0.41.0", "moment": "^2.29.3", "mongodb": "6.3.0", "mysql2": "^3.9.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 29f5a0886..29594ad78 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -304,6 +304,9 @@ importers: mammoth: specifier: ^1.5.1 version: 1.7.0 + meilisearch: + specifier: ^0.41.0 + version: 0.41.0(encoding@0.1.13) moment: specifier: ^2.29.3 version: 2.30.1 @@ -11698,6 +11701,9 @@ packages: resolution: { integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== } engines: { node: '>= 0.6' } + meilisearch@0.41.0: + resolution: { integrity: sha512-5KcGLxEXD7E+uNO7R68rCbGSHgCqeM3Q3RFFLSsN7ZrIgr8HPDXVAIlP4LHggAZfk0FkSzo8VSXifHCwa2k80g== } + mem-fs-editor@9.7.0: resolution: { integrity: sha512-ReB3YD24GNykmu4WeUL/FDIQtkoyGB6zfJv60yfCo3QjKeimNcTqv2FT83bP0ccs6uu+sm5zyoBlspAzigmsdg== } engines: { node: '>=12.10.0' } @@ -31754,6 +31760,12 @@ snapshots: media-typer@0.3.0: {} + meilisearch@0.41.0(encoding@0.1.13): + dependencies: + cross-fetch: 3.1.8(encoding@0.1.13) + transitivePeerDependencies: + - encoding + mem-fs-editor@9.7.0(mem-fs@2.3.0): dependencies: binaryextensions: 4.19.0