From 76f689cb93272e1715f8e74d5a565e70fb5e9dab Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 13 Oct 2023 21:33:32 +0100 Subject: [PATCH] add fix for illegal invocation --- .../OpenSearch_Upsert.ts | 40 +++++++++++++- .../OpenSearch_existing.ts | 52 +++++++++++++++++- .../nodes/vectorstores/OpenSearch/core.ts | 8 +++ .../opensearch.png | Bin .../OpenSearch_Upsert/opensearch.png | Bin 5216 -> 0 bytes 5 files changed, 97 insertions(+), 3 deletions(-) rename packages/components/nodes/vectorstores/{OpenSearch_Upsert => OpenSearch}/OpenSearch_Upsert.ts (73%) rename packages/components/nodes/vectorstores/{OpenSearch_Existing => OpenSearch}/OpenSearch_existing.ts (62%) create mode 100644 packages/components/nodes/vectorstores/OpenSearch/core.ts rename packages/components/nodes/vectorstores/{OpenSearch_Existing => OpenSearch}/opensearch.png (100%) delete mode 100644 packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts similarity index 73% rename from packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts rename to packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts index c11d8b115..6cb774144 100644 --- a/packages/components/nodes/vectorstores/OpenSearch_Upsert/OpenSearch_Upsert.ts +++ b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_Upsert.ts @@ -2,9 +2,10 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/I import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' import { Embeddings } from 'langchain/embeddings/base' import { Document } from 'langchain/document' -import { Client } from '@opensearch-project/opensearch' +import { Client, RequestParams } from '@opensearch-project/opensearch' import { flatten } from 'lodash' import { getBaseClasses } from '../../../src/utils' +import { buildMetadataTerms } from './core' class OpenSearchUpsert_VectorStores implements INode { label: string @@ -95,9 +96,44 @@ class OpenSearchUpsert_VectorStores implements INode { const vectorStore = await OpenSearchVectorStore.fromDocuments(finalDocs, embeddings, { client, - indexName: indexName + indexName }) + vectorStore.similaritySearchVectorWithScore = async ( + query: number[], + k: number, + filter?: object | undefined + ): Promise<[Document, number][]> => { + const search: RequestParams.Search = { + index: indexName, + body: { + query: { + bool: { + filter: { bool: { must: buildMetadataTerms(filter) } }, + must: [ + { + knn: { + embedding: { vector: query, k } + } + } + ] + } + }, + size: k + } + } + + const { body } = await client.search(search) + + return body.hits.hits.map((hit: any) => [ + new Document({ + pageContent: hit._source.text, + metadata: hit._source.metadata + }), + hit._score + ]) + } + if (output === 'retriever') { const retriever = vectorStore.asRetriever(k) return retriever diff --git a/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts similarity index 62% rename from packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts rename to packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts index c8d09470a..aeab0e3f4 100644 --- a/packages/components/nodes/vectorstores/OpenSearch_Existing/OpenSearch_existing.ts +++ b/packages/components/nodes/vectorstores/OpenSearch/OpenSearch_existing.ts @@ -1,8 +1,10 @@ import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface' import { OpenSearchVectorStore } from 'langchain/vectorstores/opensearch' import { Embeddings } from 'langchain/embeddings/base' -import { Client } from '@opensearch-project/opensearch' +import { Document } from 'langchain/document' +import { Client, RequestParams } from '@opensearch-project/opensearch' import { getBaseClasses } from '../../../src/utils' +import { buildMetadataTerms } from './core' class OpenSearch_Existing_VectorStores implements INode { label: string @@ -42,6 +44,13 @@ class OpenSearch_Existing_VectorStores implements INode { name: 'indexName', type: 'string' }, + { + label: 'OpenSearch Metadata Filter', + name: 'openSearchMetadataFilter', + type: 'json', + optional: true, + additionalParams: true + }, { label: 'Top K', name: 'topK', @@ -73,6 +82,7 @@ class OpenSearch_Existing_VectorStores implements INode { const output = nodeData.outputs?.output as string const topK = nodeData.inputs?.topK as string const k = topK ? parseFloat(topK) : 4 + const openSearchMetadataFilter = nodeData.inputs?.openSearchMetadataFilter const client = new Client({ nodes: [opensearchURL] @@ -83,6 +93,46 @@ class OpenSearch_Existing_VectorStores implements INode { indexName }) + vectorStore.similaritySearchVectorWithScore = async ( + query: number[], + k: number, + filter?: object | undefined + ): Promise<[Document, number][]> => { + if (openSearchMetadataFilter) { + const metadatafilter = + typeof openSearchMetadataFilter === 'object' ? openSearchMetadataFilter : JSON.parse(openSearchMetadataFilter) + filter = metadatafilter + } + const search: RequestParams.Search = { + index: indexName, + body: { + query: { + bool: { + filter: { bool: { must: buildMetadataTerms(filter) } }, + must: [ + { + knn: { + embedding: { vector: query, k } + } + } + ] + } + }, + size: k + } + } + + const { body } = await client.search(search) + + return body.hits.hits.map((hit: any) => [ + new Document({ + pageContent: hit._source.text, + metadata: hit._source.metadata + }), + hit._score + ]) + } + if (output === 'retriever') { const retriever = vectorStore.asRetriever(k) return retriever diff --git a/packages/components/nodes/vectorstores/OpenSearch/core.ts b/packages/components/nodes/vectorstores/OpenSearch/core.ts new file mode 100644 index 000000000..49a1d6e39 --- /dev/null +++ b/packages/components/nodes/vectorstores/OpenSearch/core.ts @@ -0,0 +1,8 @@ +export const buildMetadataTerms = (filter?: object): { term: Record }[] => { + if (filter == null) return [] + const result = [] + for (const [key, value] of Object.entries(filter)) { + result.push({ term: { [`metadata.${key}`]: value } }) + } + return result +} diff --git a/packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch/opensearch.png similarity index 100% rename from packages/components/nodes/vectorstores/OpenSearch_Existing/opensearch.png rename to packages/components/nodes/vectorstores/OpenSearch/opensearch.png diff --git a/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png b/packages/components/nodes/vectorstores/OpenSearch_Upsert/opensearch.png deleted file mode 100644 index 3fdcfd3f09ed02c1478f690884ecafe844cb32ae..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5216 zcmcgwS5y-~vkpy=5~`pOy7b-ys0f56y@o0sX-Wtkq)6`&dIt?vq=&9_fl!pDAT0=? zO7955pa1bb+{b&*p4mD3&6(Yu**UYb8>gqEMnTF*3IG5oG}IvmIF9-+k>KO{#yW%} zjuF06)K&xl8dAt^Y>9AVHhXmgZ2%yU8vqE41OWcxps+0fz(*7S*s%csVPtE_FT+hti>ComH1eO{D|CTb80SS} z1kh=M2X3F`ll>ayrBi_#S~O`Bo^o0=_)CTNNEvwEs_y)oyP@YNoP{UN+3{qGGt1 z1q^{vN}i{`zvzOl<5c%}?3z@YuHI@~Z6|OcY12_LyhH+e$uZZ{f3?fb zM{s|QPvZoO9#YnIr#)ogRI~;SmDOqU&vlOK*0^r(`hr0yHIZimYQRi`t4`=kEnST) zlwl^^&}1-qQD|>AkjLYpb0{%K6C4W;__XrAaG}bZnGxh|a{;U|S(;gHP6M(FK$0^C z+Mw3{s_zeXvVJhpk-iC5HQ(PjgYM+N5)#1jJ49&}E1X>Fm!;jBiW{V1ZYyS`9bZiV zAs~}J1ZVK;sV0T=QHpjG8S*-)}izSw5Z%tPfksPGK!$j)Go(2K|wk#`lq%*B2Z|5?vQxUwMiY z@9GU~uFs|Bs>%~WK|%hQ>6AN)J>DlcP+eQ6gtoV=o8z9es`E2qEQ9(%2B zWTy^pu7^!)$m9EI#HR%R6|?SWKMRoEB_l&AQ}rPJU11wjAw!$%qPeeJV%Px5Yau6# zwCsxSPz0W2#o6}m@OWR6J`zfbfSpNB6cUERke#8p6~bk6x;eSTihSut*G}Lw&8TUh zq9|$9>Q#EmRB~Su*oAmS2)dIu*3{+wt6})(nc0Gu^mMY_qqi@U1FCE4+Y){0iVm?z zHHTX*9VOXS0}`p5(Bfi=huTvVkjDlzYjEk2KpiW7S#QPR5w4k*0N?X*h3-^}mLttZVCnF4jc27!&-l?qfO{iAZK5k+jnL(L*-hJx^Qhw;0=X z^5#1Qy-sAA8XZG8x|x&LGWZ?Wf83jYCz$kDE%}-}fzW4yHON1N(y*P^CKt=kFj&ulvcnZ3A09rxHm2Jw1el#|mV zLK`(dNrV5b3pC5s_`X&uxcM$lMn<;&HxPIFc($_m;SCprbfkbBMiq~02LhbAj#55l z#(_2Fw5&`*3(7KKQu@>t-x$y^W-eP<_%R|U4U)k{{kZXIeND=k~5 zUybZrcSIj<`P*CY?D6hLmx7L}<{D3~S)1i%vU(IHWQLx&x$Nz<>zR9YR65xBJJ|M8 zKVsR~RJ%*GZ(0z^G@edgeWM;ClItf7SpyH$r5`u;6Wv3$*Oz{-FSRT)T24pgwK^&< z51#R9%&;I|m6gFjJgxGAc4|PKcfl#Wy1_#tHD2O~HF{u$m$zMGOVydT{&wz%siJr9 ziDEd#1&#K2T)6A!zt@$%5NRE$7{HpQ`wz)FB2r;9R?&vO+3N{orZ?Je?K{2=*FMv`DAa=l#Bt6%8c2Ip(0r{#5a=QTaGCiL=4=HhP_PlRUVTw&bgn(LL>`6(^ zl88P|*V;^jPl^1NA-u(aK`!Kl203-DtPP;zG?(a1x;B>r=$y`r)MszZ{;zKZ=$; zyv>sYe}?Uy?Ym8dt~H&IEtKMLJgLI%ro8XuR29yo&#BO=zGP+aQLDg)aCt}Vw(ddb z=iP;S(W-=!JF97-I;c`Aakzi6`WNo|L)LxoE2!vI9xJ(AeLl>e;{WpJ;*^3+V*@5; zCdy;3NGc9Dg`~;&aO!l$_kU+oof>afKRma--RVeHFD}tyg{lGBXN85?Ib*h?owYqd z2K&8n*`B-5;W>P+e-1X^bEF?)u2v+vph8|=$nf}lqL`yDNr8Ens4>6SdGy`mmqr5> zeRiq~#@;_|QJ5J1RAG2m+wSn~Q%&J&57z^@TqHRUIQ)X^sOzn`8t{aIV;ujGa^R;G z_(;4ckhzmp(UKI4x*3qtsW?H_eG|D#TN!Z@E>}$YkO|HB=Mt4tLgl_3wf=`fu#LEt>!w&dxH%~qo{K^5azYVk-0x!fA}aCm4iFZ*#r31{6%4d$ zK$RU%Oe(sVkR`D75t)gHQFsE^Og{PQ*q1fTPZzt6;qfWe9RSCn#b&gBH}(s4H9fwg z_LhDgdfyO9qb4fmyT$xO5VCBs4HW%oAL?0Xn#tYPGF(vjrD=0u;PtAY@Ka3Yfxc7pKO>-N;Pw(dHjwW=MMiS@= z@|Q?^w!K3FO;*ea0qC0_>$MlLOgtL0f9Gv{1inW-1I6rG7Tm}`)GI0EMfv>Zk&a_mi&7z@h8nV(J_8jP_?2JsskE zZ`FYSAN;zO%6?D(mHJpsg)B(L5BjXE3(EV&>BB454sugGcHBc8k?T1R>8)PzT~4MT zIuMp8x8~zPEJm)gMC?VO(Z~vwQ!u*1^`#K|a?~Yk&Z}>!%&d=3kdO9lIzF5HcVkzf zcM{+L*}n^7TpOrD$mO5mLQ7of@Z`$Wm*qec`R}m_b9#3FHsq`9>X%fTf5EjsaSsxu zkJTzng^-W-nvM7?=Ct>o{rpBll27%vwjqxWr)FE*? z3=A5}Y1DM_?zekKTGLyvIcJ(8;ihA4m+UKsd}{;yPDHSS5Ga|rt0uazuFNWJ9+NCr zJWpVv7=(VKURvEh_&I6|ZGPOFU7YnSv@a&0q{p~9YLIk}WJ5qhIA$rj>A-H8!>7B5 z-8C@MhGxoge05<%U(@$IchJ_$mHZV3*?AP-8QZ#VBuYx;%j$U+-C;1rH=XgVilFdv zgI3Op%q}p|e0G5aLU@;i@&Psvj51GmuDI4l-_kTMFGc>mNBJHx95noc>7z*eZP;y$ z-WNs@`3Tp52i{%zii|;)Fw)(iMPsXGrnSm!4*lr!8uN8oqx*ad0$eBbi4(j%g{^+M zA62uvPwRK`o;#1?m3*W!jzUG4#($F|g^r79wE1;c%bd2P4gWEtc{GeyvV$N0EN!Ds z5pb6Feb&VXd`SkC@Q@aWSD>9J@f%hK0>+ImcVCZ@R#mEHxFs1_C&59$BFpky*p(Tz zfS_Q4I~D-@!ob=U*MK+g2_Pe*qsj0EnR5p_v@uZ>85v(Z>|4kr`zjZc`RU1oT$27IaB*YpUP86hCM` z71Nq3)T2lI!WBgls*+HGo;(Q2l3O2Bx$4n+Xs=Ud?NA#X-4lXYi%Uu9`*QV;)Gp;( zH~=OmIv4em@zRCr5s!*(8``$uZ+a=^0>|3n@X6>5$3xb>Ul)}#JZTM}1k=sQ>G0Zn zTj97XLl3cR^7ic2{f*LY9yz>}CzLhcegp|hfX`S_EWpv1$tr>pTc6Y{DCl=d66gcI zhAqW6*6IBvB79T4_P~>GG{2!fOyVxd(={l^Ygk^j+JjR6V`>A@wnb{r{%b}13fk9F zl}|)`&U%~8{4de#bbhO=G0qQ6g|&>{>LpuTRv|Lte3(+}fr|JSwc?K^s??ODGQ3{M z{HkZZQp@hvuj4Phok3@>w=pLve4wR%kP<63LDZ`NxP5m>8P<x9OGj@6jLB6yZQ*3Xp>eWk3-#|xM0eb`Ey z9vM^oJe8^1reo*m%zEACUJhvBH`sl-U5n;QXub}&*JE_8BRuu7B+WI8pPxeGAe=6g zJKcVeLklR*OE-)j3MO9c{N?*$2mhnU$OyTpY;0<3^pcw8dNSr)AHHxf#P3-IC3j3^ zBxLm+KHk~Dh7;^uq*3T)AC@YvL=TI#@At(^Z&_0Yp9n{pfm?0i{@TuTxFWxh$Gy68 z!<8cPU|l;XAsAHQOkC3B;B z;%WEaAUKJ0Z{@;9pgnm&F)Jjv3yq(pfiG6$Sh`_r5Zv`NX`g$Z;3U$RJEoYCl@aG6O&xfFDjp7r#P67?Y6eB<<0g$uW~X@`kH;;S*5yMzZY=(Ac7ZT1xg|@yXV= z*p(nk^fx7fS4x5Z3P5=LF;mB5789oL=4o=>4R^SzpC>N{8|2vLsaoC7`xs{eOV2T8 zQyZiYPm%|1xu|Gz1und83~ALgc?ZH{hvYFcJy&}=f7S)f`t=$ImAgc$#!qKVze9wa zN+O%^?j2poAeeA-o>#YdbVG%-Qf%9Z|1ocQT%0jcDSpD2b&bbG>6_YRj&eRN%ONap z?S)-wHoVF_@KbWx{TC157i<#5z0`m=%X+4pq6);LTO`_#|DFY zIJvkxvUvx<9NAo8-u3{1f5F-@If