import{s as He,n as We,o as Qe}from"../chunks/scheduler.5f3e6389.js";import{S as Xe,i as Re,e as p,s as a,c as o,h as Ye,a as i,d as l,b as n,f as Ge,g as r,j as y,k as Ae,l as ze,m as t,n as c,t as w,o as M,p as m}from"../chunks/index.62df735e.js";import{C as Ne,H as fe,E as Se}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e306bd5b.js";import{C as u}from"../chunks/CodeBlock.dc1e8be0.js";function Fe(je){let h,F,N,V,T,q,d,E,J,be='The evaluator is designed to work with <code>transformer</code> pipelines out-of-the-box. However, in many cases you might have a model or pipeline that’s not part of the <code>transformer</code> ecosystem. You can still use <code>evaluator</code> to easily compute metrics for them. In this guide we show how to do this for a Scikit-Learn <a href="https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline" rel="nofollow">pipeline</a> and a Spacy <a href="https://spacy.io" rel="nofollow">pipeline</a>. Let’s start with the Scikit-Learn case.',L,f,P,j,Ue='First we need to train a model. We’ll train a simple text classifier on the <a href="https://huggingface.co/datasets/imdb" rel="nofollow">IMDb dataset</a>, so let’s start by downloading the dataset:',K,b,D,U,Ie="Then we can build a simple TF-IDF preprocessor and Naive Bayes classifier wrapped in a <code>Pipeline</code>:",O,I,ee,$,$e="Following the convention in the <code>TextClassificationPipeline</code> of <code>transformers</code> our pipeline should be callable and return a list of dictionaries. In addition we use the <code>task</code> attribute to check if the pipeline is compatible with the <code>evaluator</code>. We can write a small wrapper class for that purpose:",se,x,le,C,xe="We can now pass this <code>pipeline</code> to the <code>evaluator</code>:",te,k,ae,B,Ce="Implementing that simple wrapper is all that’s needed to use any model from any framework with the <code>evaluator</code>. In the <code>__call__</code> you can implement all logic necessary for efficient forward passes through your model.",ne,g,pe,v,ke="We’ll use the <code>polarity</code> feature of the <code>spacytextblob</code> project to get a simple sentiment analyzer. First you’ll need to install the project and download the resources:",ie,_,oe,Z,Be="Then we can simply load the <code>nlp</code> pipeline and add the <code>spacytextblob</code> pipeline:",re,G,ce,A,ge="This snippet shows how we can use the <code>polarity</code> feature added with <code>spacytextblob</code> to get the sentiment of a text:",we,H,Me,W,ve="Now we can wrap it in a simple wrapper class like in the Scikit-Learn example before. It just has to return a list of dictionaries with the predicted lables. If the polarity is larger than 0 we’ll predict positive sentiment and negative otherwise:",me,Q,ye,X,_e="That class is compatible with the <code>evaluator</code> and we can use the same instance from the previous examlpe along with the IMDb test set:",he,R,ue,Y,Ze="This will take a little longer than the Scikit-Learn example but after roughly 10-15min you will have the evaluation results!",Te,z,de,S,Je;return T=new Ne({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),d=new fe({props:{title:"Using the evaluator with custom pipelines",local:"using-the-evaluator-with-custom-pipelines",headingTag:"h1"}}),f=new fe({props:{title:"Scikit-Learn",local:"scikit-learn",headingTag:"h2"}}),b=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyaW1kYiUyMik=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

ds = load_dataset(<span class="hljs-string">&quot;imdb&quot;</span>)`,wrap:!1}}),I=new u({props:{code:"ZnJvbSUyMHNrbGVhcm4ucGlwZWxpbmUlMjBpbXBvcnQlMjBQaXBlbGluZSUwQWZyb20lMjBza2xlYXJuLm5haXZlX2JheWVzJTIwaW1wb3J0JTIwTXVsdGlub21pYWxOQiUwQWZyb20lMjBza2xlYXJuLmZlYXR1cmVfZXh0cmFjdGlvbi50ZXh0JTIwaW1wb3J0JTIwVGZpZGZUcmFuc2Zvcm1lciUwQWZyb20lMjBza2xlYXJuLmZlYXR1cmVfZXh0cmFjdGlvbi50ZXh0JTIwaW1wb3J0JTIwQ291bnRWZWN0b3JpemVyJTBBJTBBdGV4dF9jbGYlMjAlM0QlMjBQaXBlbGluZSglNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAoJ3ZlY3QnJTJDJTIwQ291bnRWZWN0b3JpemVyKCkpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKCd0ZmlkZiclMkMlMjBUZmlkZlRyYW5zZm9ybWVyKCkpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKCdjbGYnJTJDJTIwTXVsdGlub21pYWxOQigpKSUyQyUwQSU1RCklMEElMEF0ZXh0X2NsZi5maXQoZHMlNUIlMjJ0cmFpbiUyMiU1RCU1QiUyMnRleHQlMjIlNUQlMkMlMjBkcyU1QiUyMnRyYWluJTIyJTVEJTVCJTIybGFiZWwlMjIlNUQp",highlighted:`<span class="hljs-keyword">from</span> sklearn.pipeline <span class="hljs-keyword">import</span> Pipeline
<span class="hljs-keyword">from</span> sklearn.naive_bayes <span class="hljs-keyword">import</span> MultinomialNB
<span class="hljs-keyword">from</span> sklearn.feature_extraction.text <span class="hljs-keyword">import</span> TfidfTransformer
<span class="hljs-keyword">from</span> sklearn.feature_extraction.text <span class="hljs-keyword">import</span> CountVectorizer

text_clf = Pipeline([
        (<span class="hljs-string">&#x27;vect&#x27;</span>, CountVectorizer()),
        (<span class="hljs-string">&#x27;tfidf&#x27;</span>, TfidfTransformer()),
        (<span class="hljs-string">&#x27;clf&#x27;</span>, MultinomialNB()),
])

text_clf.fit(ds[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;text&quot;</span>], ds[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-string">&quot;label&quot;</span>])`,wrap:!1}}),x=new u({props:{code:"Y2xhc3MlMjBTY2lraXRFdmFsUGlwZWxpbmUlM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmJTJDJTIwcGlwZWxpbmUpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5waXBlbGluZSUyMCUzRCUyMHBpcGVsaW5lJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi50YXNrJTIwJTNEJTIwJTIydGV4dC1jbGFzc2lmaWNhdGlvbiUyMiUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9fY2FsbF9fKHNlbGYlMkMlMjBpbnB1dF90ZXh0cyUyQyUyMCoqa3dhcmdzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMCU1QiU3QiUyMmxhYmVsJTIyJTNBJTIwcCU3RCUyMGZvciUyMHAlMjBpbiUyMHNlbGYucGlwZWxpbmUucHJlZGljdChpbnB1dF90ZXh0cyklNUQlMEElMEFwaXBlJTIwJTNEJTIwU2Npa2l0RXZhbFBpcGVsaW5lKHRleHRfY2xmKQ==",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">ScikitEvalPipeline</span>:
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, pipeline</span>):
        self.pipeline = pipeline
        self.task = <span class="hljs-string">&quot;text-classification&quot;</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, input_texts, **kwargs</span>):
        <span class="hljs-keyword">return</span> [{<span class="hljs-string">&quot;label&quot;</span>: p} <span class="hljs-keyword">for</span> p <span class="hljs-keyword">in</span> self.pipeline.predict(input_texts)]

pipe = ScikitEvalPipeline(text_clf)`,wrap:!1}}),k=new u({props:{code:"JTBBJTBBJTdCJ2FjY3VyYWN5JyUzQSUyMDAuODI5NTYlN0Q=",highlighted:`<span class="hljs-keyword">from</span> evaluate <span class="hljs-keyword">import</span> evaluator

task_evaluator = evaluator(<span class="hljs-string">&quot;text-classification&quot;</span>)
task_evaluator.compute(pipe, ds[<span class="hljs-string">&quot;test&quot;</span>], <span class="hljs-string">&quot;accuracy&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.82956</span>}`,wrap:!1}}),g=new fe({props:{title:"Spacy",local:"spacy",headingTag:"h2"}}),_=new u({props:{code:"cGlwJTIwaW5zdGFsbCUyMHNwYWN5dGV4dGJsb2IlMEFweXRob24lMjAtbSUyMHRleHRibG9iLmRvd25sb2FkX2NvcnBvcmElMEFweXRob24lMjAtbSUyMHNwYWN5JTIwZG93bmxvYWQlMjBlbl9jb3JlX3dlYl9zbQ==",highlighted:`pip install spacytextblob
python -m textblob.download_corpora
python -m spacy download en_core_web_sm`,wrap:!1}}),G=new u({props:{code:"aW1wb3J0JTIwc3BhY3klMEElMEFubHAlMjAlM0QlMjBzcGFjeS5sb2FkKCdlbl9jb3JlX3dlYl9zbScpJTBBbmxwLmFkZF9waXBlKCdzcGFjeXRleHRibG9iJyk=",highlighted:`<span class="hljs-keyword">import</span> spacy

nlp = spacy.load(<span class="hljs-string">&#x27;en_core_web_sm&#x27;</span>)
nlp.add_pipe(<span class="hljs-string">&#x27;spacytextblob&#x27;</span>)`,wrap:!1}}),H=new u({props:{code:"dGV4dHMlMjAlM0QlMjAlNUIlMjJUaGlzJTIwbW92aWUlMjBpcyUyMGhvcnJpYmxlJTIyJTJDJTIwJTIyVGhpcyUyMG1vdmllJTIwaXMlMjBhd2Vzb21lJTIyJTVEJTBBcmVzdWx0cyUyMCUzRCUyMG5scC5waXBlKHRleHRzKSUwQSUwQWZvciUyMHR4dCUyQyUyMHJlcyUyMGluJTIwemlwKHRleHRzJTJDJTIwcmVzdWx0cyklM0ElMEElMjAlMjAlMjAlMjBwcmludChmJTIyJTdCdGV4dCU3RCUyMCU3QyUyMFBvbGFyaXR5JTNBJTIwJTdCcmVzLl8uYmxvYi5wb2xhcml0eSU3RCUyMik=",highlighted:`texts = [<span class="hljs-string">&quot;This movie is horrible&quot;</span>, <span class="hljs-string">&quot;This movie is awesome&quot;</span>]
results = nlp.pipe(texts)

<span class="hljs-keyword">for</span> txt, res <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(texts, results):
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{text}</span> | Polarity: <span class="hljs-subst">{res._.blob.polarity}</span>&quot;</span>)`,wrap:!1}}),Q=new u({props:{code:"Y2xhc3MlMjBTcGFjeUV2YWxQaXBlbGluZSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYlMkMlMjBubHApJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi5ubHAlMjAlM0QlMjBubHAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLnRhc2slMjAlM0QlMjAlMjJ0ZXh0LWNsYXNzaWZpY2F0aW9uJTIyJTBBJTBBJTIwJTIwJTIwJTIwZGVmJTIwX19jYWxsX18oc2VsZiUyQyUyMGlucHV0X3RleHRzJTJDJTIwKiprd2FyZ3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmVzdWx0cyUyMCUzRCU1QiU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMHAlMjBpbiUyMHNlbGYubmxwLnBpcGUoaW5wdXRfdGV4dHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBwLl8uYmxvYi5wb2xhcml0eSUzRSUzRDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXN1bHRzLmFwcGVuZCglN0IlMjJsYWJlbCUyMiUzQSUyMDElN0QpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJlc3VsdHMuYXBwZW5kKCU3QiUyMmxhYmVsJTIyJTNBJTIwMCU3RCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjByZXN1bHRzJTBBJTBBcGlwZSUyMCUzRCUyMFNwYWN5RXZhbFBpcGVsaW5lKG5scCk=",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">SpacyEvalPipeline</span>:
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, nlp</span>):
        self.nlp = nlp
        self.task = <span class="hljs-string">&quot;text-classification&quot;</span>

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, input_texts, **kwargs</span>):
        results =[]
        <span class="hljs-keyword">for</span> p <span class="hljs-keyword">in</span> self.nlp.pipe(input_texts):
            <span class="hljs-keyword">if</span> p._.blob.polarity&gt;=<span class="hljs-number">0</span>:
                results.append({<span class="hljs-string">&quot;label&quot;</span>: <span class="hljs-number">1</span>})
            <span class="hljs-keyword">else</span>:
                results.append({<span class="hljs-string">&quot;label&quot;</span>: <span class="hljs-number">0</span>})
        <span class="hljs-keyword">return</span> results

pipe = SpacyEvalPipeline(nlp)`,wrap:!1}}),R=new u({props:{code:"JTdCJ2FjY3VyYWN5JyUzQSUyMDAuNjkxNCU3RA==",highlighted:`<span class="hljs-built_in">eval</span>.compute(pipe, ds[<span class="hljs-string">&quot;test&quot;</span>], <span class="hljs-string">&quot;accuracy&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>{<span class="hljs-string">&#x27;accuracy&#x27;</span>: <span class="hljs-number">0.6914</span>}`,wrap:!1}}),z=new Se({props:{source:"https://github.com/huggingface/evaluate/blob/main/docs/source/custom_evaluator.mdx"}}),{c(){h=p("meta"),F=a(),N=p("p"),V=a(),o(T.$$.fragment),q=a(),o(d.$$.fragment),E=a(),J=p("p"),J.innerHTML=be,L=a(),o(f.$$.fragment),P=a(),j=p("p"),j.innerHTML=Ue,K=a(),o(b.$$.fragment),D=a(),U=p("p"),U.innerHTML=Ie,O=a(),o(I.$$.fragment),ee=a(),$=p("p"),$.innerHTML=$e,se=a(),o(x.$$.fragment),le=a(),C=p("p"),C.innerHTML=xe,te=a(),o(k.$$.fragment),ae=a(),B=p("p"),B.innerHTML=Ce,ne=a(),o(g.$$.fragment),pe=a(),v=p("p"),v.innerHTML=ke,ie=a(),o(_.$$.fragment),oe=a(),Z=p("p"),Z.innerHTML=Be,re=a(),o(G.$$.fragment),ce=a(),A=p("p"),A.innerHTML=ge,we=a(),o(H.$$.fragment),Me=a(),W=p("p"),W.textContent=ve,me=a(),o(Q.$$.fragment),ye=a(),X=p("p"),X.innerHTML=_e,he=a(),o(R.$$.fragment),ue=a(),Y=p("p"),Y.textContent=Ze,Te=a(),o(z.$$.fragment),de=a(),S=p("p"),this.h()},l(e){const s=Ye("svelte-u9bgzb",document.head);h=i(s,"META",{name:!0,content:!0}),s.forEach(l),F=n(e),N=i(e,"P",{}),Ge(N).forEach(l),V=n(e),r(T.$$.fragment,e),q=n(e),r(d.$$.fragment,e),E=n(e),J=i(e,"P",{"data-svelte-h":!0}),y(J)!=="svelte-186h53x"&&(J.innerHTML=be),L=n(e),r(f.$$.fragment,e),P=n(e),j=i(e,"P",{"data-svelte-h":!0}),y(j)!=="svelte-1lrcj9x"&&(j.innerHTML=Ue),K=n(e),r(b.$$.fragment,e),D=n(e),U=i(e,"P",{"data-svelte-h":!0}),y(U)!=="svelte-1ufqsq"&&(U.innerHTML=Ie),O=n(e),r(I.$$.fragment,e),ee=n(e),$=i(e,"P",{"data-svelte-h":!0}),y($)!=="svelte-dxo869"&&($.innerHTML=$e),se=n(e),r(x.$$.fragment,e),le=n(e),C=i(e,"P",{"data-svelte-h":!0}),y(C)!=="svelte-13cidte"&&(C.innerHTML=xe),te=n(e),r(k.$$.fragment,e),ae=n(e),B=i(e,"P",{"data-svelte-h":!0}),y(B)!=="svelte-hq9iw6"&&(B.innerHTML=Ce),ne=n(e),r(g.$$.fragment,e),pe=n(e),v=i(e,"P",{"data-svelte-h":!0}),y(v)!=="svelte-7vu00g"&&(v.innerHTML=ke),ie=n(e),r(_.$$.fragment,e),oe=n(e),Z=i(e,"P",{"data-svelte-h":!0}),y(Z)!=="svelte-xlpb51"&&(Z.innerHTML=Be),re=n(e),r(G.$$.fragment,e),ce=n(e),A=i(e,"P",{"data-svelte-h":!0}),y(A)!=="svelte-1k5k8id"&&(A.innerHTML=ge),we=n(e),r(H.$$.fragment,e),Me=n(e),W=i(e,"P",{"data-svelte-h":!0}),y(W)!=="svelte-y497q6"&&(W.textContent=ve),me=n(e),r(Q.$$.fragment,e),ye=n(e),X=i(e,"P",{"data-svelte-h":!0}),y(X)!=="svelte-148sz56"&&(X.innerHTML=_e),he=n(e),r(R.$$.fragment,e),ue=n(e),Y=i(e,"P",{"data-svelte-h":!0}),y(Y)!=="svelte-86ij1b"&&(Y.textContent=Ze),Te=n(e),r(z.$$.fragment,e),de=n(e),S=i(e,"P",{}),Ge(S).forEach(l),this.h()},h(){Ae(h,"name","hf:doc:metadata"),Ae(h,"content",Ve)},m(e,s){ze(document.head,h),t(e,F,s),t(e,N,s),t(e,V,s),c(T,e,s),t(e,q,s),c(d,e,s),t(e,E,s),t(e,J,s),t(e,L,s),c(f,e,s),t(e,P,s),t(e,j,s),t(e,K,s),c(b,e,s),t(e,D,s),t(e,U,s),t(e,O,s),c(I,e,s),t(e,ee,s),t(e,$,s),t(e,se,s),c(x,e,s),t(e,le,s),t(e,C,s),t(e,te,s),c(k,e,s),t(e,ae,s),t(e,B,s),t(e,ne,s),c(g,e,s),t(e,pe,s),t(e,v,s),t(e,ie,s),c(_,e,s),t(e,oe,s),t(e,Z,s),t(e,re,s),c(G,e,s),t(e,ce,s),t(e,A,s),t(e,we,s),c(H,e,s),t(e,Me,s),t(e,W,s),t(e,me,s),c(Q,e,s),t(e,ye,s),t(e,X,s),t(e,he,s),c(R,e,s),t(e,ue,s),t(e,Y,s),t(e,Te,s),c(z,e,s),t(e,de,s),t(e,S,s),Je=!0},p:We,i(e){Je||(w(T.$$.fragment,e),w(d.$$.fragment,e),w(f.$$.fragment,e),w(b.$$.fragment,e),w(I.$$.fragment,e),w(x.$$.fragment,e),w(k.$$.fragment,e),w(g.$$.fragment,e),w(_.$$.fragment,e),w(G.$$.fragment,e),w(H.$$.fragment,e),w(Q.$$.fragment,e),w(R.$$.fragment,e),w(z.$$.fragment,e),Je=!0)},o(e){M(T.$$.fragment,e),M(d.$$.fragment,e),M(f.$$.fragment,e),M(b.$$.fragment,e),M(I.$$.fragment,e),M(x.$$.fragment,e),M(k.$$.fragment,e),M(g.$$.fragment,e),M(_.$$.fragment,e),M(G.$$.fragment,e),M(H.$$.fragment,e),M(Q.$$.fragment,e),M(R.$$.fragment,e),M(z.$$.fragment,e),Je=!1},d(e){e&&(l(F),l(N),l(V),l(q),l(E),l(J),l(L),l(P),l(j),l(K),l(D),l(U),l(O),l(ee),l($),l(se),l(le),l(C),l(te),l(ae),l(B),l(ne),l(pe),l(v),l(ie),l(oe),l(Z),l(re),l(ce),l(A),l(we),l(Me),l(W),l(me),l(ye),l(X),l(he),l(ue),l(Y),l(Te),l(de),l(S)),l(h),m(T,e),m(d,e),m(f,e),m(b,e),m(I,e),m(x,e),m(k,e),m(g,e),m(_,e),m(G,e),m(H,e),m(Q,e),m(R,e),m(z,e)}}}const Ve='{"title":"Using the evaluator with custom pipelines","local":"using-the-evaluator-with-custom-pipelines","sections":[{"title":"Scikit-Learn","local":"scikit-learn","sections":[],"depth":2},{"title":"Spacy","local":"spacy","sections":[],"depth":2}],"depth":1}';function qe(je){return Qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class De extends Xe{constructor(h){super(),Re(this,h,qe,Fe,He,{})}}export{De as component};
