From 8b7714197d15661a64a73631a62b917876178c76 Mon Sep 17 00:00:00 2001 From: Leon Martin Date: Wed, 17 Apr 2024 13:52:46 +0200 Subject: [PATCH] Dev (#37) * added branch dev * finished license paragraph in readme * improved logging * removed obsolete token * optimized docker configuration * fixed docker config * fixed issues with docker compose (issue #3) * refactored navigation * styling improvements * package update * fixed bug in OWL approach * prepared interface for verbalization * removed unnecessary code * added fair project type * simplified access to stores; form simplification; navbar improvements * migrated from rollup to vite * updated docker config * jsconfig fix * updated .gitignore * ui and code improvments * mode is now also stored in localStorage * restructured frontend * restructured frontend * improved componentization + refactoring * minor * Fix #8 by adding error handling to license retrieval * Implement FAIR Project Type for Validation (#27) * Add comments to existing SHACL project shapes #11 * Temporarily fix error when there is no result path #15 * Model FAIR principles as SHACL rules #11 * Fix semantic versioning regex * Use match-case syntax #20 * Switch from short SHACL notation to more verbose one * For files, make check in root directory explicit * Extract the sh:or and sh:and components into node shapes * Extend repository representation with homepage, tags, and DOIs in readme * Introduce property shapes * Split the shapes graph into several files. * Fix selection and display of project types * Extend repository representation for FAIR type (#13) * Add check for valid version increment to repository representation (#13) * Create separate methods for each repository property (#22) * Limit repository representation to requirements of project type (#22) * Remove unnecessary graph returns and process description literal earlier (#22) * Make check for valid version increment less strict (#13) * Visualize repository representation ontology (#21) * Fix cut off letters at the end of words in diagram (#21) * Adapt verbalized explanation for FAIR project type (#15) * For sh:or,and,xone: use source shape instead of message (#15) * Add URLs of IRIs in README.md (#21) * Replace "has_default_branch" with "is_default_branch" (#13, #21, #22) * Add minCount to qualifiedValueShapes (#11) Previously, qualifiedValueShape only included qualifiedMinCount. This meant that a graph was valid in which the specified path did not exist at all. * Add missing minCount constraints (#11) * Adapt property shapes to changed default branch representation (#11) * Include number of violations in validation response (#19) * Show share of fulfilled criteria, v1 (#19) * Adjust button of progress bar (#19) * Adapt checks for files in root directory of default branch (#11) * Use properties from the Software Description Ontology (#24) * Integration tests for validation of FAIR project type, part 1 (#18) * Use camelCase for own properties (as in SDO) (#24) * Replace "has_section" property with more specific ones from SDO (#24) * Integration tests for validation of FAIR project type, part 2 (#18) * Introduce mocking (#18) * Remove OWL part (#25) * Fix ZeroDivisionError in benchmark.py (#26) * Adapt FAIR criteria as a result of tests with 26 repos (#14) * Remove "using" as keyword for usage documentation because of false positives (#14) * Recognize all keywords per heading (#31) (#33) * Improve verbalization on validation and specification page (#34) * Add sh:message to node- and property-shapes (#29) * Use messages from the validation report in verbalization (#29) * Add sh:message to node- and property-shapes (#29) * Use messages from the validation report in verbalization (#29) * Make check for open issues more precise * Display verbalized explanation as an unordered list (#29) * Extract quality criteria from the shapes graph via RDFLib (#30) * Add descriptions to the node- and property shapes (#30) * Use Markdown to format descriptions (#30) * Use adapted Markdown renderer (#30) * Update Node and package versions * Add information about SW requirements and citation to README.md * Organize quality criteria in Markdown tables (#30) * Correct cardinality in README.md * Include lower-case letters in the DOI regex * Add code and resulting files of the quantitative evaluation. * Add information on how to execute the quantitative evaluation * benchmark fusion * change benchmark to scatterplot * tests with scatter plot * reran benchmark * refactored evaluation * refactored evaluation * formatting benchmark results * cleanup * statistical analysis of runtime benchmark results * updated benchmark figures --------- Co-authored-by: Leon Martin Co-authored-by: Leon Martin Co-authored-by: Tobias Hummel <71486844+Tobias-TH@users.noreply.github.com> --- .gitignore | 2 +- README.md | 41 +- backend/Dockerfile | 2 + backend/benchmark.py | 33 +- .../conformity_per_best_practice.pdf | Bin 0 -> 16738 bytes .../evaluation/repos_expected_to_be_fair.json | 1 + .../data/evaluation/runtime_benchmark_box.pdf | Bin 0 -> 6309 bytes .../evaluation/runtime_benchmark_results.json | 1 + .../evaluation/runtime_benchmark_scatter.pdf | Bin 0 -> 22012 bytes backend/data/evaluation/trending_repos.json | 1 + backend/data/shacl/property_shapes.ttl | 4 +- backend/evaluation.py | 505 ++++++++++++++++++ backend/shacl_validator.py | 4 +- docker-compose.yml | 8 + 14 files changed, 574 insertions(+), 28 deletions(-) create mode 100644 backend/data/evaluation/conformity_per_best_practice.pdf create mode 100644 backend/data/evaluation/repos_expected_to_be_fair.json create mode 100644 backend/data/evaluation/runtime_benchmark_box.pdf create mode 100644 backend/data/evaluation/runtime_benchmark_results.json create mode 100644 backend/data/evaluation/runtime_benchmark_scatter.pdf create mode 100644 backend/data/evaluation/trending_repos.json create mode 100644 backend/evaluation.py diff --git a/.gitignore b/.gitignore index d410dc8..5917194 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ frontend/node_modules/ frontend/dist/ backend/__pycache__ -backend/git_access_token +.github_access_token backend/data/benchmarks/ !backend/data/benchmarks/.gitkeep diff --git a/README.md b/README.md index 8a59695..2500883 100644 --- a/README.md +++ b/README.md @@ -137,13 +137,44 @@ Instead of running frontend and backend using `docker compose up`, you can run b Note that the frontend depends on the backend. The backend should therefore be started first. -### Benchmarks +### Evaluation -To reproduce the performance benchmarks shown in the paper, perform the following steps: +WARNING: Rerunning the evaluation will overwrite the results that are provided in the repository! -- Create a file called `git_access_token` in the [backend](./backend/) folder. Then enter your GitHub access token in that file and save. -- Run `docker compose run --service-ports --entrypoint bash backend` to get a bash that is attached to the frontend container. -- Run `./benchmark.py` to start the backend in development mode. +To reproduce the evaluation results discussed in the SEMANTiCS 2024 paper, perform the following steps: + +- Create a file called `.github_access_token` in the [backend](./backend/) folder. Then, enter your GitHub access token in that file and save. +- Run `docker compose run evaluation` to get a bash that is attached to the backend container. +- Run `./python3 evaluation.py` to rerun the evaluation. This includes the FAIRness assessment of GitHub repositories and the runtime benchmark on the same repositories. + +The resulting files are place in the [evaluation](./backend/data/evaluation/) folder. + +## Citation +If you use this software, please cite it as below: + +```bibtex +@inproceedings{DBLP:conf/lwa/MartinH22, + author = {Leon Martin and + Andreas Henrich}, + editor = {Pascal Reuss and + Viktor Eisenstadt and + Jakob Michael Sch{\"{o}}nborn and + Jero Sch{\"{a}}fer}, + title = {Specification and Validation of Quality Criteria for Git Repositories + using {RDF} and {SHACL}}, + booktitle = {Proceedings of the {LWDA} 2022 Workshops: FGWM, FGKD, and FGDB, Hildesheim + (Germany), Oktober 5-7th, 2022}, + series = {{CEUR} Workshop Proceedings}, + volume = {3341}, + pages = {124--135}, + publisher = {CEUR-WS.org}, + year = {2022}, + url = {https://ceur-ws.org/Vol-3341/WM-LWDA\_2022\_CRC\_1149.pdf}, + timestamp = {Sat, 30 Sep 2023 09:52:13 +0200}, + biburl = {https://dblp.org/rec/conf/lwa/MartinH22.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +``` ## Citation If you use this software, please cite it as below: diff --git a/backend/Dockerfile b/backend/Dockerfile index 616fc46..0d0b060 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -14,7 +14,9 @@ RUN pip install -U flask-cors RUN pip install -U fire RUN pip install -U pyshacl RUN pip install -U numpy +RUN pip install -U pandas RUN pip install -U matplotlib +RUN pip install -U brokenaxes VOLUME [ "/src" ] WORKDIR /src diff --git a/backend/benchmark.py b/backend/benchmark.py index 449e43b..798b2c1 100644 --- a/backend/benchmark.py +++ b/backend/benchmark.py @@ -8,6 +8,7 @@ import matplotlib.pyplot as plt +logging.basicConfig(level=logging.INFO) def run_benchmark(): # retrieved 2022/03/22 from https://github.com/trending?since=monthly @@ -34,12 +35,11 @@ def run_benchmark(): "vulhub/vulhub" ] - with open("./git_access_token") as file: + with open(".github_access_token") as file: github_access_token = file.readline().strip() - benchmark_scenarios = [(github_access_token, repo_name, "FinishedResearchProject") for repo_name in - trending_github_repos] + [(github_access_token, repo_name, "InternalDocumentation") - for repo_name in trending_github_repos] + benchmark_scenarios = [(github_access_token, repo_name, "FAIRSoftware") for repo_name in + trending_github_repos] for github_access_token, repo_name, repo_type in benchmark_scenarios: file_name = f"{repo_name.split('/')[1]}-{repo_type}" @@ -47,29 +47,25 @@ def run_benchmark(): cmd = ["./shacl_validator.py", "--github_access_token", github_access_token, "--repo_name", repo_name, "--expected_type", repo_type] - run(["python3", "-m", "cProfile", "-o", - f"data/benchmarks/{file_name}", "-s", "cumulative"] + cmd) + run(["python3", "-m", "cProfile", "-o", f"data/benchmarks/{file_name}", "-s", "cumulative"] + cmd) sleep(3) def process_results(): - all_finished = [] - all_internal = [] + all_fair = [] step_durations = [0, 0, 0] - for result_file in glob.glob("./data/benchmarks/*"): + for result_file in glob.glob("./data/benchmarks/*FAIRSoftware"): stats = pstats.Stats(result_file) for k, v in stats.stats.items(): _, _, function = k if function == "validate_repo_against_specs": - if "FinishedResearchProject" in result_file: - all_finished.append(v[3]) - else: - all_internal.append(v[3]) + if "FAIRSoftware" in result_file: + all_fair.append(v[3]) elif function == "create_project_type_representation": step_durations[0] += v[3] @@ -86,16 +82,14 @@ def process_results(): ylabel='Seconds', ) - ax.boxplot([all_finished, all_internal]) + ax.boxplot([all_fair]) - ax.set_xticklabels( - ["$T_{F}$", "$T_{I}$"]) + ax.set_xticklabels(["$T_{FAIRSoftware}$"]) plt.tight_layout(pad=0) - plt.savefig("./data/benchmarks/benchmark_results.pdf") - total = sum(all_finished) + sum(all_internal) + total = sum(all_fair) step_one_percent = '{:.2f}%'.format( step_durations[0] / total * 100) @@ -109,5 +103,8 @@ def process_results(): if __name__ == "__main__": + # fairness assessment run_benchmark() process_results() + + # runtime diff --git a/backend/data/evaluation/conformity_per_best_practice.pdf b/backend/data/evaluation/conformity_per_best_practice.pdf new file mode 100644 index 0000000000000000000000000000000000000000..554d3a438eb34b14fbdb900888ba2dd2df4eb229 GIT binary patch literal 16738 zcmb_^2{@Hc8+Tv%mLnwV*$>AS*|Q(AE8DST?KmW<>`N&7ma;?yQ4448nz2JmjC6(OWJ-`zRD)y%!px3`8 zr%iHqq&UG*YrUyCy8>R}2sKwA02Pv*hdl`**^5FZ*|;(KW^|ah#~VuR7@icX$S}(? zm@HQ1)ZSubn!82&2&Z6_JrZ^R1LNhGT_jZZ*B!T-++!HI)sQ47V_f)s{K>GN<#OrL z_l~&_iwRnbKbTG!X!Y~awchzQq#^O@?H99n|Ddjz?gRH=oqPOC!aWk$fA(COJ{amN zA<3KbLA}>^U#+poVHn5R&g7S(}1raA1cN;PzBsKQb2 zTc?g^^6L)fC|d?Sa{SD@;E}3X-K?uft-@WptRTU zHpaE?4xo#B=CAB>W&tQJn zkT`Z>)^+Eoi2k$AEYh5()`g6QrIyrv;Z@unbN5q*E}Gv(Mun>79%x||Ta2m=b@}QU zgAjT^!*X_3s$%|d<lQk;lJ%;H%}*_ZPMgQj9W+ibT{vO{zOzUSQ=-Q%G8Zl!hm zwf99<%&`0CDFpl$nl800OdQAJp>BfeAiYlFoI*9lsE%KxN9HuhX8WMjW&6a z{;|hLN;GGjOWI9>sFklwi%f!WHOEAc8!dfxOg(ydv@Ly(_h;x3Z{2*pzosy$1KCg+ z!qrO}I?tSF5l8A2lK=e3CBZHr({lgE3tG?NMrMZZ2UV&Mo#}~b(Wqt{RNkW?W;%Vl znSDU2=jYvAE8@$v5Ji04P5n%Y06N}XWj;1+;e2O`uc`4v^n|Bn&y4&7i{-*xv0mZf zi0ItCaq_uu9>%k>w4N)d(U##&!OHDmS5l}E&ecm}ijJB*=X9b{6V&DwDM<}>3fbVF)_9TDN@TqSS7F=hztDJ2D?-U%@b1^vX#GYqYQU@C?(5~# zp}27LJR0s4#((1>_s?YiXcorP`qKkd%gZ^o9sKUnFa<6J7KJ{xvKO~6U%Kc`^Es_4 zPmlB{ZM-ZM=EWd;Qe=|;__iZ&4Byx>Jn7u(GT_~~?Bg)3I7F5_gO1KH;S5Z%67NL# z_c3QrB{4nu5U7{rHsjGI?3^cX%Ez$EE4un6NJsZUDe@jAe=k?|?uVIfZx2|NROian|N(h4CZNswTJ37d_E@Z;HhNbwBoB|^ig;heq{`yLe7|RK? z4NE3(`^%^7Qzhlst!u$ zCcJ&P8J(K0@#MAHai2E$E#p(cQ~j z?E<1U-g(bd>oK_syM2ola%qiYU##DM3m{r=3J*7T-^rJJ9U&zxK;(2a#tVquvUqH7 zHJP|z!A}-XlV;D}B=#$W7?JV)ZPk-oyxSjn@w>(9g%Cv<<7#b4|aQY)7$mpT1y#A|VK zQ14afv93k8mnobJ{%Mku?q%mrcr4L~1s+J!>Mx)X!+PgjX9DvXu?$xkf*{VCja~ygs|Fo1@+blp_ZtDNA^|AV z0>Fje1tHk84h;}c00I0962L{*Y_CUdQ#pO@F55ThfQ87Mo%284A|!SV$xlRnakaL; z94=5X!g=eOr=d1cH~%E_@jNLW*>RzcAJ|qu$vt0LPt0*tUZRON5R3~~Noh}Ve0R)W zf!TxGmI^O_BDXcbg2rS2okn?LRU8e9x&01fa{V!)d|lNE>xE3-dU;tCN{eli(t^7I zVedgVq4tZ*10+Yy(d30Jx7q5P5A+3>;9jjKS~+Z!XYaBLa!Qc)_c`7Oo;|VsI7i(K z3CSKfEbg?A3e~x?TnAF7b9xgtLtG=IBd|{SRLmjz-T{iiHRq#~swM#u8 zFi&qz1>}7WE9%YFJ;yV5UqJj}?n@2Tt^Az#pSJVd63^xCeNlCm=8e_IAXB|!xv3q_ zFOB`QI3B-$Osmc1CR-G6+sy8v5yK2;2Y;`5#N%Bu#Sx36Lr)%HkDS`GuYnjE{89cY z&T@PYyH>-Z<|Eqo_h*)hTHMa9Z1Qw+tfnGRDhLpb!Tq~HaR!?1n^4Y`hl9DLSyWw11A1$AF%B+vSy=AU5DR{-iu(&6@AF)ytRvL zjA6dAH*HG_=ChfDLK>$mr@l_(37=7Huu_DxrF#1W@~PZ7LzTkr6L#0lZrW;04ioRy z_0MLCwyF0Gi{H^5>^(f3<2a>h>2p-i&H31F@10U~aV&hYwObeWy%(31iF2R7BodyK z%P%bvz!#Zq)WuT5w*Rbypl{n1-RY~?+Igj?bN5awsk`sD7rmpeCVHg^`zShlDaL7k z!SqVtmAQL$KQ}4s7*+i(Gc<&v*Ub~@4a^gr4lqyP%(UHCMC7)-UQsYe+&6eXW;Z!UkV{Ww5e zS<5~-vW!2LWNQ=Fh{l%%wCyvcSKQ;O!}9nm_f6)T*iE=yTy#`ug0>UugoWFHurRl< zYcz!=Q=y4D0Bh>ZqOHneLT|EX)BWh}##Fk7c6IBzmXi8+ao}PeKy_QiREUDMlIw^`yOl2lVsMpx$Dv&U&P0{n%Hz--jiF7d>pqdzr)b0+bbe&S>PDi#il#$ z{jOaSgm6C!z6_%mx@;a8$V6XbGh6S)p zrsZE_`P>`1%ZEF1#(utN`IP1wd!v39c7hw`_$6XnF86Ygje|a{Hat}MnYC=zWWA%J z?(0306cNv>&qKWYFE?CbOurl6?0Q?H=<+*l{tw9gny%xTE%;6+@<_(e4WpOZF83b2 zy+l4))8_I#`Qg3rwk7EiBSN95OB|jpP1R-Z12x{M(ypt8PVHu*LsHQ?Rc1Bl8%8HA3olFngzf(y^VrVNP%wFO zEFO&ue!7DzYqKZ4$~kBX5nXoHQ=xZwc~;i7%faEbc}_~?yTq|>4%<&T{)3Oz ztjvU)K7=^P+!;3@?AnUgxsh2Yo;NJ%%hJF9ip4zJ!P@RD-d>y}gA~Q&DoF%ucjNoH zp3}?G=-0=!tz3zPr}^ZLsvH>~->M$D&n{)x*oYFt#Nmfd<`GFJRcVwy%~rIXX882t zRh31&;+}mt*3il1 zlI+Uqdw7YC_v-L|Ef(X!7x}u=#d3q)3w-AN6h11lr-pT~8^AiMaXMr={%s)(S8@(R=x?47o z8Zb5%uVqEsWsfSKtRx=1oPA?@58EEwH(0!} zw@^mXIzi~|*MOm{BnN*#q9z+M*uSXN>G2kPcS4Qhu1D!wys8&Wu&Jf8nz)&Og}0S; z3-}WkzwG)Jdh*qZ7|WUdEehv0Y5laKry@M6xMl-u4uoqOk<2yUIhN};^?j-pZpsRu zU+E{d?cAm)RC!m()=ppx0`-j$G4w4ScXj_+N3zMXxL&CG!{XPPI|EbR9Q6rm-}{Jk zkOjk6n_}FV`zV3AMHMeE$A%sd+989iq4yqecph2D|E64k*rAsEw4eQJk0@=MVdbld zAa=LHC9m&AOm5$w!*qR4f6#mfix^8Tswa(bBs7`Q%NtZ_q0Yp9aLpFZu04<$?5c1) zLM7#Zi9I>*ctxTr!by0 z-!uV%QitXj=Js%h*M{0n_vb5EVK)|D^X^NGVzLQ~^Xw}caldk5*NRV4m+97N{I~OF zM|TW7)b2Ckbd-ESXXl*JNN9DBwt|(#co$zcG*w4tGlx(r8xrAOM!8>GFQDI#OUDUc ziJG2&A?`ys%QDG?kKyMS`zGoagb&V{IyJpz4_Rr&YWN$kYR$M1d@loMpMmYCjCUc& z?)gx7Zy#XyI+U&?3FGgW5WTh|UB9UOY8KJJvq0hccv0rLcgHLxsK|>7)IZ*#gy>QHdoZLd97(1Mf#2`t1I8 zGWa39P?-qt^ka>nOkKmr!DM(cFb8d<>@L)+qUbXwDA(k-?k9YI$Qa_amo#OpAsl&Q z--QpD3nEuUPiOjaihpduTL{O?kHX&&iIJ^&nTc%&-P#fvWgMQ+inCX}wuk9Rx7TPA zqq!2%A9%0%>^oB=*K$>6++L@%NFZ(SsosmUZ`_`Djy<>6O=_m_KK_j;^zaPf=LjO; zL4WD?yKm&tN~MDDMGif3H{eQVS7Ngni{r@8)z370_{!kZ}bnvJ@(|*=9^qbD#@p(nj;E2^P_838Wr?(mJTFq zOBg?LH{?3DoEY8fPoWbnz|JPlb{{G4W-^UD#Xe71*7!Eo=gJ=@FR9N{yJGTe%bea%$_w)j?^^P<9L0A!N;5SqH`U4- z8_^1vV?_LN#7p<2*Q#M2*z92ITWDyZ+4F;3y90aOWnYEqzDu`lRbtM&ddc6T?bDua zBJ?%D9=c9n?#sS~!HDt)J+BTsKB?$aUiZbbGV>s2Jjad7m{Nn>8!%br$Vz>rJ`#&G zL}I0gNGUuKiNYCZphmYa1g2J5A|G%g+jr-SZxxTSR7}k zfId(t+&6shnUWJ_Ls0#JUs7OGQYg3(-o^p zg&}HadLs*uj)CTR8erMRIF=Ook0#V!S58wNTj_6O2_?`nl*A__r_9ib#NthY@eaQH)6h|?IEj(y}F;6J`=>Rrg_zEcSaLkmH$48AEwLf&COz-`|z%dZNpphCvq`E?%&lMNc%&!U$j3ov^yXIKDp)A zZt+As9)>pSr1`kLb-THBf5BdZs35V#J_bsTapR50ugl-}`A8{C%)ZV)^Lfe1IK?{j ziPoH{@3ByY)PgjhPw|KR-ss5gd(6w2iivsxc4P>fzgHSB#Vx7#W}I?N0#6AZ7>J-+GMYQwSYX$o#Ql*hz#YjpITvY+#j2N>WSUu1tz=i0&-NS zqz1jQ8!*VKV0lzQZF_Ow&@)iuO-k&SiW?B}wz#DItm8Fv=^ZDnIvaf2_206^Yaz7k z$iBi+ed}~D%ixTkrAGC1Vh2BW`-*;`IBA&7D5PZ3eR7KJ6)VrpAh95uQWweKk0f;O z%2GI|<@A{a+h)lC&Z4op2P~WNufr(wI@i`<(q zEJ$*FgpdVK-IJG_lFwL;tHzxFA*N_xrnubl%;&*NRI7$3=X?4K_dMmA(mb`KA8k=Q zfA8&XzZs!{_Xv}l5$Q7w-qrSq+J{%3zaKl4uJTZ!2ZP5Xm2BLQBUozq&b7(MIW=oGwLiP#(^R)Zr9gsZVkJI)h-o(Vp)iU>j@jG zH+_T;&TJKqIg?Z<=6Tt5{)rZnh4#lGSn6n$;v{q6qYlHbqwG^7BR*q+)*h$WGzNFH z#PRRu3G%sYx$?Ge@!rabFxjr#TsOI=-=0kE^JzY28_(gjIci8Gf6PVFyq`ReRzoI5sm-*%XADKfCNjb~fO2g9L7 zylhT&b?eXE=Aokgvtm<6Y@E%s+8OR#u5nOg7&0V0FO)yX$0##Tg;y#l7K7e^9^(vZ z+?tTgQ#iU^NcdTyjuE&<+?{imWSE$|WO*kfS1dA!ZRz%@2$?#Tp8+fWwFme<2R+u~N` zcPZBLGb@{qah;}eN`|EZhta4F%wT!_su)^uwP6*y$F$z>V}T=Ql-9MCuNsT<`<`&B z5=C)o@gfNCL?JW7moomzjV!0a-nrZ3g+x^m38g{{-He~lM84m-F#IgJTp631ojVIEH7qaiWc;-U%BHP|;Q@=u&zk6Z1b8RMxa-_s@fa z_lGa+s_1RljS9>}V>YlI)X{Hs1Ecr7h7pIXALp*&-5Mo#>&H3eHiq~V zC#|>HvxyeTZKMSOf=6rhJLPcQBZlce_(E$4~oT*d*WdGG^#WbNjAnwc8T;PmU)^vm#-JPm)d)#swzA z=#?GSO|VRPEp2{>a1Ub$*=hG0JgsTk%R_SWiix$Xq|M`7te(q^U=Igf*!1w#f#|s< z`WwqfrKre_3YtaXHgX)UM__3{GQar2corVuZCd#t+pA+Abfr0h z=WaslIL$QCqRe}I^T!=W1g*-M?aGR)J>_j^aZP?4@8^(qk@4!8N3?qKeqSbTKX2xXXYQ0XHs3XbU@g)@Z(p`>i z88&mns&IA4gWp!-G7x|3J@KFJ&DEtP7Pc@dn4t7_xUiX58H7tV`irQ~KSXwFA+NYi~ zu?5Rr&8gGox6?^sqD(Ejy4z)3{BXR=1I~CJ{p)rSvIjJ@HSFRwDtUti(oM^)Qqzn? z%ZYC)oSPorA2ff~q7vg7vm7}=Z?vrU!7_11C%7aYwY1x)RyH_V^V?C?A#>EnpDM^Q zu6YIz)%|oKk!1ygbzRnBj~5vGrJKKO*A9}Jq{1i_yobSTVACK$)FL;*nO}G@T4iCr zZ5@@Hpj7+bTd9msP*r2G{)&1R7J1m;41VWoU*nya$BLnEN(vu`c4@q5k%?%Jjk+5z z5qerhd=%9v*BCTi_?ee?vFLGY`PN5>!IP|F^SvP?{*flG=QbS8HXNg-rQZBEPWu|> zHWUaBj6|&{G&=0u#%dLuK6s7emaZNbewZHa4t2LwJ1-EMS+GwoE4TIW ze#W;KGkD3OG&Ud7Z&Jv6kDR!ER?_7^nGGFopAzmO=kykA zn=W(I>h31oymBwu%!zUdR+}5r^+;Dwx6TAJ$xq;5D6lm*dEmS=&6eQ>`!PcU!*oBc zpWkOceU?4@bF;XYzZaEZp#pn1a+!v~;0*x`M{Z%ib}fv`O^Nv9Kyd-4*ko==WhOhM zo2_fEwSz6ay%`(@uSVbP*_4_NqxqW}ux|ZzZ-W81#s7NIg#%kQI212 z<;IPuI&h)|yT_r8v>p*bnDuDY^$NYnYZ$fQTM#Nc^z!1z^vA@e1ijQh+(L|ac!2vJl z6Wb!m`yl(A zDs@ks1j+qQzzh8LTep%7^DdM|T}57;uW;qyJ(gp!&pZBLaHA+T?@}eUQ-T%Y?XR+Q{(ua_%5%&&De>}gCeM;gM zeZ;-A#J|wWm!K5qNVQ3h)!6To7wd!?ERSdNE|)kXZn@$rLQ@d2|Jy$ zJob#49ZQOdf3&suIUy`3&A{0Aj&j2G4pcM$7nUmxTnvYo&QL=i~Vp@QBLbP=k~k z!447znq(TKZ8cqDUqc+Ozp_ga3m;wS-`2L3etVQ6#r$nnbS7Nfr1o80p;N;v3@Ydq zUHDnCemB!mMI*STojxwx=d<5e-hmUT4387~N|_ysSeK(khQ^!Tptv0G^pkpbOj(u4 zrzy1-SvCs6c4KNz=mq@hu*l9+exd$9(_|S z22bAkJr8k_5_TjeA2sDA;dZP`N7|)(%Cu;9G?CZpgrx8;L=pGbIXulS1LvYyMUDC& z?yPfOw-R3YYK?VrJ{>vT(K`Fk$hzrLzq=y-+1ppO5hLHDOUA|)muZe!TSQQiAr*Kj zy#Z8?)2U($Kr&CthpN|K)Tm``HVFA)F%^PM*3D#V+v+m9Ip|Fcg8y3%f-B@U6RYTV z4a$JZc1I3DSVM{U%#92VH!mUBpqcj7a;5gK#UDL_d8-c&U*Kk>LLe1@h(>MT5(H;J zU;&2>Qw!kxE5T@luU^i9j-U5CqMsP^KCld{NaRnOB_`zazcWkEpx3U&d9A6*l5E1;qIrZ-5bh*vS{ZH}@ly~uV_vRTMC>^3zIDf~Tb=dB9U`i$b zSR~x`0GGH@M}aDg)2&N57k3}*^P4s>rJ zu6;KHOm&~!;)QF^V3X$99yELJBetUBG_|rnkh+^T!A`VqvCll5SU%e5QTDZ*-el?L z=1g^?ac-0WaC9zOX(OGs2Gc>kdE6JdmGxgdqK+=*^8p=WJ3m( z5!yCuX*3SUh=A&-ki6{3&YlzxG8_d>G9nyd=xs|`^&Elv0L!_qjT`X4IJCa$U-b|D z^0|;eA<-zf1QwhPp`^e8G71e0ffeW-bfIHIAv>=+W=JBDkTn8-|4Bg(8h|)`;4cDv zG9V2A&Lpbtz{LXcCi%yU<#!e%LdDs^fdqU(AmQ-6K8vh6A%$(rv7wWSsk_CAISWp6@QHD>LC1&K?DrA+BkXvF>Yu$mAvdA zCmaGAP!06}JxHLTD6@VNmB2?+qG5ndWJM39C{0XUBb z3XcPwOF;n#oSX4(2hLSepgz0fzJyvhtliV=z-A|R%qfCmEt)x`nEf3*S1f__)iVAF>NiVp>x6k#S5tH5K9ALLGfxpL*?+RLV#kZ|J6Z*1PEvV0%e5y#DO}iLitS` z6tC(J)DsqZKtdvd$c9*hVh~ox1M2QqV8j6yS8GD@{U-qk{M7^+a16bHr~;Hj)gd+b zJzT$IFl6AI>W>z%QlJp{!W<$W(u4mZ70O&yCeRX8>p!hPL?MxktMAqtAb_>ycYqY( zmkvOB@ZVuo)<7>Ho%kcH>I0+`zr(7otmy=#4}S)TuGMGI2BaIm1H?HVi34K@XoYlu zN+5uRZdEUUWy1dve(S{V7!nE?cgEEiO8*g7$NJZMsLX!?q!-YrLh(Oe<^PEglz^pK zT?@f_41~C9x|HE4Xcr8Om@0hr(`1lRgM*P>&C`T~E*uOpNHK!d8og%rh;T6A zt8b0q=v6}mt=#CfK5T(I;jg!LK(4=1_VCp&g+MDLpe4WZeyxE(kgF~IG8@1Yu0A=y z0fWC%kih}m{dxd42q+Bnzz8e@SP53Mo#9|;R#R?Z=KM;z107mRL6!+zfvy(tfUlmI z17;xuhh1aM6Ar$RTYdkJS;VgOxtg}KZjEEe)Z$i&U<8H?x7PWpEe7iqRP~>gAFA@(j;Vp!^Q-Vb z@1bUXzgI&22@U?^nu<_X0)-HEHsEXmvYATg|5*&!^nZ$B{%0|yB(UqCAQUSo{6Q^5 z-5NFjkO;B#S0*IXzcL|d{buHu(f;L~i_mhmhhPnueUNkjf>QSI21a(3pEchUu*&~a zSsA#XLg4KHKv|xy9u!w+TeuHKQd$x#4tJtZJiTNP2)F;dlJp=uiU6f2d)Ry10gu){ zYkAr`z-?{pTtF{tEr@{Lq2`@E+*KgY-vcT#Xe1ho#Db5s@mOh$l(`55P1eZrC6gQ& z!Ez|g2-elLA23Tepym$n|Kvds5BkB~*YZ#}G+2_>=OH0ITUQPVmYel?XlVAXD~E=Z ze0?7HX86DJz!QLT>+>+sjD~*yp%011fMt1I9ukcNGh}_azi9?mYkhm*gapDk|5X=- zmICH;T^<$(kmY~p0a}0)()v6cG{d0ZA2dtm~I2 zrw@!2xa41#2dvm%G6k;Kf3=6jg8TUObs0*+7X^6*Fi zo7U%1kriI*Z+`IvfXmm_m4ZG2U7v>m_prU^S41j*?>Jsn(Ca%wu29V*u^H50CU;Uzxn7?QS@Cv`4KCpv9u9rOuh5kz} zC=}-Jd5*#n*3%3&J;?Rr2JI%-k2wl_VY*(#c|VbBkfJ%w!J>`Ef9{x1YWXFn2H so504*!vk7QSIw5Dy92OKpvEslskasU7T literal 0 HcmV?d00001 diff --git a/backend/data/evaluation/repos_expected_to_be_fair.json b/backend/data/evaluation/repos_expected_to_be_fair.json new file mode 100644 index 0000000..4e4e20e --- /dev/null +++ b/backend/data/evaluation/repos_expected_to_be_fair.json @@ -0,0 +1 @@ +{"oeg-upm/oeg-software-graph": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "zenodraft/zenodraft": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "comses-education/wolf-sheep": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "fair-software/howfairis": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "GrainLearning/grainLearning": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "online-behaviour/machine-learning": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}} \ No newline at end of file diff --git a/backend/data/evaluation/runtime_benchmark_box.pdf b/backend/data/evaluation/runtime_benchmark_box.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c238d72167133b213990872147d378161a4ec5f5 GIT binary patch literal 6309 zcmd@(dt6N0{wR+|QDCThb*=nTS!3CPo&fns2{4B;rma}c)V} zD(lSoOwnW*T+}a#vr@OVq?3epf4Id1Zc^XKTl;v<|ZXTH`S((&{ zJ)6g?4r~uy7lPd0nS3^|#1cj;qbZ%%U)-{g5yZKvtS z@+p_6G&hO5AB{NVls4X9iszu-EsJLucr;H*nbgRCTkb!d%N@*rXS!xjcG$?`RU37R ze5+@3Eq^N(Rd`xnIQZb_coUJXofb23&Ll$UTNq;#Jo505^||@UbH?ONtYXGK`pK;A zQsD+s*we8mEN*XG;A6|H{VC_p?%a?+h5C}BL{Y(=>jOdacCK<-@h@M;v;laA`Yl zRvow3U$tOz#E$M17?#*6kDzfP;wA2p+h(H>mv znlygTmUzFLPahZMul5Zr&0Njxh`;N;E2aEy;p|Xmy47e6mx9v#EsN$K$@_SwnzC`p zdalg_am$I6)e(|vc12PaFQ#`DW!ad^9k@1kVD*^w``hsYwq0Ac6?--ng|7_`wDrG_ zrESlaT-n}SomAR0_D$iDoPv4*wZLPlyC0t$xgyZ)0O}*Mynbf!O;**dg9LlM#Z*uSOZszhSyXSC&3spMm%bvd%MJZt5s_d)kB+MjN}I|r+F z>pB!(7l(0Ji&uQ;c^tc@r)~7^zhBT?ZrnZRlu9Yf6c`0It)x~xcns5<^4nAct6#2Zbs>T=WOS;PxaM0 zNgPeB!|54WJA#JmVrxyT?7O#`%2*$aR_wl8GAALu>h7lM8`b8G z?6l8Ypf~5}*_RxvJ=wBzLrs!SY0~os@|Szyr}H0gGu5A|y?C4sckiWB3&P8qxpj7_ z4dTDuA_eAasc8{T4Th_dk*9--46QPmOnrRLb&U>c6_p!Sf56CR(`w7i+gkG70p-76 z`}5f2+p#QX>sNo~FMs(yI$(QPQr+}7L2>hv9CjSYj(wi)l+fg5`{OlZ%3j>o+Ol|Q z`CsD1_fxL=c7CXO*FFBQZcT4UZhuSTFs=4Uqa4Qf4d|MBapVPLnC2&cM=c8Fp0|0% z`nR7_N8roC+H{_;9X$L*yqn43ZRT?gf`)qX8)l_VJ=-L;u4)*p8#8g`y&L|yS$`}M zZuOWwsbQ~P{3o3#^DX)Db^L$48_xfGg55WyRIfD3WY4Dm7gDB_P%ogvT$cV8rn$oW zt%FbIrHALKBh197CO)ta8LyQ+NlY(u9`cK})w<9{iCtOf@Q`Vk>gyxmHzyUWe~G_x#cT(_1gH@xv#H zD9tOr`!2QaZCzBgdP>>8{lhQ!oH%;;(UtLO#z6)X@?0+973J+?v#vis5*fTAOL!qI zZf&mRZR^WHgME!2usb%s%Zs-z8j^fAJ8L6le87euRye-=bz183w}$>fNL|M`kC>DV zv$n2^O}yu>%~@&{>toVIx4`t5tF6(x^ktTRG2oPf9NH;h@L6r-%%O>NFoKn|nIZz;pD1^mhXf z%xF)1^6Bu);&b)&CgVF+Rer-cutDnM9Gt+u?;L9%52=RZu##HKuCqSAWQ~ePS{wyLd_GLns@ts9G zw60~`KB66YP`5W_QbT)16E-I5WWC_#h-aaP?Xw-L4u+JQ>5raqi#H)kXG%71b6jT6 zLfYJdw)3`#1IE(zHZzjkO_P4l*?V2nQ7`3?TPjmJ>hFEDH2J#OsrlG5nYCzMtrvU2 zjb}jzU$tf$hc2?2IGT2FT-O_>#%Lem!BvUS0AJ ztH`MrPi5X}ZY)i1`8(xsTU$@B#=NlLqT((JG#xgc6ze|V7bk8%VcNC<3AY$yo!jia&Fka__{{?1s-c#?>yLIl zn}55d|CG!4C$4w1a^%7J8l|=$(3t}U=r6iDC}Njq=CRgt$c!B`-z zV>q1-du+(Uizk-~$sXJWMPYsgzt2yh6dDw67C0J!)?3-PyNW?44G#`q4lJLiXfDFY zNC9a3VV@t2FxUt!K_HbNG>I5UDPIDfQav?8DhS2f<$|8<3K0LB;aL_zJ4)Dbqr!(Q1EXG|RjEa^c zD1{cJdISd|a8wiW9mP=#1<;3na)BrSK6vrsNR1GT&cM~xf%-vpf)OT){$Bv)f2BhQ z9%TTZgOeA+rn3Vit%i78?kFf@e6w zbvgiW6lH<^kS2%C0i$9Lj-)bl76pz_2&fbUCPpXs59Le~BoEY#0g(iYxL)G92fb+x!a(IN z)d*ZhQ3*my#b9AjZBl#~2VaQ_bs~ca1Qgu$nF@FGN+ zPI2Xx2h6a*z2yB0nRs!<-OrcHBlT9u4fs%TS@dOHz05me>N)=NbrQ|l5oS0(55Zt! zqdDUL77g@RU(o2^K|=vGwM?l>l%ULf5l^ice3?`)y-3oJ`iLU7U!7C+ z)5DdvBZijW^wQ|K+A{hPO-tfWN%2lU(#n{Ah((=Lc(pv4H`jWTRD0~L@Q#r6Yu;;9 zS#Bo{r}I2Tshe46vc)a-J$rO>48pt0nmRpCMPU(_w(0R>y3ckN+4q_>J#nS>n%;Xd zl)ubH=Nny$x_cE}%Ks6^}F69Xq3Z&%!C-?~y r1RypLny*9x!*?YMY>GRlgi03B3KwoF*9@ZcBA|YjHg}E literal 0 HcmV?d00001 diff --git a/backend/data/evaluation/runtime_benchmark_results.json b/backend/data/evaluation/runtime_benchmark_results.json new file mode 100644 index 0000000..8af8523 --- /dev/null +++ b/backend/data/evaluation/runtime_benchmark_results.json @@ -0,0 +1 @@ +{"kohya_ss": [333, 6.149273872, "trending"], "qdrant": [479, 7.587065679, "trending"], "data-engineering-zoomcamp": [8, 3.2705682570000003, "trending"], "weekly": [3328, 3.238594682, "trending"], "30-Days-Of-Python": [152, 3.264650774, "trending"], "ML-For-Beginners": [17, 3.7624553570000003, "trending"], "whisper.cpp": [618, 4.154499037, "trending"], "Ryujinx": [830, 3.308689803, "trending"], "crewAI": [253, 4.042483737, "trending"], "ControlNet-v1-1-nightly": [54, 3.259032406, "trending"], "AgentGPT": [338, 6.600420346000001, "trending"], "ollama-webui": [128, 3.71774956, "trending"], "bevy": [2099, 3.584146515, "trending"], "mrsk": [124, 3.9980926950000004, "trending"], "awesome-chatgpt-prompts-zh": [39, 3.318572526, "trending"], "bun": [3267, 13.669202926, "trending"], "chatbot-ui": [99, 3.405853775, "trending"], "one-api": [522, 4.278319043000001, "trending"], "nas-tools": [2, 3.331086596, "trending"], "alpaca-lora": [373, 3.505891541, "trending"], "coding-interview-university": [58, 3.3429647090000003, "trending"], "video-retalking": [171, 3.348955344, "trending"], "gpt-crawler": [99, 3.4358465450000004, "trending"], "AFFiNE": [463, 13.711496521, "trending"], "sing-box": [320, 11.036155195000001, "trending"], "DeepSpeed": [1740, 17.743258865, "trending"], "awesome-chatgpt-prompts": [285, 3.3288980560000003, "trending"], "DeepFaceLive": [38, 4.033112351000001, "trending"], "ColossalAI": [477, 4.176778105, "trending"], "Microsoft-Activation-Scripts": [20, 3.4195533240000002, "trending"], "appwrite": [1112, 10.030643179, "trending"], "svelte": [1035, 7.293154702000001, "trending"], "LibreChat": [125, 4.321489408000001, "trending"], "gitness": [212, 4.401358639000001, "trending"], "everyone-can-use-english": [53, 3.5295240640000003, "trending"], "public-apis": [252, 3.3280595070000003, "trending"], "developer": [84, 3.452045103, "trending"], "organicmaps": [2386, 7.422461721, "trending"], "ui": [1081, 3.759052338, "trending"], "Fooocus": [221, 3.6211491400000004, "trending"], "ChatPaper": [83, 3.348232287, "trending"], "guidance": [238, 4.414925541000001, "trending"], "wenda": [52, 3.5287527470000004, "trending"], "atproto": [608, 10.717386231, "trending"], "ChatALL": [296, 6.319981084, "trending"], "privateGPT": [210, 3.623402714, "trending"], "ijepa": [31, 3.243062199, "trending"], "localpilot": [25, 3.3025153730000003, "trending"], "maybe": [14, 3.3863975600000003, "trending"], "so-vits-svc": [42, 3.404545959, "trending"], "tinygrad": [193, 4.284437921, "trending"], "hysteria": [212, 4.783197225, "trending"], "ChatGPT-Next-Web": [301, 4.861869547, "trending"], "civitai": [119, 3.9665338130000003, "trending"], "DB-GPT": [203, 4.4344858050000004, "trending"], "lemmy": [1230, 21.783576738, "trending"], "plane": [648, 8.79040771, "trending"], "mm-cot": [47, 3.390402909, "trending"], "commerce": [38, 3.561755461, "trending"], "stable-diffusion": [574, 3.251116226, "trending"], "generative-ai-for-beginners": [81, 4.257097988, "trending"], "wipeout-rewrite": [43, 3.1935651480000002, "trending"], "devpod": [170, 5.337686221, "trending"], "localsend": [361, 3.506813127, "trending"], "FinGPT": [67, 3.394322772, "trending"], "DevOpsGPT": [24, 3.4127574870000004, "trending"], "TTS": [247, 7.123276949, "trending"], "lobe-chat": [831, 15.164889098000002, "trending"], "v4-core": [134, 4.676984656, "trending"], "CasaOS": [455, 5.1759135810000005, "trending"], "opentofu": [272, 4.863282658, "trending"], "Moore-AnimateAnyone": [68, 3.2652178640000002, "trending"], "KernelSU": [53, 5.16543623, "trending"], "MemGPT": [246, 3.709168818, "trending"], "NvChad": [5, 3.8024615760000002, "trending"], "hello-algo": [35, 3.397500945, "trending"], "memos": [145, 5.286482096, "trending"], "taxonomy": [197, 3.492674458, "trending"], "dioxus": [254, 3.66564727, "trending"], "quivr": [1110, 22.610293744, "trending"], "langchain": [2389, 22.430422025000002, "trending"], "ChatGLM-Efficient-Tuning": [13, 4.369862632, "trending"], "vanna": [111, 4.263125835, "trending"], "generative-models": [254, 3.5861268120000003, "trending"], "novu": [855, 13.330679093, "trending"], "Retrieval-based-Voice-Conversion-WebUI": [796, 3.616643342, "trending"], "ChatDev": [45, 3.459646368, "trending"], "v2rayN": [189, 7.163501849, "trending"], "project-based-learning": [115, 3.4175709980000004, "trending"], "ChatGLM-6B": [582, 3.406032895, "trending"], "text-generation-webui": [256, 4.239049888, "trending"], "vscode-chatgpt": [12, 3.338669605, "trending"], "CodeEdit": [230, 3.8134301660000003, "trending"], "free-programming-books": [73, 3.8630769860000003, "trending"], "tabby": [242, 5.4329793330000005, "trending"], "lx-music-desktop": [683, 6.265574703, "trending"], "spotube": [352, 3.7283389220000003, "trending"], "chathub": [297, 3.402538496, "trending"], "PokemonRedExperiments": [70, 3.194502882, "trending"], "langchain-ChatGLM": [389, 3.7993353180000002, "trending"], "ali-dbhub": [88, 3.3302991690000003, "trending"], "so-vits-svc-fork": [346, 7.152177324, "trending"], "system-design": [4, 3.5244565980000004, "trending"], "Auto-GPT": [246, 3.7290719190000003, "trending"], "neuralangelo": [85, 3.6044507670000003, "trending"], "dify": [234, 5.708580801, "trending"], "chatgpt-web-midjourney-proxy": [75, 4.126363961, "trending"], "ToolBench": [78, 3.298891676, "trending"], "llama.cpp": [2430, 51.244774698, "trending"], "Open-Assistant": [454, 7.1236037450000005, "trending"], "servo": [3389, 5.640805547, "trending"], "PhotoMaker": [103, 3.2825390740000002, "trending"], "comprehensive-rust": [149, 3.4163447390000004, "trending"], "Stirling-PDF": [249, 5.217433957, "trending"], "WizardLM": [165, 3.363602213, "trending"], "Baichuan-7B": [86, 3.3965621620000004, "trending"], "anything-llm": [114, 3.405243854, "trending"], "tldraw": [889, 11.762952339, "trending"], "Reactive-Resume": [489, 4.978993652000001, "trending"], "stanford_alpaca": [189, 3.278172892, "trending"], "invidious": [488, 3.500447095, "trending"], "Chat2DB": [379, 4.497567923, "trending"], "facechain": [127, 4.084653392, "trending"], "nushell": [1412, 5.373783386, "trending"], "manifesto": [54, 4.156231992, "trending"], "gpt4free-ts": [51, 3.512207897, "trending"], "tech-interview-handbook": [40, 3.386020985, "trending"], "backstage": [1353, 18.510549472, "trending"], "insanely-fast-whisper": [60, 3.293156305, "trending"], "gpt-pilot": [320, 4.097165207000001, "trending"], "serenity": [787, 3.382767723, "trending"], "picoGPT": [11, 3.324976583, "trending"], "nextui": [1560, 28.100550249, "trending"], "ai-town": [58, 3.48630705, "trending"], "screenshot-to-code": [107, 3.245872576, "trending"], "yt-dlp": [1539, 5.9729979900000005, "trending"], "gpt-engineer": [74, 3.9629081330000004, "trending"], "narrator": [31, 3.214329261, "trending"], "seamless_communication": [193, 3.6815531200000002, "trending"], "v2rayNG": [199, 7.229107557000001, "trending"], "Bert-VITS2": [19, 3.564941648, "trending"], "rails": [1375, 7.183077685000001, "trending"], "youtube-dl": [4584, 11.749591032000001, "trending"], "biome": [214, 4.678095196, "trending"], "javascript": [170, 3.39229174, "trending"], "lowcode-engine": [587, 4.2154225180000005, "trending"], "reflex": [511, 6.514794538, "trending"], "clash-verge": [294, 4.499282216, "trending"], "AI-For-Beginners": [86, 3.674364713, "trending"], "BetterChatGPT": [223, 3.464504406, "trending"], "chatgpt-web": [51, 4.1699023760000005, "trending"], "ComfyUI": [1503, 3.3521181820000003, "trending"], "pr-agent": [114, 4.795440175, "trending"], "eShop": [86, 3.388562235, "trending"], "openai-python": [161, 5.637362459, "trending"], "The-Art-of-Linear-Algebra": [2, 3.2317930030000004, "trending"], "core-js": [283, 7.811271961, "trending"], "spacedrive": [143, 4.514998257, "trending"], "system-design-primer": [423, 3.3755888420000004, "trending"], "dspy": [311, 5.122560144, "trending"], "system-design-101": [39, 3.3443792090000004, "trending"], "npc_gzip": [14, 3.3943164880000003, "trending"], "whatsapp-web.js": [227, 5.405059603000001, "trending"], "JavaScript": [87, 3.3653631720000003, "trending"], "Fay": [72, 3.4496883630000004, "trending"], "dub": [192, 4.8403561680000005, "trending"], "llama": [356, 3.5380059700000004, "trending"], "Chinese-LLaMA-Alpaca-2": [30, 3.415995505, "trending"], "platforms": [40, 3.5837775140000003, "trending"], "audiocraft": [279, 3.41720793, "trending"], "cal.com": [1835, 24.603552936, "trending"], "Xray-core": [453, 5.077475564, "trending"], "incubator-answer": [162, 3.7151237000000004, "trending"], "InvokeAI": [756, 9.3404167, "trending"], "flash-attention": [476, 6.614749672, "trending"], "hyprdots": [92, 3.367827087, "trending"], "mastodon": [4329, 9.914710110000001, "trending"], "Feishu-OpenAI": [117, 3.65414437, "trending"], "ChuanhuChatGPT": [134, 3.7315887720000003, "trending"], "Rope": [19, 3.35768783, "trending"], "QAnything": [171, 3.449559101, "trending"], "dalai": [344, 3.454272891, "trending"], "ollama": [931, 6.392430119, "trending"], "expo": [1157, 12.477746264, "trending"], "danswer": [225, 3.3080155380000003, "trending"], "ml-stable-diffusion": [167, 3.413253445, "trending"], "devops-exercises": [40, 3.361005038, "trending"], "Mr.-Ranedeer-AI-Tutor": [20, 3.33035745, "trending"], "graphcast": [28, 3.396939129, "trending"], "documenso": [220, 5.8719194230000005, "trending"], "nekoray": [375, 5.013031767, "trending"], "fairseq": [1316, 4.664430126, "trending"], "namada": [1068, 15.838671829, "trending"], "v2ray-core": [178, 7.333839822000001, "trending"], "MetaGPT": [273, 4.683900832, "trending"], "godot": [12290, 4.978857452000001, "trending"], "LLaVA": [761, 3.4754125580000004, "trending"], "clone-voice": [13, 3.363918419, "trending"], "996.ICU": [5, 3.269847513, "trending"], "langchain-tutorials": [11, 3.269953783, "trending"], "hackingtool": [50, 3.3820020640000004, "trending"], "stable-diffusion-webui": [2034, 3.6385031000000003, "trending"], "h2ogpt": [733, 12.870253486000001, "trending"], "rinha-de-backend-2023-q3": [1, 3.2564498700000004, "trending"], "AppFlowy": [798, 6.688640175000001, "trending"], "LocalAI": [332, 5.121105841, "trending"], "ControlNet": [407, 3.304860369, "trending"], "gaussian-splatting": [297, 3.372469736, "trending"], "AnimatedDrawings": [12, 3.3623673550000004, "trending"], "awesome-english-ebooks": [1, 3.2081148510000004, "trending"], "TinyLlama": [36, 3.260123215, "trending"], "FastGPT": [159, 4.3276700660000005, "trending"], "Flowise": [624, 9.437515770000001, "trending"], "chatgpt-on-wechat": [440, 4.3161181090000005, "trending"], "oeg-software-graph": [3, 3.3478682500000003, "expected"], "zenodraft": [38, 3.6558369780000004, "expected"], "wolf-sheep": [4, 3.4884464200000003, "expected"], "howfairis": [101, 3.463828781, "expected"], "grainLearning": [27, 3.457117439, "expected"], "machine-learning": [2, 3.530429262, "expected"]} \ No newline at end of file diff --git a/backend/data/evaluation/runtime_benchmark_scatter.pdf b/backend/data/evaluation/runtime_benchmark_scatter.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f3b9b750c10ae5c3624e7adddd7b6b76d65fa5d2 GIT binary patch literal 22012 zcmeHvby!v3(y){?NE^gKN)S24IdCZHZjc7)kQ5GxAWDZcNP`Ljg0z&NK1E! zNU8AcgZ}O{c)#oWJ>Q@A@GREavscWTnZ0J#tUa4eRZ{v2l!upyEq@G@S49K`K|qeC zH;9CUKw!<=P8J}rgbCWj&d~}4RyDD*Z~?)93hE#c5h4o*b4*44pBhLzI-o)D;|gFM z71J9QW@r%K_e&{vw2TJY1Z@H0{SHtyL8C349Y6@oB@tM|+Qi((!3u=@ao5?=Ov3^V z(g%i>lm<|-a7Tl{a`pfU;y-eUAGsXJ@F#Sje@*~{HwJrW3jn+C=!4ZQTpV4U&4Bq} z@K*g^49YHOXA2X1BKOpuRJmBzwwnPPpD%9_ z&AbV!1|CErjvI$)kA#`jZRJV3 z`pCmWm|-T?a`(2?d4dAbeaoKfq(Mzh4idNGg8#&u8$@MQ~HCS zfMM4T=@x>(SLLIqlI;|&V!Bvcjdxeb%T7Zjq;wCR}7ybTaXE7tBOS2N?m5p zi(79mUWP@w?hBf>h7nafDuDT$%{dq1@B>~yUQj+D^RUN?|gG&`5>LmW2M}J>Fst@Q>swsvW^vRa;3nRG5G>(SE9O}X!7dI zSCY7HoD&;Ry3bwx){Crmz*1g#HmNQ;RfR`kxq$jnbjC~d4J))<|A5P*;=nT($a5_f z7V8F3*^)usRvwEsJK-;Nc2qlN+Xbt-s+x()Pr^<2=S zrrs`XGp;XN^}-~0hs!;=ktF^Mb%)e(U+wYo@NG^P6Oop`vGgeejaJ_|_&oQns0795 z^(}}3r|ximt9t3ls)!}U^`rT%yrO$1N!%Z&1mSmUG}-l;LKSz49!1G5=dYrYW}hg& z+D>tu{d9hjbDxNkM66oqt_7E`&7GLy^Hm_7`Uo0|;NJU+Pw_UGclr7$D-yD}%&8+cO?8yS(EcP_wuEq=$mG4f0cV=7 zQPPylOREGSMUsL_%CwBS(u18VuW8!pzupIJfG!aZs`GW;UQa{4M$qXPd83NOt1K>R z3&c7sIzH>xt0^{B3ZsB$KEM6srhh$*c#oJk6H2Z+Ii^sWy|!@v8@s5xQXXU7{H?%S z5AogJLVMym&TxwRJgpk*8APu;CH2{o`SE@<$?e-w&(8%>T-VwmcuJJp{Lp!f|J{>1 zh4-9?er?WbnoY9L-c3|TY)R;EbABFpYvPHTctgI{^sd&V)BIlTsJOYsyDO=}Xn)DV zm=9}p(k@)r_GJyGSgFj#d_w#O7-iF_0cVFwH>4vGGyxshf_nLedzA&Y@Oue=ykJ zfLen7vYTR-Ui?MECy^qjbO>f-FJ3fD!fvco*QAZg$PBl!J!nZuzJu%)LfixX-I_>z zcD*~(gaZ`V#B_&v_bUgb*@$mDgCmP)W7U&!!BaFI&HeOpF(A0+^UKL89vtI0yyWBp zN6iWX2HRWSl8Omd*-zYTxTN0AHvTbCwOILEm?DX`#`)H<(s%UY@&{!g)$^8qJ~%0b zDeE|r`(<;kiusT47!M`|`rh6R3>V0km|yd?g0Ti6sTtqg2a#kzO3mBwrzE0ZP78NG zIbanG*3u`}4fdilKI0v`y(8agbDsI}+BF`hE7T8KG!xIAioK7wM;@|T4o*T}NDsP1 zH^sSWGuo=aEx}F@?mB-yF(q$)tYow5{Iv!-irQY$GET}3bKgDjxaBjI38xw*HKNtd zjv>LyjM}kxg6%8dn%!Jr^^hjtkB9^o-6~E=$ZaZlCM|-R52+oV{d`{(tt75W%))0* zg_TfK#X*P98W`xDC+9q+osL&(Yn^D@%=WF_RKIy>Q&1tnn&oquPQH$h#;N>rT?W3d zmR4upyD`7kjvMH{8!}=OqIU5NQ2@n$BAHu`J;ZPzO`_0JGOCl1z+x^*9mVwu;g>53 zU0^rD&Vdo9dh-)5>!fldyyTpXAb@@5nO0N4tx;kM-Xp;>a$8n2#$z{EDdBef>%-szDy!5sdSI&j-tJ{_QRDewL5tIfy5+*_&72?pm16 z%V{hyY50a2eSzdn!6ztu%_a3ECg0~hk~;L?t3>5v;dfF`8o> z8eQ0pOOc_s0YSV7u?RjDxvI>-Q%E(nt>(nZ~1INdpmndy`qJ|>tPAB0PV(Hti) z{DWw1=%}}hN4O$OV4~s{Wa9%JA~*0Og{W_j5rt!|z-w>aRroYmYg+j zfxR)Cbn7ybgQ|8nlN?vPd(=5~VR^IiEz6zqJ8Z(N!|se&SY6M(sZ7sJM`-2Tm&CCc zemR%)pf!nA&DhA#Let8)3p+Slu_N1UI}l}m=}Xk7F}2G!V6Q~AxKnYv^m>^O&!|MG z*>k?$n-YKbcGDD_?a(PGpBY9ds>Ve!i1P&>JJ>oq*ZdkD?a(>6SfZQ;+$QYC=7%q| zo@H`b)*Z8}#k6ud9E1nEcui4z zwmGm7LR=lBmL-?XlIWnw*{ZjiAk6P?*JEoY# zTsV>ldq|z$G1;^9u>Zof)5iW~NHeZ|%6A+{}GjdFGc=iOIbag@x2> zLOfZ|A-<%|WTbI|=Lm?RJ~`Y$zt9hTCHlCHgxI%k$|Qp<$^cqxt)(crJ(Ddab*4yGSsJw3%PLHO;2G`~ppSko}2DK-{~H@Q9c zZ6psof*_`JP)ET|aP&){B8R7cO4%4dA`|P-SYi39I=umJ*NDfl>Q9T{aRqj>p%5}wvc$6}gvV*U=DOWDOl&)YE{&s1+@M!X!ys#VXm}c>s{+Qoe z=H~kvF%u(m;Jx!arJt3#cp?JQSvX2Fq;~JR5c$eB53g&G(u93-+Y^=Is8SNpFs2Kv z*&3`hpO!+N_waRok5_88b@8k*n^AY$gw^8XPW4XB_!YmEj@Q8|0sfB#^jEj+N2BF@ zL#qQt=JJb63^L9!ZW3IYYkk*B#)o@J{GAf5MxL8lrGNn_^i`=>v`QkSm{%stXm+!N zvl$0g!0k~nLJH_-w(dvPmmu1kjyUyGQk+1d1^wv_+t z0+~q7DYl%?XVOB%l>**E*dM>rqiBR>FTu}hfgf-2uOZY^&H^|_DZ-yWrow+=w+^{!b~@VzX4l)M}#%6rW`$~nT)!Vhh$ zKNGC$&d$yGow`#7RuW}yc{6xsXONt-vtFKr$eE6-{ws~W0j3C0n0bJwv}YT>tM3W@5J8QQX6 zDUN2~x`}^^c@V3sciw`i!1zW#Q%ci(_;@%My$@#;!6eIqnBe7@&pP%Tf&-kwa4VZj z43+vUSxjLG=<)a3*Y5jwlvg-uiJAGUKTD&Mx+eZr3po{% zFO_2NHt39WWDIpa71b7~fGyMh<&8qF_jPQ$de;k~v4a(@`#N}6-zM^{kduGdzO--G z5}p%|-mKu9gt!vD9IIe~G-=YS#wm&T*~a+rAPTZ*@As}WX1)v0G;g|`6tOd({M@X| znlMy@_cpzsr!j`yrk-ZT9Vb)PRb<`p>^LX5 zIbycvcKX=?Dfg`wk2jh$il-bkK0b9WUQE+y!Fy<4>iW)lC3!6vlE)vhLZ>jOZt%V$ zFb(9;tQczpQCfAR=xFcjSksf+LB6rUk;exwmDzrJ3%k~ghc~{9pjoul9`Bw=`Z&|w z${v+ruXz^te;xob=35{vcnqzP!OTZUm_o&-=m01#+|RH)x_Y z(FI;bR<}h1&d5G@4{F`hb&dRNEzf=%lv;l3bFqd?8qSOJw+6A$d$`vVu@&WJZ}Yt> z`~>Qtb8O?a=X<8};y|d3RA#xB3w>24wwZvEK|8TlDNBXTc|Rfsk`t$^q<%xANIItz zyRVN%!6@b>(wD=ePslI5N!UX0;*(kfb>j3$!$HRvvy_x&SMqy`1%tR)tIzKAq$)Fd zb5aW1i9fFDd>AN{b9=)#^i$(Q@WhwJ5wMGn$E`y&H^lrV&H1&I-u=R{IBJ=9n>*re zbz*bbu^X2e0W-3KuHHU$9**0hUP)qjrqJv~^9U*HYr{vxn*#ngUil;bh)bTs%vIT_)NQi=wxT?LX3(`d4gG@@1t_%a#DA9*Lxo# zEby@BWTmgekg(!Fjv71a8y5}_mJc^K&m>;XZ7GgS2xIF0*w(QUaLsOp$!MJE(bo&C zAUgfGb97-$Yad;s>`}xOk@vhlH}A16))1jJ7Qe)Nj71$jp?bbBvwf>AGAwvrLpOSE zcFW|F5L8Lp>j+gfg! ze(!O18mJ&npY&OXBjT%1^P8%Xhs`B59wq!0jDh?+jvLpRZEmN!d7Qp-HHyDRo}6z> zXR=lYB-KE*5{DukD zmT9jI{d`OP+oy)}A`jB?6J+b);G~*u>5;)UXBPXTgOCe0uZ1HE-_*`Z>yfZtVJ&Bi zXs#~eduuw7Hbg0(mZ4l}=$L6(RJ=d2FvxV>q@UDpYmsO)dXy1&+phTTFimA`mV>`(P1)uKXLM~~_#Ib$I9}N_JInQ!$ zBl%v(7bg{uNDC}{w#kL(azpvL8LwmOFC}bNb~N`!O&+y`^N)U5OYx|mt#NyXns+r$ z3PK7Vowp|&Sm~RWmzRFR!i1_AY1)3fd?r1zV5gzHOrSZxEoyW8%8gVtArg9lk*V-Y z&y~Z==$Fr_guW)gRoywQ91sCkTfsW5O&}!p6wh<;o{rg(C1|p`EBw~JpCg0D2D?kc z)q~QzTXStxtl9HRPx$T~9=26p^*U<0r*gga+ao@iq5Uq4+>}i<&{O<0>+1*kr5C|# z--+^H?Lb5X}jL31kXU4D+byqJ+7qe zZ*Ps!=)kvXArFzh&A*IRUZFj={kH8;Wcjt&eJ_E}^yRE+ zVIn;eZT$C#?jFtYu4H-=LMoQZUh$HM$_oXX-OJ_ad@pD_H}(Q@Z;d!r@O7yJEE6pl z9_Cr`Y=x^$&g`P_yRp~ALN&r6GuwCRCej!D9+Q9K&z*y;_)(Hy7P>T69Puu2T-0Ai zlx}PN&N56mT>ARG8cF)8H{?vjU9Clq6JDQo#w+{HSnmL|hLh;cEMH~xqY*-FZd^S` zxJOWTcsQfsy3uj}YpLDpmbviJ;tl1t+#8;=H->g)p3K?H)=j=R?-<&(9-W8EWvKIT zLujLGOU411r_#shyqEGH3ct0!b02k`nO?mS5yDQgJ@_s)`GMQajheNG1n$n|D_x~| z3>U74@4Wx`uDQ4Wg+TMox7rIRp3Ti2)`u7YeJ)g4lbmNgLw?$t0)mZq^mz1R;CKa#ME)^!PEtTua z&gIT-Wni8oV&Xe_^%zg@4_D<6pYO3>6XWYd!1y8mID(HIq8Ok);5GeG5r*;Z{^k@# zK>SGN^YB?Srq-jR`M>e6lk(PYYrxo1uN4?WCco zuAFm=J|{ zZzw{My#ESOaqgmg}ch5w}j6YWDn%?IS&sB3v>;r8tHxU ze{Eg%jbjL04K-p4phgj#Fj zKJ)~{em+L{;XUU={%0n@5q#Gms>NleD6<3SiRPu_i8uodL`8z*y6~vFuu8iXyS=$3 z6wKp8N|>W8OicotVaO8i&Wk#yMbFzQk#%p=7m^h6oiNu>%$A(N%icI^o_W4uwVWe7r4h_c~F8szPKED4<4n%k(=DEQ+QfwsP4^Z97 zeJ1kuTm2QuJF9SlRgdRRnTk|96nA_MQ@7vptCqDZp0ge)t$WY<(l3GK0Y~!NJk!q$ zQsm=I9S_fvoPgBNM=L*g1Ov`V{^Qdu9Hrol%SfV%4ntV^j!r^H9dXin# zvxY6IoAgvxpMnW8>A&WiSgKOh1O-UEGZuWj(qJX3{NX$gn$zjYh`-C7w5AlI*Fq%Mf-syG6Nn2i(ng((?Q#Wkh0tm}?dK z)%L#Ae^@M~-=>HWtki2PVn5WK=(?TSk-6t`gv1u4;Vu@(ev6Htq5ZL)(eiiC50olzE#%gNw_{baije?es1DxK~Br!6|_2!16>d^WBXRl=G>F_ zw&J~e#h2N(W#0$FojKO6TjvMbr=t!h!f_QM3$2+yeJFAm~3B9Jg zt4iJfvZh(a4}x4>f<_XzD%7v0r9ay^kDB07N1!dt zA9kws%~qREkPwHG7!# zdJ3q{-!Hav7aQncr6?y2<%oA-R@;AeAy0tEIF@aC+k5PBoaG%4bvX)%-<|vp>uzjh zUu?A%U2C!eqf~@8Z(^~a9AeXZceW@^TQ4TkQag2(n^5Kq|e zS~+Wfq4u=RWdkJO@h|z%+^?vb}6K=5C!Z#+Zh2!^=a4l zmLnl`%u}T->OIo&{X>-7Z@F+gH7X{Td@1eo4qU$E6Wf0op;C79T$Y=o3SNlMZ?Kr5 ziEYt2Ev)uZ0qPaCY?scFUUTV@pPk5!V9A6_Rv$td8!z>dt-f5dn!d(tZ8-ka-YiF2 zzOBEWR7oj@U|*8Zs)Bc&_V8YBZQalpa%E;siW3NSg8c=9|BXgaLB#+nwCi?&2;|z? zQPKLRPb-k`_wQB3Bq>L2GI#V-F(kPsv9J_dwvFnUgDgL2D7Mt^l#x?C+x@_J@j)oD zNno_oVE&B5!w9+~x42#%{0-#ZeO;q7?_VhnYSUWrjN+Nuq&A~E9KsB#O2S@FnuR?s4_~BT#_H`2?DNiI zM~35jj8fH$Y@iE-w1aA<>#1}8*K6I-jOCXoUBr{cc&M0rmbkLdB&+6^K6$LJ=9DY) zbTR+Yy}9cKS5BbU31%UT7xfn=hpJ$NBs0khDgeC6y2aweD4lo3a$~X(Cz(@K@ooj2 zwU5F_h6MVcXojueaiLFrsQ5>R=@q|Mpgak6&}$F!VDxG>GMY+&Ar*ergwHIZK?-_0 zbqSp5MDlY2p>`%0!66FLT9lFeG*?9KpN>G zU1xK+@wr2d#mwOib}SkiIgDqBGuqA+Qa1XwTRLT7ol>hiNBbv6yeHVUf5lTJHM#pZ zz@o80b0ErW6kGFDB1vN5XsDAc0EJ6f6cZbts6$K+^=IN1-iu~-;rn*sZpLu6u18GA z@KNHzMGdt*^&B2kal_yXz3CL2!s)h@x+S$AQQS|GTU9&JVirNnF|bkmL37z*;N8IN*)KXC3$J0V0XHNp9;g z;vQ2E-~86GyT?#O-t*NU^#lr@V2Gpsf~Qj9jT8+a z0^B1_F+w7u5f%1WV9^af1G{6qK(_9WM_h4Rnh7tC-H1j9U*2p?HBz*8%K1vi7#_J3 z$Gwq0&D8k*TQ<>h*Eq*ADQ_QnUDY#@o9ejcn{7@WLG=vrkC zu6gxvWPU+Bmf%*wDKaDTrh>dDIq_>tLcTjzQqIz01EKr^%rSazQNGh}A2aIB81R)$ z+q0Cs)Sg^tno0#5>;Oj(tNzU=aOVW;m+vq3a&TBBur3tyVo?Ut0@knP>EjJ|U2e!r zaUMEZlIBjVQWEbnwp8C>EAK>*{h1aJtz=13$T>-dRIKQ(5Aryt*hXnPPQd2`8yJS* z|IaJER5Y-;1QAH?KJ;br{yHlZD{ldPd4wc=ZH{zH$ob|k&yr2ay+C~%;?=R$V_vaz z3#BD^0gqbhQewxo#S-j1L5=1l+mIV8W)`zA`UOw7TzGAW$~C|FB+OOQmCH%7H}13A z_`aL&11}je3!?Y=F7$REI_;aza!98Xtds4AfK)g%WLnlh!<+aV;a+ihmz~n=cHSsJ z$Z)@jQzg!Zims6OwDxFh&r+_=%(yN17(3pkkR3kL9?g7~!PhO#;AnbqzxL>6pkQzL zxkCDl>060|Zf)01V`yBkLdQ6B7HoO+hRz90#KPNVv%dtsercRg?Wef0e|50;;1GwA z)AHa1Ql4PaLXq&lAZ3&wB@VDp_f$@2*<-;G>9vG$B6l6l-rK1+^y`59s;PVjMtnmS zeS<34#_=`^aIMU~E+HKdl(V?5-ddS6)(_OZUbfi8p+8z9T`m(BP%&7Nc|3S>?(rD` znu)cHS?BEa*ZUGqpRa$eHyuZ%g!puHG7$7EYmDt91v9FuIu0}2#`1^mUS7RsVxz0j zMOc|uZ7E7PrhyvC6TZqsB(QVBTu-t1s>}+=*`{yB#>^V2)(tk^;vTv+ec~ee?>Mr`{cZ7hfvD zn_!K)OAP0Fc7dT^0h#Q?MOF29tO2brP{Jz* z9&X|fPp~bISjhGC*L~u?IrF}s(`QK-)@WH;i-P0{zb1=Pe$iUA+%Fi%iW5>5^+`O7 zXoYcrEhF;sXrg+W@|C%>36B;%??jvuZkY%ld(+lM7babPCFjkJIDT>nmByR6n|aYb z52#K{SjlMf5@)q{dWa)lg`?7wYMY#lak@(VGqVcRYsf9I7O@RS1ZH^UeIri2n!FVD z`QY@kuSR?)(Cq}H4vK*M=O zAGQTEyol{s#Mw~SFL7POT03)%)v%PrtfZjIN!SDz(c(cf{~2Ny5+n2In!*Ye9@XC4 z{d?;U8*W}$#x0QJvw~lb7l4`k;SYiSjpr2+*+mLWPhun4ml(IN_qmYD{l`J*BU`qo zcqECC4vt}3zvwUtMdQ^(vne)Bl>!k~mdYwQ)R|mWK3o-RMG|;wroxA~r8*0tN_SMj z0A5;D`?#(>ujIKNp};3kS80?Fz%H>lapx?^+VeHWmD@XgX%guMoF{u<6ga0-F;-k| z$`EV`c1}5Gv-fQylh5$_m^vs^5^`Jf5ASfC=?|QlmOSI)|W%pz;`#ltGw~x zC>MWFCSwVkm%fTtsndhnT4bq2I=!mTc9B~?MzWDMhC%hIS+L+GSw&g17}*L&KbB;j zQp3a~O|DY)sWO|ESM|es@7g89ox;C{ES=W;s-Z{>uEo@4^T|H4weqORcF0wNi8 zAu@an-@~63T0&R3*B({G#7aq%8E-@90z>5Q=z=Pr3^rGWcZ&v073FmY^vaI53j}vX zg}#Wn65uJxJqv9XYWCg8`^d<+pWoe4ir)$zzD0g{XTaZrd8XyuhzU)b3C*ldu`Ba4 zPj`*Xrd-zdGoeQ!EiPAZuiTOD^S(a&s3_vYehjV6ZlZmEp+~Wz;IqtPdOH;tBlhch zRxbq@d+|GV3g1(xS7TELyoA4YnYz@kC0$1AhLpWwVCNX|O-H^jz`;QJK11hw zGdsH3iKZh{8J)RsOqP=i(asmH-F*768<7%P374~6sgo^9dUe~7zx0qq!?OG~N3V0n zKrX>XiIqZMAFA+SZM?2EI)$nx)4#V>Sp_&}g|9H9Xb5vn^|TvpsFO?yM=7sssA(j7 zoICuowf<4??jaVp%N>^!*m9EX_ZQ*^ha)ur+Yf?GnVlVYg6EUF+lQNlI4Yi=M}pW4 zVsC1fX>4hF+FTa|1y#XbygikeOoej-HYZqqaM)j7}T=LWHYE zC9IPKrnxxKyS*ro<0SvcN1LFM`a+GL+$p~sd-#l5aFFVPXlu18OdZFU^}UPDO@a_- z_K{pL>Jkf|Uk^|BVe2sX&GcoU!hvW<)f!F)iGLEd{t7t$C+8P?-q_ zX!^YLO&8{|#Yv-KVm?>#u`)M4LP}Re(?ea0yyHVqerp%=*kkTZUpcNoX4TNd^U+r< z94fs>k)+R`$7y6m6o)>6MDCQ?(J)@m(7)&!bJeez3p_Sc#n7HLSUr&uT6ZrK3rcvx zY)&w!;DA@-Kc0gj)KLzA)w1%9$P47{FuE}el8ydaPntVSjdK&N{y@SIj67?)o5=wpAlISlqa{F8q+rX5|nCjxPDy7}mU7PhieTCgtDQ4p-FV zoN>ems(UYQ`&&MpG)uS~GiDf?d*D%hp#g-}PIjhPci-9w%hDE|jVjcMg z&1_V3itkKVtjMk=!FbACg!w=V(mDwyQ4U(?;gk`)0kX^i&r&j9#la&W;z8Ds9IfZ_vp%i2`{f7Uj zFmhk(lWy#GZ>8XP zWL25vb>WPLkX3?~Vk~X%Z&htR3a|E!`ezfk*vzPcwtSW^sv`x{&y;JfIzPK#I-YXb ziDvuj8;SKbymA(y$v>wJSYUscW&D4!89^w$r0|B2tOy6lG(^hQkhiJ%f74&}=Z#l> zMA3`%}^`jydg=%6CY_qU{ zzP=03lA8jJQ)XJ;vT54xQMZV)@wF!zlVxSTWsr&XFy%jMz*u%6V!CAUfTB9a(yOmk zyo$}*+cRIoW-R+ARWkzuZ<%vB^-0Z{mydF--t)$c^OnZAydbUj?9~`LdYV_$wU1YO zi|6o!S)E`c{>`f&!eN4B0NH`J3SP-r-(|%Y5B8b3m&I8Y`fZzawB?PlUNm-PL&vO0 zZohLctKgk8Da^Cl7xc;tJKrXggeHg8lTMB+4H0Ma)17c43wTTZ5Cs?%@fS!)zU+#N z_lt@ea6wE%h-I)}qfO!qtYjf9iJ><+Y7%seeFIq{`y^B_FNA&ca(qP6ek$VAiw&V-4Cwefb1tk(FmKBuv?`00lhhc@GcRoLI4Xeok1{+OsIpv8m^}3;~-?r7=XQ?Y+?_D@{0f7_4oGT-@|vW zKp`+F=n5~uLnt5M7Kg$BOT!Qt9T-B%1nq2d9OKIafnc5{;O}20Osp?}oGK7{41_sj ztWqE@S;_&3*2M%S|2rW19}p2((#F!#0thR{#1ZR*Ft5V6IGLCMA7d0;R}(uB5|BSJ5EAG?5(Jh4CIsRGNFoQ~10c#EeqeMA#ei685DIvAz=Q~F3Idyf zz~(@EAX*uyY6WCKA2vV+ENk{4umgYwfF1}61@>glz=!}cAh0VyGGGD#2DbrZftUG! zP-U>TjXByHAPa_QKSw{@15fMfK1}`pTO#`BJ{+ivNtizVNeJLf8VbP>fS(sgKu0Kk5Fa0K z4G1{`7@QB25I~GP3PA)UKsJ#e1On&;ck3zHuw;4M`QF)&2HYFf9NL;#o^K!@)oWzg|!od8T7 z;3@_NeD(sPHID0P0`m90WC8*ZIfj`6vi`kf2C(#d$s7cT>G53)5J0@+UcT!Gz;S*I zKUN{fr|iH`7_9;VY{cU`Rv=)pJ1*G(U%2>QvIhZgJ017p0OI{Inj^62e7}p)QoKLL zbHZq^D~oV|9CM(EY5p zel}pFp`c@f=I8srup0Wa_W#$1Ou!Oiz*t~26TrWPQITS>|FIgN!~b0k{y$cO@BpeD zsEEl$M1E3BKY3>U{pA%5g??z>e*-au`bQlrVV0-wkGMHM(dU@82E=mC3GG0b+S`qTY84HW+VTtkds z!i}=!z8tx(J$R!vMM-hNt~itL=E$Nu=w;qJqqjl)d~Ym2pO&{omN<)t#*fT;4^%x* z>&tO-rWKhw?_?OX$G3H4P^SN))yZ84g&JY;0^cI~3JnEqy{e&{ul@LfxN(YLOYAJG zZ!?9!l-qQubr$##SC{NhyZM+0|0US}f?z*M{*Uhw{a3>NuM+mts_+H^uGlAdx9O?8UH2N|AJtD2-x>;^??;^%rTxRU^9g=+yFC5 z!qFA*nH<}zKSDHtP{)5urv&gpI*kAA65uFvvU5b+*_eXd;5__1yxbsbG}_5U01USO z*DW4LXDd!3FyJsVcQpgPZ})R6Cv!`Xsfn2_Fv^b}IDyeI-P<@iNMgPNcu7(K27&QH zc%gg{BriXlPmdGhY5IYdyR(HQ5wQ0|5CPldA3xv<0c?mYLH~kbtXIqrHSY zBlLScUd;CV*Zu&J`km(xBog&UTRsTncbp-7yqJB`ul?~M{*W~Y-=8#q@WU{>;9uMF zBQSg0-(dWheeAC=6cq5p{0=)o)=)^yzWcYf7|++QFu-4qaYFo35AYWQXE49R_<*3a z-(jdfc@B8of5#6Bh5x}fDDdF_E>}<}3gdbI6&Dy3<9PTD2LJO}031Gl)T9WhJ$@y=e(!O{_QytE(h*Im#i&gkPM6$SxZbwq4z JQYz9!{|A+*jfemM literal 0 HcmV?d00001 diff --git a/backend/data/evaluation/trending_repos.json b/backend/data/evaluation/trending_repos.json new file mode 100644 index 0000000..9f20ade --- /dev/null +++ b/backend/data/evaluation/trending_repos.json @@ -0,0 +1 @@ +{"svc-develop-team/so-vits-svc": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "bluesky-social/atproto": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "novuhq/novu": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "v2fly/v2ray-core": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "2dust/v2rayN": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "microsoft/DeepSpeed": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "sveltejs/svelte": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "apernet/hysteria": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "jmorganca/ollama": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "dotnet/eShop": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "oven-sh/bun": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "ggerganov/whisper.cpp": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "bevyengine/bevy": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "Dooy/chatgpt-web-midjourney-proxy": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "lllyasviel/ControlNet": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "shadcn/taxonomy": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "TabbyML/tabby": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "lllyasviel/ControlNet-v1-1-nightly": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "OpenBMB/ChatDev": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "imartinez/privateGPT": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "vanna-ai/vanna": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "usememos/memos": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "openai/openai-python": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "Mintplex-Labs/anything-llm": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "prasanthrangan/hyprdots": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "ollama-webui/ollama-webui": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "lobehub/lobe-chat": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "yt-dlp/yt-dlp": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "JushBJJ/Mr.-Ranedeer-AI-Tutor": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "danielgross/localpilot": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": false, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "qdrant/qdrant": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "jianchang512/clone-voice": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "reworkd/AgentGPT": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "xiangsx/gpt4free-ts": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "spacedriveapp/spacedrive": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "2dust/v2rayNG": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "AmruthPillai/Reactive-Resume": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "iv-org/invidious": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "phoboslab/wipeout-rewrite": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": false, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "DioxusLabs/dioxus": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "FlowiseAI/Flowise": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "tloen/alpaca-lora": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "alibaba/Chat2DB": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "amazon-science/mm-cot": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "facebookresearch/AnimatedDrawings": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "yangshun/tech-interview-handbook": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "PWhiddy/PokemonRedExperiments": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "anoma/namada": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "tinygrad/tinygrad": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "opentffoundation/manifesto": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "comfyanonymous/ComfyUI": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "ymcui/Chinese-LLaMA-Alpaca-2": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "ruanyf/weekly": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "jaymody/picoGPT": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "reflex-dev/reflex": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "joaomdmoura/crewAI": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "appwrite/appwrite": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "microsoft/generative-ai-for-beginners": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "NvChad/NvChad": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "biomejs/biome": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "ggerganov/llama.cpp": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "kuafuai/DevOpsGPT": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "csunny/DB-GPT": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "krahets/hello-algo": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "lyswhut/lx-music-desktop": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "google-deepmind/graphcast": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "bmaltais/kohya_ss": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": false, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "TencentARC/PhotoMaker": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "karanpratapsingh/system-design": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "apple/ml-stable-diffusion": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "LAION-AI/Open-Assistant": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "TheRamU/Fay": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "Dao-AILab/flash-attention": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "backstage/backstage": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "zloirock/core-js": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "chathub-dev/chathub": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "microsoft/ML-For-Beginners": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "LemmyNet/lemmy": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "GaiZhenbiao/ChuanhuChatGPT": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "AUTOMATIC1111/stable-diffusion-webui": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "loft-sh/devpod": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "SerenityOS/serenity": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "Codium-ai/pr-agent": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "bazingagin/npc_gzip": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "graphdeco-inria/gaussian-splatting": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "facebookresearch/audiocraft": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "MatsuriDayo/nekoray": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "geohot/tinygrad": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "gencay/vscode-chatgpt": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "imClumsyPanda/langchain-ChatGLM": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "pedroslopez/whatsapp-web.js": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "tldraw/tldraw": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "apache/incubator-answer": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "PlexPt/awesome-chatgpt-prompts-zh": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "geekan/MetaGPT": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "AI4Finance-Foundation/FinGPT": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "oobabooga/text-generation-webui": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "996icu/996.ICU": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "Yidadaa/ChatGPT-Next-Web": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "CodeEditApp/CodeEdit": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "haotian-liu/LLaVA": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "shadcn/ui": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "vercel/commerce": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "Ryujinx/Ryujinx": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "hwchase17/langchain": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "SagerNet/sing-box": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "hiyouga/ChatGLM-Efficient-Tuning": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "Significant-Gravitas/Auto-GPT": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "mckaywrigley/chatbot-ui": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "songquanpeng/one-api": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "tatsu-lab/stanford_alpaca": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "ConnectAI-E/Feishu-OpenAI": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "AntonOsika/gpt-engineer": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "zzzgydi/clash-verge": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "f/awesome-chatgpt-prompts": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "h2oai/h2ogpt": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "netease-youdao/QAnything": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "localsend/localsend": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "IceWhaleTech/CasaOS": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "langgenius/dify": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "Asabeneh/30-Days-Of-Python": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "l15y/wenda": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "XTLS/Xray-core": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "KRTirtho/spotube": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "kenjihiranabe/The-Art-of-Linear-Algebra": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "StanGirard/quivr": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "DataTalksClub/data-engineering-zoomcamp": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "fishaudio/Bert-VITS2": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "harness/gitness": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "facebookresearch/ijepa": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "facebookresearch/llama": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "abi/screenshot-to-code": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "cocktailpeanut/dalai": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "labring/FastGPT": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "go-skynet/LocalAI": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "modelscope/facechain": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "voicepaw/so-vits-svc-fork": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "RVC-Project/Retrieval-based-Voice-Conversion-WebUI": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "danny-avila/LibreChat": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "Stirling-Tools/Stirling-PDF": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "Hillobar/Rope": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "nextui-org/nextui": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "smol-ai/developer": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "toeverything/AFFiNE": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "donnemartin/system-design-primer": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "mastodon/mastodon": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "Pythagora-io/gpt-pilot": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "MooreThreads/Moore-AnimateAnyone": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": false, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "BuilderIO/gpt-crawler": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "invoke-ai/InvokeAI": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "ztjhz/BetterChatGPT": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "alibaba/lowcode-engine": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "TheAlgorithms/JavaScript": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "nushell/nushell": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "facebookresearch/seamless_communication": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "expo/expo": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "a16z-infra/ai-town": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "THUDM/ChatGLM-6B": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "organicmaps/organicmaps": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "calcom/cal.com": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "OpenBMB/ToolBench": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "sunner/ChatALL": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "NVlabs/neuralangelo": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "hpcaitech/ColossalAI": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "Stability-AI/generative-models": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "ytdl-org/youtube-dl": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "hehonghui/awesome-english-ebooks": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "practical-tutorials/project-based-learning": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "EbookFoundation/free-programming-books": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "microsoft/AI-For-Beginners": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "cpacker/MemGPT": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "vercel/platforms": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "iperov/DeepFaceLive": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "alibaba/ali-dbhub": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "airbnb/javascript": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "lllyasviel/Fooocus": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "steven-tey/dub": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "massgravel/Microsoft-Activation-Scripts": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "ByteByteGoHq/system-design-101": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "OpenTalker/video-retalking": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "jzhang38/TinyLlama": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "maybe-finance/maybe": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "facebookresearch/fairseq": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "Uniswap/v4-core": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "public-apis/public-apis": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "kaixindelele/ChatPaper": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "bregman-arie/devops-exercises": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "zanfranceschi/rinha-de-backend-2023-q3": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "rails/rails": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "xiaolai/everyone-can-use-english": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "godotengine/godot": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "zhayujie/chatgpt-on-wechat": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "NAStool/nas-tools": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "baichuan-inc/Baichuan-7B": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "danswer-ai/danswer": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "Chanzhaoyu/chatgpt-web": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "google/comprehensive-rust": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "shadcn-ui/ui": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "civitai/civitai": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "mrsked/mrsk": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "AppFlowy-IO/AppFlowy": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "servo/servo": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "stanfordnlp/dspy": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "jwasham/coding-interview-university": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "chat2db/Chat2DB": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "opentofu/opentofu": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": true, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "microsoft/guidance": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": false}, "nlpxucan/WizardLM": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "makeplane/plane": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "cbh123/narrator": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "gkamradt/langchain-tutorials": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": false, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "documenso/documenso": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "CompVis/stable-diffusion": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": true}, "Vaibhavs10/insanely-fast-whisper": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": false, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "Z4nzu/hackingtool": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}, "tiann/KernelSU": {"DescriptionOrReadme": true, "PersistentId": true, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": true, "ExactlyOneLicense": true, "ExplicitCitation": false, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": false, "SoftwareRequirements": false}, "coqui-ai/TTS": {"DescriptionOrReadme": true, "PersistentId": false, "PublicRepository": true, "SemanticVersioning": false, "UsageNotesInReadme": false, "ExactlyOneLicense": true, "ExplicitCitation": true, "DescriptionOrAtLeastOneTopic": true, "InstallationInstructionsInReadme": true, "SoftwareRequirements": true}} \ No newline at end of file diff --git a/backend/data/shacl/property_shapes.ttl b/backend/data/shacl/property_shapes.ttl index e987982..6b5eb25 100644 --- a/backend/data/shacl/property_shapes.ttl +++ b/backend/data/shacl/property_shapes.ttl @@ -85,8 +85,8 @@ propertyShapes:DoiAsHomepage # The homepage link is internally mapped via the "website" property. sh:path sd:website ; # https://www.crossref.org/blog/dois-and-matching-regular-expressions/ - # Put into the right format with the help of ChatGPT. - sh:pattern "^https://doi\\.org/10\\.[0-9]{4,}/[-._;()/:A-Z0-9]+$" ; + # Put into the right format with the help of ChatGPT, adapted. + sh:pattern "^https://doi\\.org/10\\.[0-9]{4,}/[-._;()/:A-Za-z0-9]+$" ; sh:minCount 1 ; sh:message "The homepage attribute is not set or its value is not a DOI. Make sure this attribute is filled with a DOI." ; sh:description diff --git a/backend/evaluation.py b/backend/evaluation.py new file mode 100644 index 0000000..a063495 --- /dev/null +++ b/backend/evaluation.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 + +import json +import logging +import pstats +from subprocess import run + +import pandas as pd +import numpy as np +from github import UnknownObjectException, Github, Auth +from matplotlib import pyplot as plt +from brokenaxes import brokenaxes + +import validation_interface, verbalization_interface + +logging.basicConfig(level=logging.INFO) + +def perform_evaluation() -> None: + with open(".github_access_token") as file: + github_access_token = file.readline().strip() + + repos_expected_to_be_fair = get_repos_expected_to_be_fair() + trending_repos = get_trending_repo_set() + + results_per_criterion_fair = get_validation_result_per_criterion(repos_expected_to_be_fair, github_access_token) + with open("./data/evaluation/repos_expected_to_be_fair.json", "w") as file: + json.dump(results_per_criterion_fair, file) + + results_per_criterion_trending = get_validation_result_per_criterion(trending_repos, github_access_token) + with open("./data/evaluation/trending_repos.json", "w") as file: + json.dump(results_per_criterion_trending, file) + + runtime_benchmark_results = execute_runtime_benchmark(repos_expected_to_be_fair, trending_repos, github_access_token) + with open("./data/evaluation/runtime_benchmark_results.json", "w") as file: + json.dump(runtime_benchmark_results, file) + + +def get_repos_expected_to_be_fair() -> list[str]: + return ["oeg-upm/oeg-software-graph", "zenodraft/zenodraft", + "comses-education/wolf-sheep", "fair-software/howfairis", + "GrainLearning/grainLearning", "online-behaviour/machine-learning"] + + +def get_trending_repo_set() -> list[str]: + # First 20 repositories on the 15th of each month from https://github.com/trending?since=monthly (via Web Archive) + trending_github_repos = { + "march_2023": [ + "lllyasviel/ControlNet", + "hpcaitech/ColossalAI", + "AUTOMATIC1111/stable-diffusion-webui", + "amazon-science/mm-cot", + "usememos/memos", + "civitai/civitai", + "ggerganov/whisper.cpp", + "zhayujie/chatgpt-on-wechat", + "openai/openai-python", + "f/awesome-chatgpt-prompts", + "mrsked/mrsk", + "yt-dlp/yt-dlp", + "zloirock/core-js", + "Asabeneh/30-Days-Of-Python", + "jaymody/picoGPT", + "bmaltais/kohya_ss", + "ytdl-org/youtube-dl", + "alibaba/lowcode-engine", + "NAStool/nas-tools", + "CompVis/stable-diffusion" + ], "april_2023": [ + "tloen/alpaca-lora", + "cocktailpeanut/dalai", + "THUDM/ChatGLM-6B", + "ggerganov/llama.cpp", + "tatsu-lab/stanford_alpaca", + "GaiZhenbiao/ChuanhuChatGPT", + "hwchase17/langchain", + "microsoft/DeepSpeed", + "Chanzhaoyu/chatgpt-web", + "oobabooga/text-generation-webui", + "gencay/vscode-chatgpt", + "PlexPt/awesome-chatgpt-prompts-zh", + "shadcn/taxonomy", + "apple/ml-stable-diffusion", + "ConnectAI-E/Feishu-OpenAI", + "ztjhz/BetterChatGPT", + "kaixindelele/ChatPaper", + "AUTOMATIC1111/stable-diffusion-webui", + "comfyanonymous/ComfyUI", + "chathub-dev/chathub" + ], "may_2023": [ + "Significant-Gravitas/Auto-GPT", + "voicepaw/so-vits-svc-fork", + "reworkd/AgentGPT", + "Ryujinx/Ryujinx", + "LAION-AI/Open-Assistant", + "imClumsyPanda/langchain-ChatGLM", + "svc-develop-team/so-vits-svc", + "pengzhile/pandora", + "hwchase17/langchain", + "lllyasviel/ControlNet-v1-1-nightly", + "facebookresearch/AnimatedDrawings", + "qdrant/qdrant", + "gkamradt/langchain-tutorials", + "RVC-Project/Retrieval-based-Voice-Conversion-WebUI", + "l15y/wenda", + "TheRamU/Fay", + "Yidadaa/ChatGPT-Next-Web", + "bluesky-social/atproto", + "shadcn/ui", + "calcom/cal.com" + ], "june_2023": [ + "microsoft/guidance", + "smol-ai/developer", + "sunner/ChatALL", + "google/comprehensive-rust", + "makeplane/plane", + "facebookresearch/fairseq", + "imartinez/privateGPT", + "pengzhile/pandora", + "xiangsx/gpt4free-ts", + "csunny/DB-GPT", + "FlowiseAI/Flowise", + "geohot/tinygrad", + "iperov/DeepFaceLive", + "steven-tey/dub", + "langgenius/dify", + "alibaba/Chat2DB", + "go-skynet/LocalAI", + "JushBJJ/Mr.-Ranedeer-AI-Tutor", + "iv-org/invidious", + "RVC-Project/Retrieval-based-Voice-Conversion-WebUI" + ], "july_2023": [ + "AntonOsika/gpt-engineer", + "AI4Finance-Foundation/FinGPT", + "alibaba/ali-dbhub", + "baichuan-inc/Baichuan-7B", + "tinygrad/tinygrad", + "StanGirard/quivr", + "h2oai/h2ogpt", + "CodeEditApp/CodeEdit", + "vercel/platforms", + "toeverything/AFFiNE", + "Uniswap/v4-core", + "LemmyNet/lemmy", + "sveltejs/svelte", + "facebookresearch/ijepa", + "hiyouga/ChatGLM-Efficient-Tuning", + "mastodon/mastodon", + "Mintplex-Labs/anything-llm", + "996icu/996.ICU", + "RVC-Project/Retrieval-based-Voice-Conversion-WebUI", + "loft-sh/devpod" + ], "august_2023": [ + "facebookresearch/llama", + "geekan/MetaGPT", + "bregman-arie/devops-exercises", + "comfyanonymous/ComfyUI", + "nextui-org/nextui", + "OpenBMB/ToolBench", + "Stability-AI/generative-models", + "EbookFoundation/free-programming-books", + "facebookresearch/audiocraft", + "invoke-ai/InvokeAI", + "Codium-ai/pr-agent", + "kenjihiranabe/The-Art-of-Linear-Algebra", + "Dao-AILab/flash-attention", + "DioxusLabs/dioxus", + "reflex-dev/reflex", + "bazingagin/npc_gzip", + "oobabooga/text-generation-webui", + "vercel/commerce", + "krahets/hello-algo", + "songquanpeng/one-api" + ], "september_2023": [ + "oven-sh/bun", + "opentffoundation/manifesto", + "krahets/hello-algo", + "lllyasviel/Fooocus", + "hehonghui/awesome-english-ebooks", + "kuafuai/DevOpsGPT", + "graphdeco-inria/gaussian-splatting", + "modelscope/facechain", + "zanfranceschi/rinha-de-backend-2023-q3", + "Z4nzu/hackingtool", + "a16z-infra/ai-town", + "phoboslab/wipeout-rewrite", + "organicmaps/organicmaps", + "NVlabs/neuralangelo", + "pedroslopez/whatsapp-web.js", + "documenso/documenso", + "prasanthrangan/hyprdots", + "IceWhaleTech/CasaOS", + "nlpxucan/WizardLM", + "ymcui/Chinese-LLaMA-Alpaca-2" + ], "october_2023": [ + "OpenBMB/ChatDev", + "opentofu/opentofu", + "Pythagora-io/gpt-pilot", + "godotengine/godot", + "appwrite/appwrite", + "coqui-ai/TTS", + "spacedriveapp/spacedrive", + "novuhq/novu", + "krahets/hello-algo", + "jwasham/coding-interview-university", + "massgravel/Microsoft-Activation-Scripts", + "haotian-liu/LLaVA", + "EbookFoundation/free-programming-books", + "graphdeco-inria/gaussian-splatting", + "bevyengine/bevy", + "harness/gitness", + "airbnb/javascript", + "TabbyML/tabby", + "fishaudio/Bert-VITS2", + "TheAlgorithms/JavaScript" + ], "november_2023": [ + "ByteByteGoHq/system-design-101", + "zzzgydi/clash-verge", + "cpacker/MemGPT", + "localsend/localsend", + "2dust/v2rayN", + "yangshun/tech-interview-handbook", + "SagerNet/sing-box", + "XTLS/Xray-core", + "public-apis/public-apis", + "PWhiddy/PokemonRedExperiments", + "chat2db/Chat2DB", + "v2fly/v2ray-core", + "2dust/v2rayNG", + "lyswhut/lx-music-desktop", + "OpenTalker/video-retalking", + "MatsuriDayo/nekoray", + "apache/incubator-answer", + "apernet/hysteria", + "danielgross/localpilot", + "donnemartin/system-design-primer" + ], "december_2023": [ + "abi/screenshot-to-code", + "BuilderIO/gpt-crawler", + "Stability-AI/generative-models", + "lllyasviel/Fooocus", + "microsoft/generative-ai-for-beginners", + "facebookresearch/seamless_communication", + "tldraw/tldraw", + "microsoft/ML-For-Beginners", + "lobehub/lobe-chat", + "dotnet/eShop", + "microsoft/AI-For-Beginners", + "comfyanonymous/ComfyUI", + "ollama-webui/ollama-webui", + "cbh123/narrator", + "Vaibhavs10/insanely-fast-whisper", + "biomejs/biome", + "google-deepmind/graphcast", + "AppFlowy-IO/AppFlowy", + "langgenius/dify", + "danny-avila/LibreChat" + ], "january_2024": [ + "Stirling-Tools/Stirling-PDF", + "Pythagora-io/gpt-pilot", + "mckaywrigley/chatbot-ui", + "jzhang38/TinyLlama", + "tachiyomiorg/tachiyomi", + "Hillobar/Rope", + "jianchang512/clone-voice", + "shadcn-ui/ui", + "anoma/namada", + "DataTalksClub/data-engineering-zoomcamp", + "jmorganca/ollama", + "AmruthPillai/Reactive-Resume", + "ruanyf/weekly", + "danswer-ai/danswer", + "karanpratapsingh/system-design", + "tiann/KernelSU", + "lobehub/lobe-chat", + "practical-tutorials/project-based-learning", + "apache/incubator-answer", + "labring/FastGPT" + ], "february_2024": [ + "maybe-finance/maybe", + "netease-youdao/QAnything", + "TencentARC/PhotoMaker", + "KRTirtho/spotube", + "stanfordnlp/dspy", + "vanna-ai/vanna", + "xiaolai/everyone-can-use-english", + "usememos/memos", + "DataTalksClub/data-engineering-zoomcamp", + "joaomdmoura/crewAI", + "SerenityOS/serenity", + "EbookFoundation/free-programming-books", + "MooreThreads/Moore-AnimateAnyone", + "expo/expo", + "backstage/backstage", + "servo/servo", + "Dooy/chatgpt-web-midjourney-proxy", + "NvChad/NvChad", + "rails/rails", + "nushell/nushell" + ]} + + repo_list: list = [] + + for key in trending_github_repos: + repo_list.extend(trending_github_repos[key]) + + return list(set(repo_list)) + + +def get_validation_result_per_criterion(repos: list, github_access_token: str) -> dict[str, dict[str, bool]]: + results_per_criterion: dict[str, dict[str, bool]] = {} + + for repo_name in repos: + try: + _, _, report = validation_interface.run_validator(github_access_token, repo_name, "FAIRSoftware") + except UnknownObjectException as e: + logging.exception(f"Could not validate {repo_name} against the FAIRSoftware project type. {e}") + continue + verbalized_explanation = verbalization_interface.run_verbalizer(report) + result_per_criterion = process_verbalized_explanation(verbalized_explanation) + results_per_criterion[repo_name] = result_per_criterion + + return results_per_criterion + + +def process_verbalized_explanation(verbalized_explanation: list[str]) -> dict[str, bool]: + possible_messages_with_mapping: dict[str, str] = { + "The repository has no description and no README file.": "DescriptionOrReadme", + "No persistent ID was found.": "PersistentId", + "The repository is private.": "PublicRepository", + "There are no releases or Semantic Versioning is violated.": "SemanticVersioning", + "There is no README file or it does not contain usage instructions for the software.": "UsageNotesInReadme", + "No license information was found.": "ExactlyOneLicense", + "No citation information was found.": "ExplicitCitation", + "The repository has no description and no topics assigned.": "DescriptionOrAtLeastOneTopic", + "There is no README file or it does not contain installation instructions.": "InstallationInstructionsInReadme", + "No information on the requirements of the software was found.": "SoftwareRequirements" + } + + result_per_criterion: dict[str, bool] = {} + for possible_message, mapping in possible_messages_with_mapping.items(): + if any(actual_message.startswith(possible_message) for actual_message in verbalized_explanation): + result_per_criterion[mapping] = False + else: + result_per_criterion[mapping] = True + + return result_per_criterion + + +def execute_runtime_benchmark(repos_expected_to_be_fair: list, trending_repos: list, github_access_token: str) -> None: + auth = Auth.Token(github_access_token) + g = Github(auth=auth) + + runtime_benchmark_results = dict() + + runtime_per_repo = [] + step_durations = [0, 0, 0] + + for repo_name in trending_repos + repos_expected_to_be_fair: + # skip repo for evaluation if relevant data cannot be fetched + try: + repo = g.get_repo(repo_name) + # compute repo size as the sum of releases branches and issues + repo_size = repo.get_releases().totalCount + repo.get_branches().totalCount + repo.get_issues().totalCount + except: + continue + + # perform fairness assessment with profiler + file_name = f"{repo_name.split('/')[1]}" + + logging.info(f"{repo_name} has in total {repo_size} releases, branches, and issues.") + + cmd = ["./shacl_validator.py", "--github_access_token", github_access_token, "--repo_name", repo_name, + "--expected_type", "FAIRSoftware"] + + run(["python3", "-m", "cProfile", "-o", f"data/benchmarks/{file_name}", "-s", "cumulative"] + cmd) + + # process stats of fairness assessment runtime + stats = pstats.Stats(f"data/benchmarks/{file_name}") + + for k, v in stats.stats.items(): + _, _, function = k + + if function == "validate_repo_against_specs": + runtime = v[3] + + elif function == "create_project_type_representation": + step_durations[0] += v[3] + + elif function == "create_repository_representation": + step_durations[1] += v[3] + + elif function == "run_validation": + step_durations[2] += v[3] + + runtime_per_repo.append(runtime) + origin = "trending" if repo_name in trending_repos else "expected" + runtime_benchmark_results[file_name] = (repo_size, runtime, origin) + + total_runtime = sum(runtime_per_repo) + step_durations = ["{:.2f}%".format(duration / total_runtime * 100) for duration in step_durations] + + logging.info( + f"Shapes graph composition/repository representation generation/validation account for {'/'.join(step_durations)} of the total runtime.") + + return runtime_benchmark_results + +def visualize_results() -> None: + + def get_results_in_percent(results_per_criterion: dict[str, dict[str, bool]]) -> dict[str, float]: + first_inner_dict = list(results_per_criterion.values())[0] + numeric_results: dict[str, int] = {key: 0 for key in first_inner_dict} + + for _, result_per_criterion in results_per_criterion.items(): + for criterion, value in result_per_criterion.items(): + if value: + numeric_results[criterion] += 1 + + return {key: value / len(results_per_criterion) * 100 for key, value in numeric_results.items()} + + # plot FAIRness assessment + with open("./data/evaluation/repos_expected_to_be_fair.json") as file_fair: + results_per_criterion_fair: dict[str, dict[str, bool]] = json.load(file_fair) + + with open("./data/evaluation/trending_repos.json") as file_trending: + results_per_criterion_trending: dict[str, dict[str, bool]] = json.load(file_trending) + + normalized_numeric_results_fair = get_results_in_percent(results_per_criterion_fair) + normalized_numeric_results_trending = get_results_in_percent(results_per_criterion_trending) + + best_practices = ("BP1", "BP2", "BP3", "BP4", "BP5", "BP6", "BP7", "BP8", "BP9", "BP10") + scores_fair = list(normalized_numeric_results_fair.values()) + scores_trending = list(normalized_numeric_results_trending.values()) + + x = np.arange(len(best_practices)) + width = 0.35 + fig, ax = plt.subplots(figsize=(6, 4)) + + ax.bar(x - width / 2, scores_fair, width, label="Repositories Expected to be FAIR (N=6)", color="#00407A") + ax.bar(x + width / 2, scores_trending, width, label="Trending Repositories (N=217)", color="#8ed7d7") + ax.legend(loc="lower left", ncol=1, bbox_to_anchor=(0, 1, 1, 0)) + + ax.set_ylabel("Percentage of Conform Repositories") + + ax.set_yticks(np.arange(0, 101, 10)) + ax.set_yticks(np.arange(5, 100, 5), minor=True) + ax.set_xticks(x) + ax.set_xticklabels(best_practices) + + ax.set_axisbelow(True) + ax.yaxis.grid(which="major", linestyle="dashed", linewidth=0.5, color="black") + ax.yaxis.grid(which="minor", linestyle="dashed", linewidth=0.5) + + fig.tight_layout() + + plt.savefig("./data/evaluation/conformity_per_best_practice.pdf") + + # plot runtime benchmark + x_trending = [] + y_trending = [] + + x_expected = [] + y_expected = [] + + with open("./data/evaluation/runtime_benchmark_results.json") as file_benchmark: + runtime_benchmark_results: dict[str, dict[str, bool]] = json.load(file_benchmark) + + for size, runtime, origin in runtime_benchmark_results.values(): + if origin == "trending": + x_trending.append(size) + y_trending.append(runtime) + + else: + x_expected.append(size) + y_expected.append(runtime) + + fig = plt.figure(figsize=(6, 4)) + + bax = brokenaxes(xlims=((0, 5000), (12000, 12500)), ylims=((0, 30), (50, 55))) + + bax.scatter(x_trending, y_trending, c="#8ed7d7", s=15, label="Trending Repositories\n(N=217)") + bax.scatter(x_expected, y_expected, c="#00407A", s=15, label="Repositories Expected to\nbe FAIR (N=6)") + + bax.set_xlabel("Repository Size") + bax.set_ylabel("Runtime Duration in Seconds") + + bax.legend(loc=1, ncol=1) + + bax.grid(axis='y', which='major', ls="dashed") + bax.grid(axis='y', which='minor', ls="dashed", linewidth=0.5) + + plt.savefig("./data/evaluation/runtime_benchmark_scatter.pdf") + + data = {'size': x_expected, + 'runtime': y_expected} + df_expected = pd.DataFrame(data) + + data = {'size': x_trending, + 'runtime': y_trending} + df_trending = pd.DataFrame(data) + + print(df_expected.describe(), df_trending.describe()) + +if __name__ == "__main__": + perform_evaluation() + visualize_results() diff --git a/backend/shacl_validator.py b/backend/shacl_validator.py index b484b4e..0fd9f2e 100644 --- a/backend/shacl_validator.py +++ b/backend/shacl_validator.py @@ -204,7 +204,7 @@ def versions_have_valid_increment(release_list: PaginatedList | list[dict]) -> b def version_pair_has_valid_version_increment(pair: tuple[Version, Version]) -> bool: # If the first number (major) is increased, the second (minor) and third (micro) must be set to zero. - if (pair[0].major + 1 == pair[1].major) & (pair[1].minor == 0) & (pair[1].micro == 0): + if (pair[0].major >= pair[1].major) & (pair[1].minor == 0) & (pair[1].micro == 0): return True # If minor is increased, micro must be set to zero. @@ -303,7 +303,7 @@ def include_readme(graph: Graph, repo_entity: URIRef, repo: Repository, include_ if include_check_for_doi: # Check whether there is at least one DOI in the README file (as text or link href). # Regex adapted from https://www.crossref.org/blog/dois-and-matching-regular-expressions/ - doi_pattern = re.compile(r"https://doi\.org/10\.\d{4,}/[-._;()/:A-Z0-9]+") + doi_pattern = re.compile(r"https://doi\.org/10\.\d{4,}/[-._;()/:A-Za-z0-9]+") if soup.find_all(string=doi_pattern) or soup.find_all(href=doi_pattern): graph.add((readme_entity, props["containsDoi"], Literal("true"))) else: diff --git a/docker-compose.yml b/docker-compose.yml index cafcf58..95fa4c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,3 +17,11 @@ services: ports: - "3000:3000" command: bash -c "npm install && npm run dev" + evaluation: + build: ./backend + container_name: evaluation + volumes: + - ./backend:/src + ports: + - "5000:5000" + entrypoint: bash