Significativity Indices for Agreement Values
Extracted source text · arXiv 2504.15325
%!TeX spellcheck = en-GB
%\documentclass[a4]{article}
%\documentclass[pdflatex,sn-mathphys-ay,iicol]{sn-jnl}
\documentclass[a4]{article}
\usepackage[affil-it]{authblk}
\usepackage{amsfonts, amstext, amssymb}
\usepackage{amsmath, amsthm}
\usepackage{xspace}
\usepackage{siunitx}
\usepackage{subcaption}
\usepackage{algorithmicx}
\usepackage{algpseudocode}
\usepackage{algorithm}
\usepackage{hyperref}
\usepackage{bookmark}
%undefine orcidlogo
\let\orcidlogo\undefined
\usepackage{orcidlink}
\usepackage{pgfplots}
\usepgfplotslibrary{fillbetween}
\usepackage{tikz,ifthen}
\usepackage{tikz-3dplot}
\pgfplotsset{compat=1.18}
%\theoremstyle{thmstyleone}
\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
%\theoremstyle{thmstyletwo}
\newtheorem{example}{Example}
\newtheorem{remark}{Remark}
%\theoremstyle{thmstylethree}
\newtheorem{definition}{Definition}
\begin{filecontents*}{kappa_2.csv}
kappa R_kappa_2_2 num_of_samples_R_kappa_2_2 mean_sampled_R_kappa_2_2 mean_delta_R_kappa_2_2 stdev_sampled_R_kappa_2_2 stdev_delta_R_kappa_2_2 R_kappa_2_4 num_of_samples_R_kappa_2_4 mean_sampled_R_kappa_2_4 mean_delta_R_kappa_2_4 stdev_sampled_R_kappa_2_4 stdev_delta_R_kappa_2_4 R_kappa_2_8 num_of_samples_R_kappa_2_8 mean_sampled_R_kappa_2_8 mean_delta_R_kappa_2_8 stdev_sampled_R_kappa_2_8 stdev_delta_R_kappa_2_8 R_kappa_2_16 num_of_samples_R_kappa_2_16 mean_sampled_R_kappa_2_16 mean_delta_R_kappa_2_16 stdev_sampled_R_kappa_2_16 stdev_delta_R_kappa_2_16
-1 0 4 0 0 0 0 0 6 0 0 0 0 0 13 0 0 0 0 0 32 0 0 0 0
-0.9 0 4 0 0 0 0 0 6 0 0 0 0 0 13 0 0 0 0 0 32 0 0 0 0
-0.8 0 4 0 0 0 0 0 6 0 0 0 0 0 13 0 0 0 0 0 32 0 0 0 0
-0.7 0 4 0 0 0 0 0 6 0 0 0 0 0 13 0 0 0 0 0.00619195046439628 32 0.0053125 -0.000879450464396285 0.0117385249818706 0.0117385249818706
-0.6 0 4 0 0 0 0 0 6 0 0 0 0 0 13 0 0 0 0 0.0206398348813209 32 0.02125 0.000610165118679053 0.0245586033804857 0.0245586033804857
-0.5 0 4 0 0 0 0 0 6 0 0 0 0 0.0121212121212121 13 0.0153846153846154 0.00326340326340326 0.036080121229411 0.036080121229411 0.0412796697626419 32 0.0396875 -0.00159216976264189 0.0402276238267935 0.0402276238267935
-0.4 0 4 0 0 0 0 0 6 0 0 0 0 0.0545454545454545 13 0.0584615384615385 0.00391608391608392 0.0616537381831196 0.0616537381831197 0.0743034055727554 32 0.07125 -0.00305340557275542 0.0428660704987056 0.0428660704987056
-0.3 0 4 0 0 0 0 0.0285714285714286 6 0.0266666666666667 -0.00190476190476192 0.0611010092660779 0.0611010092660779 0.0666666666666667 13 0.0623076923076923 -0.00435897435897435 0.069397235761671 0.069397235761671 0.118679050567595 32 0.128125 0.00944594943240456 0.0583798177883419 0.0583798177883419
-0.2 0 4 0 0 0 0 0.0285714285714286 6 0.0283333333333333 -0.000238095238095244 0.0626054665699765 0.0626054665699764 0.157575757575758 13 0.16 0.00242424242424245 0.0933784055722679 0.0933784055722678 0.183694530443756 32 0.1725 -0.0111945304437565 0.0617897139498153 0.0617897139498153
-0.1 0 4 0 0 0 0 0.0285714285714286 6 0.0233333333333333 -0.00523809523809525 0.0667499479816693 0.0667499479816693 0.175757575757576 13 0.180769230769231 0.00501165501165503 0.120342214208939 0.120342214208939 0.264189886480908 32 0.269375 0.00518511351909187 0.0757488655030556 0.0757488655030556
0 0 4 0 0 0 0 0.0285714285714286 6 0.035 0.00642857142857142 0.0756270086969228 0.0756270086969228 0.181818181818182 13 0.216153846153846 0.0343356643356644 0.101290489749085 0.101290489749085 0.306501547987616 32 0.3078125 0.00131095201238391 0.0747881121485895 0.0747881121485895
0.1 0.3 4 0.2675 -0.0325 0.255575331360443 0.255575331360443 0.485714285714286 6 0.53 0.0442857142857143 0.205993527406405 0.205993527406405 0.496969696969697 13 0.510769230769231 0.0137995337995338 0.132575757063449 0.132575757063449 0.498452012383901 32 0.493125 -0.00532701238390091 0.0807170017716218 0.0807170017716218
0.2 0.3 4 0.3 -7.7715611723761e-18 0.244948974278318 0.244948974278318 0.514285714285714 6 0.495 -0.0192857142857142 0.21915874510399 0.21915874510399 0.539393939393939 13 0.553846153846154 0.0144522144522145 0.134999452113725 0.134999452113725 0.586171310629515 32 0.5815625 -0.00460881062951489 0.0967130703098087 0.0967130703098087
0.3 0.3 4 0.2975 -0.00249999999999992 0.2170685375636 0.2170685375636 0.514285714285714 6 0.506666666666666 -0.00761904761904759 0.215921796542689 0.215921796542689 0.587878787878788 13 0.603076923076923 0.0151981351981353 0.134876669503407 0.134876669503407 0.640866873065015 32 0.644375 0.00350812693498457 0.0820608577520367 0.0820608577520367
0.4 0.3 4 0.29 -0.01 0.236431808350738 0.236431808350738 0.514285714285714 6 0.541666666666667 0.0273809523809524 0.193469779437398 0.193469779437398 0.648484848484848 13 0.677692307692307 0.0292074592074593 0.129949922079702 0.129949922079702 0.708978328173375 32 0.709375 0.000396671826625417 0.0789951541235284 0.0789951541235284
0.5 0.3 4 0.275 -0.025 0.210653744329409 0.210653744329409 0.514285714285714 6 0.508333333333333 -0.00595238095238092 0.189113428631837 0.189113428631837 0.654545454545455 13 0.669230769230769 0.0146853146853147 0.145341874059163 0.145341874059163 0.758513931888545 32 0.7603125 0.00179856811145512 0.0761699118336762 0.0761699118336762
0.6 0.3 4 0.3375 0.0375000000000002 0.227417567483253 0.227417567483253 0.571428571428571 6 0.603333333333333 0.0319047619047619 0.196892525674965 0.196892525674965 0.739393939393939 13 0.747692307692307 0.00829836829836833 0.125495467750973 0.125495467750973 0.815273477812177 32 0.8015625 -0.0137109778121774 0.0689450358528444 0.0689450358528444
0.7 0.3 4 0.295 -0.005 0.221867077323338 0.221867077323338 0.571428571428571 6 0.601666666666666 0.0302380952380953 0.209357482683461 0.209357482683461 0.739393939393939 13 0.731538461538461 -0.00785547785547783 0.122351225924415 0.122351225924414 0.853457172342621 32 0.854375 0.000917827657378805 0.0657320555360929 0.0657320555360929
0.8 0.3 4 0.285 -0.015 0.220850628253578 0.220850628253578 0.571428571428571 6 0.556666666666667 -0.0147619047619047 0.201687326765411 0.201687326765411 0.812121212121212 13 0.826153846153845 0.0140326340326342 0.113377413402417 0.113377413402417 0.894736842105263 32 0.8975 0.00276315789473695 0.0571251367613242 0.0571251367613242
0.9 0.3 4 0.3225 0.0225 0.207047699818182 0.207047699818182 0.571428571428571 6 0.59 0.0185714285714286 0.194964384211863 0.194964384211863 0.812121212121212 13 0.792307692307691 -0.0198135198135197 0.106309807392963 0.106309807392963 0.934984520123839 32 0.930625 -0.00435952012383892 0.0408837911035657 0.0408837911035657
1 0.3 4 0.29 -0.01 0.202237484161567 0.202237484161567 0.571428571428571 6 0.566666666666667 -0.00476190476190475 0.195789002074512 0.195789002074512 0.812121212121212 13 0.81076923076923 -0.00135198135198124 0.111163827073847 0.111163827073847 0.934984520123839 32 0.934375 -0.000609520123838925 0.0418096505008114 0.0418096505008114
\end{filecontents*}
\begin{filecontents*}{kappa_3.csv}
kappa R_kappa_3_2 num_of_samples_R_kappa_3_2 mean_sampled_R_kappa_3_2 mean_delta_R_kappa_3_2 stdev_sampled_R_kappa_3_2 stdev_delta_R_kappa_3_2 R_kappa_3_4 num_of_samples_R_kappa_3_4 mean_sampled_R_kappa_3_4 mean_delta_R_kappa_3_4 stdev_sampled_R_kappa_3_4 stdev_delta_R_kappa_3_4 R_kappa_3_8 num_of_samples_R_kappa_3_8 mean_sampled_R_kappa_3_8 mean_delta_R_kappa_3_8 stdev_sampled_R_kappa_3_8 stdev_delta_R_kappa_3_8 R_kappa_3_16 num_of_samples_R_kappa_3_16 mean_sampled_R_kappa_3_16 mean_delta_R_kappa_3_16 stdev_sampled_R_kappa_3_16 stdev_delta_R_kappa_3_16
-1 0 7 0 0 0 0 0 23 0 0 0 0 0 114 0 0 0 0 0 858 0 0 0 0
-0.9 0 7 0 0 0 0 0 23 0 0 0 0 0 114 0 0 0 0 0 858 0 0 0 0
-0.8 0 7 0 0 0 0 0 23 0 0 0 0 0 114 0 0 0 0 0.000171318787552466 858 0.00013986013986014 -3.14586476923265e-05 0.000378743049167178 0.000378743049167178
-0.7 0 7 0 0 0 0 0 23 0 0 0 0 0.00248640248640249 114 0.00245614035087719 -3.02621355252936e-05 0.00465491198646372 0.00465491198646372 0.00071382828146861 858 0.000792540792540793 7.87125110721825e-05 0.000959399329942036 0.000959399329942037
-0.6 0 7 0 0 0 0 0 23 0 0 0 0 0.00248640248640249 114 0.00245614035087719 -3.02621355252936e-05 0.00448656556229629 0.00448656556229629 0.00224074096735289 858 0.00210955710955711 -0.000131183857795784 0.0015435301825502 0.0015435301825502
-0.5 0 7 0 0 0 0 0 23 0 0 0 0 0.00994560994560995 114 0.00903508771929825 -0.0009105222263117 0.00808303704324142 0.00808303704324142 0.00749315744604478 858 0.00710955710955711 -0.000383600336487669 0.00281414836075749 0.00281414836075749
-0.4 0 7 0 0 0 0 0.0282828282828283 23 0.0360869565217392 0.00780412823891086 0.0369206987504202 0.0369206987504202 0.0289044289044289 114 0.027719298245614 -0.00118513065881487 0.0147159417372005 0.0147159417372005 0.0213291890502821 858 0.0213752913752914 4.6102325009314e-05 0.00459810520673699 0.00459810520673699
-0.3 0 7 0 0 0 0 0.0464646464646465 23 0.0504347826086957 0.00397013614404919 0.0483058936081084 0.0483058936081083 0.0452214452214452 114 0.0442982456140351 -0.000923199607410132 0.0191932951640711 0.0191932951640711 0.0506668515821834 858 0.050979020979021 0.000312169396837595 0.00758134991724928 0.00758134991724928
-0.2 0 7 0 0 0 0 0.0545454545454545 23 0.0569565217391305 0.00241106719367589 0.0458875917596982 0.0458875917596982 0.118259518259518 114 0.118333333333333 7.38150738150786e-05 0.0303487221626161 0.030348722162616 0.107711928818403 858 0.107645687645688 -6.62411727157956e-05 0.00986143477640523 0.00986143477640523
-0.1 0 7 0 0 0 0 0.0606060606060606 23 0.053913043478261 -0.00669301712779973 0.0394288140826248 0.0394288140826248 0.182517482517483 114 0.184473684210526 0.0019562016930438 0.033847716286549 0.033847716286549 0.201761864165956 858 0.19990675990676 -0.00185510425919633 0.0146069243831707 0.0146069243831707
0 0 7 0 0 0 0 0.0727272727272727 23 0.0765217391304348 0.00379446640316206 0.0546928180644048 0.0546928180644048 0.239316239316239 114 0.235175438596491 -0.00414080071974808 0.03818954122253 0.03818954122253 0.322145944571574 858 0.319871794871795 -0.00227414969977899 0.016087610636109 0.016087610636109
0.1 0.288888888888889 7 0.305714285714286 0.0168253968253968 0.176149287334838 0.176149287334838 0.545454545454545 23 0.545217391304348 -0.000237154150197592 0.108333309097957 0.108333309097957 0.518414918414918 114 0.514122807017544 -0.00429211139737452 0.0465143107212985 0.0465143107212985 0.493457933759455 858 0.49025641025641 -0.00320152350304453 0.0174658195315307 0.0174658195315307
0.2 0.288888888888889 7 0.32 0.0311111111111111 0.172756759518068 0.172756759518068 0.604040404040404 23 0.617391304347826 0.0133509003074222 0.106321906614239 0.106321906614239 0.62051282051282 114 0.616666666666667 -0.00384615384615374 0.0415256153633145 0.0415256153633145 0.631712195314295 858 0.634475524475524 0.00276332916122936 0.013980943136686 0.013980943136686
0.3 0.288888888888889 7 0.311428571428571 0.0225396825396825 0.174238869582735 0.174238869582735 0.604040404040404 23 0.595652173913044 -0.00838823012736045 0.104075732205094 0.104075732205094 0.703885003885004 114 0.698070175438596 -0.00581482844640736 0.0468815130901448 0.0468815130901448 0.738378535659462 858 0.737692307692308 -0.000686227967153955 0.0155154603979025 0.0155154603979025
0.4 0.333333333333333 7 0.318571428571428 -0.0147619047619047 0.194679223339318 0.194679223339318 0.636363636363636 23 0.649565217391305 0.013201581027668 0.0962078719823901 0.0962078719823901 0.773892773892774 114 0.773859649122807 -3.31247699668646e-05 0.0347921763775757 0.0347921763775757 0.823265635218792 858 0.823368298368298 0.000102663149506662 0.0129043402704557 0.0129043402704557
0.5 0.333333333333333 7 0.337142857142857 0.00380952380952381 0.198484050662382 0.198484050662382 0.688888888888889 23 0.686521739130435 -0.00236714975845411 0.104401254437705 0.104401254437705 0.825796425796426 114 0.826228070175438 0.000431644379012837 0.029164391152496 0.0291643911524959 0.88344883754764 858 0.883146853146852 -0.000301984400786331 0.0102466312779323 0.0102466312779323
0.6 0.333333333333333 7 0.317142857142857 -0.0161904761904762 0.162355115944861 0.162355115944861 0.745454545454545 23 0.752173913043478 0.00671936758893289 0.0846431405823121 0.0846431405823121 0.87979797979798 114 0.879649122807016 -0.00014885699096216 0.0363602575446947 0.0363602575446947 0.928171199136336 858 0.928706293706294 0.000535094569957989 0.00822327769923889 0.00822327769923889
0.7 0.333333333333333 7 0.361428571428571 0.0280952380952381 0.162197913874705 0.162197913874705 0.747474747474747 23 0.753478260869565 0.0060035133948178 0.0869684774389717 0.0869684774389717 0.895881895881896 114 0.896228070175437 0.000346174293542774 0.0256060730235002 0.0256060730235002 0.958581643599816 858 0.958706293706294 0.000124650106478136 0.00652109098728792 0.00652109098728793
0.8 0.333333333333333 7 0.354285714285714 0.020952380952381 0.182365410254091 0.182365410254091 0.747474747474747 23 0.737391304347826 -0.0100834431269213 0.0860649950028426 0.0860649950028426 0.936130536130536 114 0.936666666666666 0.00053613053613056 0.0218271279587506 0.0218271279587506 0.978094309632875 858 0.977249417249417 -0.000844892383457445 0.00505478272016029 0.00505478272016029
0.9 0.333333333333333 7 0.332857142857143 -0.000476190476190471 0.179631141339066 0.179631141339066 0.747474747474747 23 0.744782608695653 -0.00269213877909523 0.0805592456549308 0.0805592456549308 0.93954933954934 114 0.941666666666666 0.00211732711732715 0.0216072770820239 0.0216072770820239 0.991817488384994 858 0.991585081585082 -0.000232406799912471 0.00294657504341911 0.00294657504341911
1 0.333333333333333 7 0.344285714285714 0.010952380952381 0.179676580191709 0.179676580191709 0.747474747474747 23 0.750434782608696 0.00296003513394824 0.0855583242186299 0.0855583242186299 0.93954933954934 114 0.94157894736842 0.00202960781908153 0.0224403727538582 0.0224403727538582 0.993033035972866 858 0.992843822843823 -0.000189213129043446 0.00245597396763693 0.00245597396763693
\end{filecontents*}
\begin{filecontents*}{kappa_rho.csv}
kappa rho2 rho2_err rho3 rho3_err rho4 rho4_err rho5 rho5_err
-1 0 0 0 0 0 0 0 0
-0.9 0.000955 0.000301454805899657 0 0 0 0 0 0
-0.8 0.005753 0.000776460559204394 1e-06 9.94987437106618e-06 0 0 0 0
-0.7 0.016362 0.00118817338802045 8e-06 2.71293199325011e-05 0 0 0 0
-0.6 0.036024 0.00172731699464806 9.89999999999999e-05 9.84835011562851e-05 0 0 0 0
-0.5 0.066447 0.00278084357704636 0.000991 0.000322829676454938 0 0 0 0
-0.4 0.11113 0.00320398189757683 0.009522 0.000886406227414948 2e-05 4.24264068711929e-05 0 0
-0.3 0.173141 0.00386996369492015 0.050017 0.00197428746640402 0.00151 0.000369729630946723 2e-06 1.4e-05
-0.2 0.257305 0.00419557802930657 0.150729 0.00367620170828533 0.042949 0.00193997396889752 0.002928 0.000540755027715878
-0.1 0.364407 0.00489724932998107 0.318532 0.00435896501477128 0.230784 0.00359039607842923 0.135493 0.00399284998466008
0 0.499722 0.00533793180923098 0.519876 0.0050095133496179 0.530493 0.00496923042331507 0.534287 0.00506190981745033
0.1 0.620758 0.00516809790928926 0.702289 0.00485915414449881 0.784037 0.00389963216214042 0.854526 0.00349958911873951
0.2 0.713557 0.00443667116203128 0.837208 0.00345203360354443 0.92541 0.00278249887690903 0.972774000000001 0.00149188605463017
0.3 0.787623 0.00405999642856986 0.921262 0.00279334136832576 0.980434 0.00122370094385842 0.996883 0.00054148961208873
0.4 0.847674 0.00367289041491848 0.967093 0.0017200729635687 0.996333 0.000621297835180519 0.999804999999999 0.000140267601391044
0.5 0.895907 0.00318578891328349 0.988475 0.00090866660552702 0.999529 0.000190942399691656 0.999993 2.55147016443433e-05
0.6 0.933935 0.00226280246597002 0.996946 0.000466566179657294 0.999958999999999 6.79632253501773e-05 1 0
0.7 0.963572 0.00181472201727978 0.999443 0.000247489393712152 0.999998 1.39999999999985e-05 1 0
0.8 0.983845 0.00113255242704256 0.999945999999999 7.6707235643052e-05 1 0 1 0
0.9 0.995994 0.00064090872985161 0.999998 1.39999999999984e-05 1 0 1 0
1 1 0 1 0 1 0 1 0
\end{filecontents*}
\begin{filecontents*}{IA_rho.csv}
IA rho2 rho2_err rho3 rho3_err rho4 rho4_err rho5 rho5_err
0 0 0 0 0 0 0 0 0
0.05 0.387591 0.00504127156578576 0.089924 0.00259457588056314 0.013502 0.00104278281535514 0.001357 0.000400063744920731
0.1 0.53452 0.00520599654244987 0.28233 0.0046956043274535 0.152041 0.00279710904328022 0.081984 0.00264657212257667
0.15 0.638333 0.0049689044064059 0.490131 0.00454773998816995 0.430815 0.00482189537422786 0.410798 0.00523276179469313
0.2 0.716663 0.00420337138497183 0.667631 0.00530876058981755 0.703671 0.00468816157997994 0.769439 0.00404294187442758
0.25 0.777618 0.00360039942228637 0.799542 0.00371699287058773 0.875499 0.00322085376880106 0.94094 0.00236647417057529
0.3 0.826955 0.0039875650464914 0.885978 0.0031257824620405 0.956912 0.00187884432564276 0.989489 0.000992158757457696
0.35 0.865827 0.00365895763845388 0.93985 0.00223367410335528 0.987141 0.00121656031498647 0.998516 0.000365983606190228
0.4 0.897574 0.00306622308386067 0.970058 0.00174546154354658 0.99673 0.000575760366819397 0.999850999999998 0.000122061459928991
0.45 0.922682 0.00267250369503954 0.986282 0.00113458186130397 0.999289 0.000275642884907274 0.999984 3.92937654087727e-05
0.5 0.943087 0.00206875107250728 0.994213 0.000745205340828958 0.999871999999999 0.000125761679378091 0.999999 9.9498743710651e-06
0.55 0.958529 0.00188097288656695 0.997694 0.000411295514198732 0.99998 3.99999999999956e-05 1 0
0.6 0.971096000000001 0.00172533590932317 0.999263 0.000280233830934102 0.999998 1.39999999999984e-05 1 0
0.65 0.979988 0.00140892015387672 0.999745999999999 0.000149946657181808 0.999999 9.9498743710651e-06 1 0
0.7 0.987189 0.00107469949288161 0.999932999999999 7.88098978555273e-05 1 0 1 0
0.75 0.992267 0.000944992592563563 0.999982 4.09389789809129e-05 1 0 1 0
0.8 0.995635 0.000595378031170109 0.999999 9.9498743710651e-06 1 0 1 0
0.85 0.997998 0.000405457765988027 0.999999 9.9498743710651e-06 1 0 1 0
0.9 0.999295 0.000228199474144889 1 0 1 0 1 0
0.95 0.999859999999999 0.000135646599662494 1 0 1 0 1 0
1 1 0 1 0 1 0 1 0
\end{filecontents*}
\begin{filecontents*}{kappa_delta.csv}
kappa rho2 varrho2_10 delta2_10 varrho2_100 delta2_100 varrho2_1000 delta2_1000 rho3 varrho3_10 delta3_10 varrho3_100 delta3_100 varrho3_1000 delta3_1000
-1 0 0 0 0 0 0 0 0 0 0 0 0 0 0
-0.9 7e-04 0 7e-04 0.00084 -0.00014 0.00097 -0.00027 0 0 0 0 0 0 0
-0.8 0.0051 0 0.0051 0.00537 -0.000269999999999999 0.00567 -0.000569999999999999 0 0 0 0 0 0 0
-0.7 0.0157 0 0.0157 0.01558 0.000119999999999999 0.01588 -0.00018 0 0.00092 -0.00092 3e-05 -3e-05 0 0
-0.6 0.0335 0.00666 0.02684 0.03377 -0.000270000000000006 0.03603 -0.00253 0 0.0039 -0.0039 2e-04 -2e-04 7e-05 -7e-05
-0.5 0.0675 0.03223 0.03527 0.06254 0.00495999999999999 0.0662 0.0013 0.001 0.01118 -0.01018 0.00151 -0.00051 0.00086 0.00014
-0.4 0.1098 0.07068 0.03912 0.10573 0.00406999999999998 0.11129 -0.00149000000000001 0.0095 0.02722 -0.01772 0.00956 -5.99999999999993e-05 0.00908 0.00042
-0.3 0.174 0.09812 0.07588 0.16358 0.01042 0.17256 0.00144 0.0485 0.05418 -0.00567999999999999 0.04505 0.00345000000000001 0.04832 0.00018
-0.2 0.2506 0.13273 0.11787 0.24103 0.00956999999999997 0.25394 -0.00334000000000001 0.151 0.09947 0.05153 0.1259 0.0251 0.14653 0.00447
-0.1 0.3653 0.21473 0.15057 0.3412 0.0241 0.36325 0.00205000000000005 0.3167 0.19272 0.12398 0.26506 0.05164 0.30894 0.00776000000000004
0 0.5106 0.22746 0.28314 0.46115 0.04945 0.49077 0.0198300000000001 0.5263 0.27496 0.25134 0.43765 0.08865 0.50906 0.0172399999999999
0.1 0.6162 0.50042 0.11578 0.58158 0.03462 0.61897 -0.00277000000000005 0.7043 0.50283 0.20147 0.61896 0.0853400000000001 0.69172 0.01258
0.2 0.7119 0.56656 0.14534 0.67796 0.03394 0.70852 0.00337999999999994 0.8319 0.62291 0.20899 0.77023 0.06167 0.82955 0.00235000000000007
0.3 0.7852 0.61399 0.17121 0.75568 0.0295199999999999 0.78455 0.000650000000000039 0.9226 0.71923 0.20337 0.87337 0.0492299999999999 0.91593 0.00667000000000018
0.4 0.8461 0.66962 0.17648 0.8182 0.0278999999999999 0.84388 0.00221999999999989 0.9676 0.78726 0.18034 0.93957 0.02803 0.96438 0.00322
0.5 0.896 0.70667 0.18933 0.87039 0.0256099999999999 0.89296 0.00303999999999993 0.9888 0.8515 0.1373 0.97555 0.0132499999999999 0.98763 0.00117000000000012
0.6 0.9322 0.7401 0.1921 0.91336 0.0188400000000001 0.932 0.000199999999999867 0.9979 0.88653 0.11137 0.99062 0.00727999999999995 0.99683 0.0010699999999999
0.7 0.9641 0.79524 0.16886 0.94721 0.0168899999999998 0.96147 0.00263000000000002 0.9997 0.92748 0.0722200000000002 0.99773 0.00197000000000003 0.99927 0.00042999999999993
0.8 0.9826 0.83661 0.14599 0.9722 0.0104000000000001 0.98234 0.000260000000000038 1 0.94673 0.05327 0.99961 0.000390000000000001 0.99996 3.9999999999929e-05
0.9 0.9953 0.8617 0.1336 0.98988 0.00541999999999998 0.99526 3.9999999999929e-05 1 0.96611 0.0338900000000001 0.99994 5.9999999999838e-05 1 0
1 1 0.8645 0.1355 0.99787 0.00212999999999997 0.99998 1.9999999999909e-05 1 0.96599 0.03401 1 0 1 0
\end{filecontents*}
\begin{filecontents*}{IA_delta.csv}
IA rho2 varrho2_10 delta2_10 varrho2_100 delta2_100 varrho2_1000 delta2_1000 rho3 varrho3_10 delta3_10 varrho3_100 delta3_100 varrho3_1000 delta3_1000
0 0 0.00684 -0.00684 0.00039 -0.00039 2e-05 -2e-05 0 0 0 0 0 0 0
0.05 0.3817 0.10774 0.27396 0.34851 0.0331899999999999 0.38099 0.000709999999999933 0.0915 0 0.0915 0.0539 0.0376 0.08703 0.00446999999999999
0.1 0.5434 0.13303 0.41037 0.4909 0.0525 0.52859 0.01481 0.278 0.00021 0.27779 0.18958 0.08842 0.27078 0.00722
0.15 0.641 0.21094 0.43006 0.59414 0.0468600000000001 0.6354 0.00559999999999994 0.4861 0.00187 0.48423 0.35954 0.12656 0.47908 0.00702000000000008
0.2 0.7136 0.2618 0.4518 0.67092 0.04268 0.71344 0.000159999999999938 0.6842 0.0059 0.6783 0.52967 0.15453 0.65564 0.02856
0.25 0.775 0.343 0.432 0.73428 0.04072 0.77522 -0.000219999999999998 0.7904 0.00972 0.78068 0.67455 0.11585 0.79128 -0.000880000000000103
0.3 0.8278 0.35213 0.47567 0.78447 0.04333 0.82319 0.00461 0.8847 0.02038 0.86432 0.78384 0.10086 0.88292 0.00178
0.35 0.8705 0.45033 0.42017 0.82859 0.0419100000000001 0.86451 0.00598999999999994 0.9396 0.09931 0.84029 0.86254 0.0770600000000001 0.93625 0.00335000000000008
0.4 0.8998 0.45222 0.44758 0.86414 0.0356599999999999 0.89342 0.00638000000000016 0.9708 0.10998 0.86082 0.91495 0.0558500000000001 0.96929 0.00150999999999979
0.45 0.9263 0.56171 0.36459 0.89201 0.0342899999999998 0.91938 0.00692000000000004 0.9872 0.12649 0.86071 0.94912 0.0380799999999999 0.98651 0.00069000000000019
0.5 0.9417 0.56343 0.37827 0.9161 0.0256000000000001 0.94005 0.00165000000000004 0.9946 0.14818 0.84642 0.96815 0.0264499999999999 0.99355 0.00104999999999977
0.55 0.9589 0.59009 0.36881 0.93519 0.0237099999999999 0.9555 0.00339999999999996 0.9968 0.16394 0.83286 0.98003 0.0167700000000001 0.99771 -0.000909999999999855
0.6 0.9728 0.61153 0.36127 0.94956 0.02324 0.9701 0.00270000000000015 0.9995 0.18309 0.81641 0.98659 0.0129100000000001 0.99901 0.000490000000000101
0.65 0.9799 0.67815 0.30175 0.96102 0.0188799999999999 0.97894 0.000960000000000072 0.9997 0.19798 0.80172 0.98962 0.01008 0.99964 6.000000000006e-05
0.7 0.9879 0.67871 0.30919 0.9703 0.0176000000000001 0.98598 0.00191999999999981 1 0.45756 0.54244 0.99164 0.00835999999999992 0.99991 8.99999999999235e-05
0.75 0.9919 0.67702 0.31488 0.97815 0.01375 0.99058 0.0013200000000001 1 0.4836 0.5164 0.99292 0.00707999999999986 0.99998 1.9999999999909e-05
0.8 0.9966 0.68 0.3166 0.98455 0.0120500000000001 0.99527 0.00133000000000005 1 0.51394 0.48606 0.9932 0.00680000000000014 1 0
0.85 0.9976 0.67841 0.31919 0.98821 0.00939000000000012 0.99764 -3.9999999999929e-05 1 0.51868 0.48132 0.99311 0.00688999999999995 0.99998 1.9999999999909e-05
0.9 0.9992 0.67944 0.31976 0.99172 0.00747999999999982 0.99899 0.000210000000000155 1 0.52071 0.47929 0.99346 0.00653999999999999 0.99999 9.99999999995449e-06
0.95 1 0.67743 0.32257 0.99381 0.00619000000000014 0.99979 0.000210000000000043 1 0.52147 0.47853 0.9934 0.00660000000000005 0.99999 9.99999999995449e-06
1 1 0.68016 0.31984 0.99331 0.00668999999999986 0.99997 2.99999999999745e-05 1 0.52336 0.47664 0.99367 0.00632999999999995 0.99998 1.9999999999909e-05
\end{filecontents*}
\def\IR{\textit{IR}\xspace}
\def\IA{\textit{IA}\xspace}
\def\defeq{\stackrel{\text{\tiny{}def}}{=}}
\newcommand{\Mdist}[2]{\left\|{#1}-{#2}\right\|}
\newcommand{\interior}[1]{%
({\kern0pt#1})^{\mathrm{o}}%
}
\newcommand{\closure}[1]{%
\text{cl}\left({#1}\right)
}
\newcommand{\sumM}[1]{%
%\text{sum}\left({#1}\right)
\sum{#1}}
%\makeatletter
%\newcommand{\citep}[2][]{%
% \@ifnextchar[{(\cite[#1]{#2})}{(\cite{#2})}%
%}
%\makeatother
\raggedbottom
%%\unnumbered% uncomment this for unnumbered level heads
\begin{document}
\title{Significativity Indices for Agreement Values}
\author{Alberto Casagrande}
\affil{Dept.~of Mathematics, Computer Science, and\\Physics\\ University of Udine}
\author{Francesco Fabris}
\affil{Dept~of Mathematics, Computer Science, and\\Geosciences\\ University of Trieste}
\author{Rossano Girometti}
\affil{Dept.~of Medicine\\ University of Udine}
\author{Roberto Pagliarini}
\affil{Dept.~of Mathematics, Computer Science, and\\Physics\\ University of Udine}
\maketitle
\begin{abstract}
Agreement measures, such as Cohen's kappa or intraclass correlation, gauge
the matching between two or more classifiers. They are used in a wide range of contexts from medicine, where they evaluate the effectiveness of medical treatments and clinical trials, to artificial intelligence, where they can quantify the approximation due to the reduction of a classifier.
The consistency of different classifiers to a golden standard can be compared simply by using the order induced by their agreement measure with respect to the golden standard itself. Nevertheless, labelling an approach as good or bad exclusively by using the value of an agreement measure requires a scale or a significativity index. Some quality scales have been proposed in the literature for Cohen's kappa, but they are mainly na\"ive, and their boundaries are arbitrary.
This work proposes a general approach to evaluate the significativity of any agreement value between two classifiers and introduces two significativity indices: one dealing with finite data sets, the other one handling classification probability distributions. Moreover, this manuscript considers the computational issues of evaluating such indices and identifies some efficient algorithms to evaluate them.
\end{abstract}
\section{Introduction}
%In healthcare domain, it is often needed that units are rated on a scale by human observers. As an example, it is possible to consider the construction of diagnostic models to analyze the cancer progression. The importance of a model reliability lies in the fact that it represents the extent to which the data collected in the study are correct representations of the variables measured. Measurement of the extent to which models assign the same score to the same variable is known as interrater reliability~\cite{McHugh:2012}.
%\textit{Kappa coefficients} are commonly used to quantify reliability on a categorical scale, while \textit{correlation coefficients} are usually applied to assess the reliability on an interval scale. Both types of coefficients can be employed to assess the reliability of ordinal rating scales~\cite{deRaadt2021ACO}.
%Considering the kappa coefficients, Cohen's $\kappa$~\cite{Cohen:1960} is the most famous and used agreement measure on a nominal unordered scale, while the weighted $\kappa$ coefficient is widely used for quantifying agreement between rating on an ordinal scale \cite{Cohen1968WeightedKN}. However, for many real-world data the $\kappa$ coefficients have several limitations \cite{Crewson2005,Maclure87,Vanbelle2016ANI,Warrens2012SomePR,Warrens2013ConditionalIB}. To overcome these drawbacks, Shannon-oriented agreement measures, such as the \textit{Information Agreement} (\textit{IA}) \cite{Casagrande:2020a} and the \textit{Information Ratio} (\IR)~\cite{Casagrande:2022, Casagrande:2024aa,Girometti:2015}, have been developed.
%One of the main drawbacks is that there is usually more interest in the magnitude of the agreement measures than in their statistical significance.
Classifiers are processes that label entries in a data set. They may be fully automated, such as algorithms (e.g., see~\cite{726791,7780459}), or require human activities, such as in clinical evaluations (e.g., see~\cite{PIRADS2,birads2013}). The ideal (or perfect) classifier is the one that correctly labels the entries according to the \emph{golden standard}: a labelling that represents the highest and unquestionable knowledge about the domain. In machine learning, the golden standard corresponds to training and test data set labelling. In the clinical context, it is the definition of the investigated disease or condition which may correspond to the result of a clinical exam; for example, hypoglycemia is diagnosed when the result of the fasting blood glucose test is below \SI{70}{\milli\gram\per\deci\liter}~\cite{Elamin2024}.
When resources are limited, the ideal classifier may be non-feasible, and alternatives must be considered. For instance, PET-CT (\emph{Positron Emission Tomography-Computed Tomography}) is widely used to detect, stage, and monitor various types of cancer (e.g., see~\cite{MA2025110852}), but it is not suggested in the screening of low-risk subjects due to its costs and drawbacks. Analogously, quantised neural networks are valid alternatives to their plain counterparts to gain efficiency or even low-resource hardware evaluability at the price of some accuracy~\cite{hinton2015distillingknowledgeneuralnetwork}.
Agreement measures have been introduced since the XX century to measure differences among classifiers.
Some of them gauge the agreement between pairs of classifiers, such as log odds ratio~\cite{dda9a3b2-7e66-38cd-8ce8-7d36cd60b878}, McNemar’s test~\cite{McNemar:1947}, Cohen’s kappa~\cite{Cohen:1960} and its multiclass generalization~\cite{Cohen:1968}, intraclass correlation~\cite{Shrout:1979}, and information agreement ($\IA$)~\cite{Casagrande:2020a, Casagrande:2020b, Casagrande:2022, Casagrande:2023}. Others deal with sets of classifiers such as Fleiss's kappa~\cite{Fleiss71}, the adjusted rand index~\cite{hubert1985comparing}, the consensus clustering~\cite{monti2003consensus}, and Krippendorff's alpha~\cite{krippendorff2004reliability}. Most of the agreement measures report the agreement values in the real interval $[-1,1]$ where $-1$ means total disagreement, while $1$ is associated with complete agreement. Information agreement instead rates the agreement in the interval $[0,1]$ because this measure quantifies the information exchanged by two classifiers during the classification process~\cite{Casagrande:2022}.
Agreement measures are usually used to order the performances of different classifiers with respect to the perfect one. If the agreement of a classifier $C_1$ with the ideal one is higher than that of the classifier $C_2$, then $C_1$ is preferable to $C_2$ in terms of performance. However, the meaningfulness of the agreement values by themselves, i.e., the values reported by the agreement measures, is not easily interpretable, and their significativity may be obscure. How much do two classifiers with Cohen's kappa $0.7$ agree? Is $0.7$ a significant value?
In order to address such questions, Landies and Koch proposed a linear scale to interpret the strength of the agreement based on Cohen's $\kappa$: $[0, 0.2)$ (``none
to slight''), $[0.2, 0.4)$ (``fair''), $[0.4, 0.6)$ (``moderate''),
$[0.6, 0.8)$ (``substantial'') and $[0.8, 1.0)$ (``perfect or almost perfect agreement'')~\cite{Landies1977}.
%However, this interpretation allows for very little agreement among raters to be described as ``substantial''.
This scale considers the $0.61$ agreement to be ``substantial''. A different scale suggests that values greater than $0.75$ represent excellent agreement beyond chance, values in the interval $[0.40 , 0.75]$ correspond to fair to good agreement, and values below $0.40$ are poor agreement~\cite{fleiss2013statistical}.
Therefore, these scales are either missing or, in the best case, totally arbitrary.
This work deals with agreements among pairs of classifiers.
In such cases, the joint behaviour of two classifiers can be summarised by
a \emph{confusion matrix} or a \emph{probability matrix}. The cell $(i,j)$
in the former kind of matrices stores the number of entries in the data set that are labelled
as belonging to the $i$-th class by the first classifier and to the $j$-th class by
the second classifier. Instead, the cell $(i,j)$
in a probability matrix contains the joint probability for an entry to be labelled as
belonging to the $i$-th class and, at the same time, to the $j$-class by the first and the
second classifiers, respectively.
Thus, any agreement measure is a function from the set of confusion or probability matrices to the set of agreement values.
In this context, we introduce two significativity indices for agreement values:
one on the confusion matrices and the other one on the probability matrices.
Both of them define the significativity of
an agreement value obtained over a data set as
the probability to randomly select a matrix built
over the same data set with a lower agreement value.
%This induces a parametric scale on the considered agreement measure and relates any value to its probability.
%For example, if $75\%$ of the classifier pairs have a Cohen's $\kappa$ of lower than $0.281567$, the significativity of $0.281567$ with respect to Cohen's $\kappa$ will be $0.75$.
%and placed at one fourth of the $\kappa$ significativity scale.
The more likely it is to choose a matrix with a lower agreement value, the higher the
significativity of the agreement.
From another point of view, the more difficult it is to find two classifiers whose agreement is
at least equivalent to the investigated value, the higher its significativity.
The proposed indices are parametric in both the investigated agreement measure and
the number of classes. The index on the confusion matrices also depends on the size of the original data set.
This approach is analogous to the classical statistical coefficient $p$-value~\cite{fisher1925statistical}, which measures the probability for the null hypothesis to comply with the data. As long as the $p$-value is under a selected threshold -- usually $0.05$ -- the null hypothesis is discarded. Our method does not set any threshold, leaving the task of identifying one to the user. Instead, it evaluates the probability for a random matrix to have an agreement value lower than the considered one to measure its significativity.
The proposed indices do not gauge the meaningfulness of the original data set, but exclusively deal with the agreement values. The same agreement value can be obtained from two data sets with substantially different cardinalities, and the agreement value of a confusion matrix may have a high significativity even though the data set used to build it consists of a few entries. Thus, we can deduce that agreement and data set significativities are not directly related.
Our method has three main advantages. % over those available in the literature.
First, it associates any agreement measure, even those whose meaning is obfuscated,
with a clear and consolidated index: the probability of decreasing the agreement value by chance.
Second, the induced scale is not arbitrary, as it reports the probabilities of the agreement values, and it deals with objective quantities. Finally, this method may help compare agreement measures, providing a familiar and unifying approach.
%The resulting scales are not equivalent because they are parametric on the original agreement measures, which establish the quality order among the matrices.
This work is organised as follows: Section~\ref{sec:notions}
introduces the basic notions and notation.
Section~\ref{sec:significativity_confusion} defines the notion of
$\sigma$-significativity over confusion matrices, where $\sigma$ is any agreement
measure between two classifiers, it studies the asymptotic time complexity
of computing this value, and it proposes a time-feasible
numeric estimator for it.
Section~\ref{sec:significativity_probability} introduces the $\sigma$-significativity over
probability matrices and proposes an efficient algorithm to numerically
estimate it. The same section also
proves that, under some reasonable conditions on the syntactic form
of $\sigma$, the $\sigma$-significativity over confusion
matrices converges to the $\sigma$-significativity over probability matrices
as the size of the confusion matrix data set tends to infinity.
Finally, Section~\ref{sec:conclusions} summarises the achieved results, contains the
concluding remarks,
and suggests some possible future developments.
\section{Basic Notions and Notation}\label{sec:notions}
For any set $S$ and for any pair of positive natural values $n, m \in \mathbb{N}_{>0}$, the set of $n \times m$ matrices with elements in $S$ is denoted by $S^{n \times m}$. If $M$ is a matrix, then $M(i,j)$ is
$M$'s element in the $i$-th row and the $j$-th column. We may write $\sumM{M}$ meaning the sum of the
elements in $M$, i.e., when $M \in S^{n \times m}$, $\sumM{M}\defeq\sum_{i=1}^n\sum_{j=1}^m M(i,j)$.
If $f: A \rightarrow B$ and $C \subseteq A$, we may write $f(C)$ meaning the image set of $C$, i.e., $f(C)=\{f(c)\,|\, c \in C\}$.
For any set $S$, the \emph{indicator function of $S$}, %$\mathbf{1}_{\mathcal{S}_{\sigma}(c)}(x): \mathbb{R}^{n^2} \rightarrow \{0,1\}$,
\begin{equation}
\mathbf{1}_{S}(x) \defeq
\begin{cases}
0 & \text{if $x \not \in S$}\\
1 & \text{if $x \in S$},
\end{cases}
%\begin{cases}
%0 & \text{if $x \not \in \Delta^{(n^2-1)} \lor \sigma(\gamma(x)) < c$}\\
%1 & \text{if $x \in \Delta^{(n^2-1)} \land \sigma(\gamma(x)) < c$},
%\end{cases}
\end{equation}
maps the elements of $S$ to $1$ and all the other elements to $0$.
A \emph{classifier} is a process or a rater that partitions the investigated domain $\mathcal{D}$ into
$n$ distinct classes. It may be implemented as a digital system (e.g., AI models), a medical exam (e.g., COVID-19 test), or
a clinical evaluation (e.g., BI-RADS or PI-RADS).
Any classifier corresponds to a function $\chi: \mathcal{D} \rightarrow [1,n]$. When $\chi(d)=i$, we say that \emph{$d$ is in the class $i$ according to $\chi$}.
Two classifiers $\chi_1$ and $\chi_2$ can be compared by evaluating $m \in \mathbb{N}$ distinct elements in $\mathcal{D}$.
The result is the evaluations can be collected in a \emph{$n\times n$ confusion matrix} consisting of $m$ tests that is a
$n \times n$ matrix $M_C \in \mathbb{N}^{n\times n}$ whose value in position $(i, j)$ reports the number of the $m$ elements which are in the
classes $i$ and $j$ according to $\chi_1$ and $\chi_2$, respectively, i.e., $M_C(i,j)=|\{d \in \mathcal{D}\,|\, \chi_1(d)=i \land \chi_2(d)=j\}|$.
For any $n\times n$ confusion matrix, $M_C$, built from a data set of size $m$, the sum of its values is $m$, i.e., %$m=\sum_{i=1}^n\sum_{j=1}^n M_C(i,j)$.
$m=\sumM{M_C}$.
The set of all the $n\times n$ confusion matrices built over $m$ tests is denoted by $\mathcal{M}_{n, m}$, i.e.,
%\begin{equation}
%\mathcal{M}_{n, m} \defeq \left\{\begin{pmatrix}c_{1}&\ldots&c_{n}\\ \vdots& \ddots&\vdots\\c_{n(n-1)+1}&\ldots& c_{n^2}\end{pmatrix} \in \mathbb{N}^{n \times n}\, |\, \sum_{i=1}^{n^2} c_i = m \right\}.
%\end{equation}
%
\begin{equation}
\mathcal{M}_{n, m} \defeq \left\{M_C \in \mathbb{N}^{n \times n}\, \big|\, \sumM{M_C} = m \right\}.
\end{equation}
A \emph{$n\times n$ probability matrix} is a real-valued matrix representing the joint probability distribution of a pair of
independent and discrete random variables ranging in the interval $[1,n]$.
The value in position $(i,j)$ is the probability that the first random variable returns $i$ and, at the same time, the second returns $j$.
Since every probability matrix $M_P$ is a discrete probability distribution, its values are non-negative and sum up to $1$, i.e.,
%$1=\sum_{i=1}^{n} \sum_{j=1}^{n} M_P(i,j)$
$\sumM{M_P}=1$
and $M_P(i,j) \geq 0$ for all $i \in [1,n]$ and for all $j \in [1,n]$.
The set of all the $n\times n$ probability matrices, $\mathcal{P}_n$, is defined as follows
%\begin{equation}
%\mathcal{P}_n \defeq \left\{\begin{pmatrix}p_{1}&\ldots&p_{n}\\ \vdots& \ddots&\vdots\\p_{n(n-1)+1}&\ldots& p_{n^2}\end{pmatrix} \in \mathbb{R}^{n \times n}\, |\, \sum_{i=1}^{n^2} p_i = 1 \land \bigwedge_{i = 1}^{n^2} p_i \geq 0 \right\}.
%\end{equation}
\begin{equation}
\mathcal{P}_n \defeq \left\{M_P \in \mathbb{R}_{\geq 0}^{n \times n}\, \bigg|\, \sumM{M_P} = 1 \right\}.
\end{equation}
Every confusion matrix $M_C \in \mathcal{M}_{n,m}$ induces a probability matrix $M_P \in \mathcal{P}_n$ by means of the function $T_P$ defined as follows
%\begin{equation}\label{def:T_P}
%T_P\left(\begin{pmatrix}c_{1}&\ldots&c_{n}\\ \vdots& \ddots&\vdots\\c_{n(n-1)+1}&\ldots& c_{n^2}\end{pmatrix}\right)
%\defeq \frac{1}{\sum_{i=1}^{n^2}c_{i}} \begin{pmatrix}c_{1}&\ldots&c_{n}\\ \vdots& \ddots&\vdots\\c_{n(n-1)+1}&\ldots& c_{n^2}\end{pmatrix}.
%\end{equation}
\begin{equation}\label{def:T_P}
T_P\left(M_C\right) \defeq \frac{1}{\sumM{M_C}}M_C.
\end{equation}
A \emph{agreement measure} is a function $\sigma: \mathcal{P}_n \cup \bigcup_{m \in \mathbb{N}} \mathcal{M}_{n, m} \rightarrow I_{\sigma}$, where $I_{\sigma}$ is an interval over $\mathbb{R}$,
meant to
measure the agreement between two classifiers or the effectiveness of a classifier with respect to a golden standard
using either a confusion or a probability matrix. Cohen's $\kappa$~\cite{Cohen:1960}, Scott's $\pi$~\cite{10.1086/266577}, Yule's $Y$~\cite{Yule12},
Fleiss's $\kappa$~\cite{Fleiss71}, and %$\IR$~\cite{Girometti:2015}, and
$\IA$~\cite{Casagrande:2020a,Casagrande:2023,Casagrande:2024aa} are agreement measures.
Later on, we will assume agreement-ordered agreement measures, meaning that greater agreement between the classifiers will correspond to
greater agreement values. This condition can be easily relaxed when the lower values correspond to large agreement, for instance by considering the opposite of the investigated agreement measure.
Nevertheless, this assumption simplifies the presentation of the formal aspects and the notation.
\subsection{O-Minimal Theories and Definability}~\label{sec:ominimal}
Section~\ref{sec:significativity_probability} relates some of the properties of the
investigated agreement measures and the language used to define them.
Because of this, we need to introduce the notion of theory as a syntactic characterisation for
sets.
A \emph{theory} is a set of first-order formulas that describe a class of structures. Any theory
is defined by the variable domain, a set of constants, a set of functional symbols, and a
set relational symbols. When the variables in the theory assume values in the set $\mathcal{Q}$,
we say that the theory is over $\mathcal{Q}$.
The Presburger arithmetic theory, $(\mathbb{N}, \{0, 1\}, +, >)$, describes the properties of the natural numbers without the multiplication~\cite{presburger1929},
the Zermelo-Fraenkel set theory is the set of the formulas defining the notion of set, and Tarski's theory~\cite{Fraenkel1958}, also known as semi-algebraic theory
over the reals,
$(\mathbb{R}, \{0, 1\}, +, *, >)$, is the set of the first-order formulas whose expressions are polynomials with integer coefficients~\cite{Tarski51}.
A set $S$ is \emph{definable} in a theory $\mathcal{T}$ if there exists a formula
$\psi(x) \in \mathcal{T}$ such that $S=\{x\, |\, \psi(x)\}$.
\begin{example}
Let us consider the set $S$ of the pairs $\langle x, y \rangle \in \mathbb{R}^2$ such that $1/(x*y)>1$, i.e., $S=\{\langle x, y \rangle \in \mathbb{R}^2\,|\, 1/(x*y) > 1\}$.
%
%%
%\begin{figure}
%\begin{center}
%\resizebox{0.45\textwidth}{!}{
%\begin{tikzpicture}
% \begin{axis}[
% axis lines=middle,
% %restrict y to domain=-2.5:2.5,
% %restrict x to domain=-2.5:2.5,
% xlabel=$x$, ylabel=$y$,
% xtick={1, 2, 3},
% extra x ticks={-3, -2, -1},
% extra x tick style={
% xticklabel style={yshift=0.2em, anchor=near xticklabel opposite}
% },
% ytick={10, 5},
% extra y ticks={-5, -10},
% extra y tick style={
% yticklabel style={xshift=0.2em, anchor=near yticklabel opposite}
% },
% %xtick distance=1, ytick distance=1,
% grid=both, grid style={line width=.1pt, draw=gray!10},
% major grid style={line width=.2pt,draw=gray!50},
% xlabel style={at={(axis description cs:1,0.5)}, anchor=west},
% ylabel style={at={(axis description cs:0.5,1)}, anchor=south},
% clip=false
% ]
%
% % Plot the region where 1/(xy) > 1, which is equivalent to xy < 1 (excluding x=0 and y=0)
% \addplot[name path=L, draw=none, domain=-3:-0.1, samples=200] {1/x};
% \addplot[name path=R, draw=none, domain=0.1:3, samples=200] {1/x};
%
% \addplot[name path=x_axis, draw=none, domain=-3:3, samples=400] {0};
%
% \addplot[name path=min_y, draw=none, domain=-0.1:0] {-10};
% \addplot[name path=max_y, draw=none, domain=0:0.1] {10};
%
% \addplot[draw=none,fill=blue!80, opacity=0.5, samples=200]fill between[of=L and x_axis, soft clip={domain=-3:-0.1}];
% \addplot[draw=none,fill=blue!80, opacity=0.5, samples=200]fill between[of=R and x_axis, soft clip={domain=0.1:3}];
%
% \addplot[draw=none,fill=blue!80, opacity=0.5, samples=200]fill between[of=x_axis and min_y, soft clip={domain=-0.1:0}];
% \addplot[draw=none,fill=blue!80, opacity=0.5, samples=200]fill between[of=x_axis and max_y, soft clip={domain=0:0.1}];
%
%\end{axis}
%\end{tikzpicture}}
%\end{center}
%\caption{A graphical representation of the set $S=\{\langle x, y \rangle \in \mathbb{R}^2\,|\, 1/(x*y) > 1\}$. Although the formula $1/(x*y)>1$ does not belong to Tarski's theory because of the division, $S$ is definable in the same theory.}\label{fig:set_definable}
%\end{figure}
%
The formula $1/(x*y)>1$ does not belong to Tarski's theory because the left expression includes a division, which is not one of the $(\mathbb{R}, \{0, 1\}, +, *, >)$'s
functions.
However, the formulas $(y > 0 \land x > 0 \land 1>x*y) \lor (0 > y \land 0 > x \land 1>x*y)$ and $1/(x*y)>1$ are equivalent on $\mathbb{R}$
and the former belongs to $(\mathbb{R}, \{0, 1\}, +, *, >)$. Thus, the set $S$ is definable in Tarski's theory.
\end{example}
%
A theory $\mathcal{T}$ over $\mathcal{Q}$ is \emph{o-minimal} if any set
$S \subseteq \mathcal{Q}$, definable $\mathcal{T}$, in is the finite union of open intervals and points~\cite{Dries_1998}.
Tarski's theory and its Pfaffian extensions~\cite{speissegger1999pfaffian}, e.g., $(\mathbb{R}, \{0,1\}, +, $ $*,e^{x}, >)$ or $(\mathbb{R}, \{0,1\}, +, $ $*,\ln{x}, >)$,
are o-minimal theories.
%
An \emph{o-minimal set} is a set definable in some o-minimal theory.
%\textbf{
%We must describe:
%\begin{itemize}
%\item confusion matrices $M_C=\begin{pmatrix}c_{1}&\ldots&c_{n}\\ \vdots& \ddots&\vdots\\c_{n(n-1)+1}&\ldots& c_{n^2}\end{pmatrix}$.
%\item the set $\mathcal{M}_{n, m}$ of the $n\times n$-confusion matrices $\mathbb{N}^{n\times n}$.
%\item the set of the $n\times n$-probability matrices
%$$
%\mathcal{P}_n \defeq \left\{\begin{pmatrix}p_{1}&\ldots&p_{n}\\ \vdots& \ddots&\vdots\\p_{n(n-1)+1}&\ldots& p_{n^2}\end{pmatrix} \in \mathbb{R}^{n \times n}\, |\, \sum_{i \in [1,n^2]} p_i = 1 \land \bigwedge_{i \in [1, n^2]} p_i \geq 0 \right\}
%$$
%\item The function $T_{P} = \bigcup_{n \in \mathbb{N}}T_{P,n}: \mathbb{N}^{n\times n} \rightarrow \mathcal{P}_n$ that maps any confusion matrix into a probability matrix
%$$
%\begin{pmatrix}p_{1}&\ldots&p_{n}\\ \vdots& \ddots&\vdots\\p_{n(n-1)+1}&\ldots& p_{n^2}\end{pmatrix} = T_P\left(\begin{pmatrix}c_{1}&\ldots&c_{n}\\ \vdots& \ddots&\vdots\\c_{n(n-1)+1}&\ldots& c_{n^2}\end{pmatrix}\right)
%$$
%where
%\begin{equation}\label{def:T_P}
%T_P\left(\begin{pmatrix}c_{1}&\ldots&c_{n}\\ \vdots& \ddots&\vdots\\c_{n(n-1)+1}&\ldots& c_{n^2}\end{pmatrix}\right)
%\defeq \frac{1}{\sum_{i \in [1,n^2]}c_{i}} \begin{pmatrix}c_{1}&\ldots&c_{n}\\ \vdots& \ddots&\vdots\\c_{n(n-1)+1}&\ldots& c_{n^2}\end{pmatrix}.
%\end{equation}
%\item $T_P$ is injective. Hence, $T_P: \mathcal{M}_{n,m} \rightarrow \mathcal{P}_{n,m}$ where $\mathcal{P}_{n,m} \defeq T_P(\mathcal{M}_{n,m})$ is bijective.
%\item agreement measure as a function $\sigma: \mathbb{N}^{n \times n} \cup \mathcal{P}_n \rightarrow \mathbb{R}$ from the confusion and probability matrices to the reals
%that estimate the quality of the corresponding test or the agreement levels of different tests.
%\item $\sigma(\cdot)$ must be such that the set
%\begin{equation*}
%\mathcal{S}_{\sigma,n}(c) \defeq \left\{x \in \Delta^{(n^2-1)} \,|\, \sigma(\gamma(x)) < c \right\}
%\end{equation*}
%is Lebesgue-measurable.
%This is the case, for instance, when $\sigma(\cdot)$ is definable in an o-minimal structure (e.g., see~\cite{Fornasiero_Rifo_2012}) such as $(\mathbb{R}, \{0,1\}, +, $ $*, >)$ and
%its Pfaffian extensions~\cite{speissegger1999pfaffian}. Cohen's $\kappa$, $\IR$, $\IA$, and all the rational functions are definable in
%o-minimal structures.
%\item quality-ordered agreement measure, i.e., if the test associated with the confusion matrix $M_1$ is better than of matrix $M_2$, then $\sigma(M_2) < \sigma(M_1)$
%(extendable to $\sigma(M)$ such that $\sigma(M_1) < \sigma(M_2)$).
%\end{itemize}}
%In the following section, we introduce an approach to define a scale for any agreement measure such that $\sigma(M_C)=\sigma(T_P(M_C))$.
\section{Significativity over Confusion Matrices}\label{sec:significativity_confusion}
Let $\sigma$ be a quality-ordered agreement measure for $n\times n$ matrices.
The ratio between the number of confusion matrices in $\mathcal{M}_{n,m}$ to be such that $\sigma(M) < c$
and the total number of confusion matrices in $\mathcal{M}_{n,m}$ is
\begin{equation}\label{eq:varrho}
\varrho_{\sigma,n,m}(c) \defeq \frac{\left|\left\{M \in \mathcal{M}_{n,m}\, |\, \sigma(M) < c \right\}\right|}{\left|\mathcal{M}_{n,m}\right|}.
\end{equation}
This value belongs to the real interval $[0,1]$ and reports how many of all the $n \times n$ confusion matrices consisting of $m$ tests have
an agreement value less than $c$. From a probabilistic point of view, $\varrho_{\sigma,n,m}(c)$ is also
the probability of selecting by chance over a uniform distribution a $n\times n$-confusion matrix of $m$ tests whose agreement value
is less than $c$. Because of this, we say that $\varrho_{\sigma,n,m}(c)$ is
\emph{the $\sigma$-significativity of $c$ in $\mathcal{M}_{n,m}$}.
The reader must be aware that the $\sigma$-significativity does not rate the confusion matrices, which is what the agreement measure $\sigma$ deals with.
Instead, it provides a significativity measure for the agreement values.
\begin{figure*}[!ht]
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[xmin=-1,xmax=1,
ymin=0,ymax=1,
xlabel={$c$},
ylabel={$\kappa$-Significativity},
legend pos=north west,
legend style={cells={align=left}}]
\addplot table [x=kappa,y=R_kappa_2_2,col sep=space] {kappa_2.csv};
\addlegendentry{$\varrho_{\kappa,2,2}(c)$}
\addplot table [x=kappa,y=R_kappa_2_4,col sep=space] {kappa_2.csv};
\addlegendentry{$\varrho_{\kappa,2,4}(c)$}
\addplot table [x=kappa,y=R_kappa_2_16,col sep=space] {kappa_2.csv};
\addlegendentry{$\varrho_{\kappa,2,16}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The function $\varrho_{\kappa,2,m}(c)$.}
\end{subfigure}
%\hfill
% \begin{subfigure}[b]{0.31\textwidth}
%\resizebox{\textwidth}{!}{
%\begin{tikzpicture}
% \begin{axis}[xmin=0,xmax=1,
% ymin=0,ymax=1,
% xlabel={$x$},
% ylabel={\% of $2\times 2$-matrices (Probability)},
% legend pos=south east,
% legend style={cells={align=left}}]
% \addplot table [x=ir,y=p,col sep=space] {IR_rho.csv};
% \addlegendentry{$\rho_{\IR}(x)$}
%\end{axis}
%\end{tikzpicture}}
%\caption{The function $\rho_{\IR}(x)$.}
%\end{subfigure}
\hfill
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[xmin=-1,xmax=1,
ymin=0,ymax=1,
xlabel={$c$},
ylabel={$\kappa$-Significativity},
legend pos=north west,
legend style={cells={align=left}}]
\addplot table [x=kappa,y=R_kappa_3_2,col sep=space] {kappa_3.csv};
\addlegendentry{$\varrho_{\kappa,3,2}(c)$}
\addplot table [x=kappa,y=R_kappa_3_4,col sep=space] {kappa_3.csv};
\addlegendentry{$\varrho_{\kappa,3,4}(c)$}
\addplot table [x=kappa,y=R_kappa_3_8,col sep=space] {kappa_3.csv};
\addlegendentry{$\varrho_{\kappa,3,8}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The function $\varrho_{\kappa,3,m}(c)$.}
\end{subfigure}
\caption{The functions $\varrho_{\kappa,n,m}(c)$ for $n=2$ and $n=3$ as the number of tests $m$ changes.}\label{fig:varrho-kappa}
\end{figure*}
%
%\begin{algorithm}
%\caption{An iterator for the set $\mathcal{C}_{m,k}$. \textbf{Complexity $\Theta\left(k\binom{m+k-1}{m}\right)$}?}\label{alg:wc_iterator}
%\begin{algorithmic}[1]
%\Require $m \in \mathbb{N}$, $k \in \mathbb{N}_{>0}$, and $x \in \mathbb{N}^i$ for some $i \geq k$.
%When $x$ is not provided, it is initialised to $\langle 0, \ldots, 0 \rangle\in \mathbb{N}^k$.
%\Ensure To yield all the weak compositions of $m$ in $k$ parts.
%\Procedure{WeakCompositions}{$m, k, x \gets \langle 0, \ldots, 0 \rangle$}
%\If{$k=1$}
%\State $x[k] \gets m$
%\State \textbf{yield} $x$
%\Else
%\For{$v \gets 0, \dots, m$}
%\State $x[k] \gets m$
%\For{$y \gets $\Call{WeakCompositions}{$m-v, k-1, x$}}
%\State \textbf{yield} $y$
%\EndFor
%\EndFor
%\EndIf
%\EndProcedure
%\end{algorithmic}
%\end{algorithm}
%\subsection{Computing the Significativity}~\label{sec:significativity_computation}
A \emph{weak composition of $m$ in $k$ parts} is a tuple $\langle x_1, \ldots, x_k \rangle \in \mathbb{N}^{k}$ such that
$\sum_{i=1}^{k} x_i = m$~\cite{bona2016}. The set $\mathcal{C}_{m,k}$ that contains all of them, i.e.,
\begin{equation}\label{eq:compositions}
\mathcal{C}_{m,k} \defeq \left\{\langle x_1, \ldots, x_k \rangle \in \mathbb{N}^{k} \,|\, \sum_{i=1}^{k} x_i = m\right\}
\end{equation}
has cardinality $\binom{m+k-1}{m}$~\cite[Theorem~5.2]{bona2016}.
Let $\gamma: \mathbb{R}^{n^2} \rightarrow \mathbb{R}^{n \times n}$ be defined as
\begin{equation}
\gamma(\langle x_1, \ldots, x_{n^2} \rangle) \defeq \begin{pmatrix}
x_1 & \ldots & x_n\\
x_{n+1} & \ldots & x_{2n}\\
\vdots & \ddots & \vdots&\\
x_{n(n^2-1)+1} & \ldots & x_{n^2}
\end{pmatrix}.
\end{equation}
It is easy to see that $\gamma$ is bijective and maps any weak composition of $m$ in $n^2$ parts
into a $n\times n$ confusion matrix over $m$ tests, i.e., $\mathcal{M}_{n,m} = \gamma\left(\mathcal{C}_{m,n^2}\right)$ and
$\mathcal{C}_{m,n^2} = \gamma^{-1}\left(\mathcal{M}_{n,m}\right)$.
%and $\left|\mathcal{M}_{n,m}\right|$
%equals $\left|\mathcal{C}_{n^2,m}\right|$ which is $\binom{m+n^2-1}{m}$~\cite[Theorem~5.2]{bona2016}.
Hence, the sets $\left\{M \in \mathcal{M}_{n,m}\, |\, \sigma(M) < c \right\}$ and
\begin{equation}
\mathcal{R}_{\sigma,n,m}(c)\defeq \left\{x \in \mathcal{C}_{m,n^2}\, |\, \sigma(\gamma(x)) < c \right\}
\end{equation}
have the same cardinality. Moreover, $|\mathcal{M}_{n,m}|=|\mathcal{C}_{m,n^2}|$ because
$\gamma$ is bijective.
Thus, $\varrho_{\sigma,n,m}(c) = |\mathcal{R}_{\sigma,n,m}(c)|/|\mathcal{C}_{m,n^2}|$.
The cardinality of $\mathcal{C}_{m,n^2}$ is
$\binom{m+n^2-1}{m}$~\cite[Theorem~5.2]{bona2016}
%If $\left|\mathcal{C}_{m,n^2}\right| = \binom{m+n^2-1}{m}$ is reasonably small,
and we can compute $\left|\mathcal{R}_{\sigma,n,m}(c)\right|$ by iterating over
all the elements in $\mathcal{C}_{m,n^2}$ %(for instance, by using Algorithm~\ref{alg:wc_iterator})
and summing up their images through the indicator function of $\mathcal{R}_{\sigma,n,m}(c)$, i.e.,
$\left|\mathcal{R}_{\sigma,n,m}(c)\right|= \sum_{x \in \mathcal{C}_{m,n^2}} \mathbf{1}_{\mathcal{R}_{\sigma,n,m}(c)}(x)$
(see Algorithm~\ref{alg:R_computation}).
%This is the case in clinical studies for dichotomous rare disease tests involving up to a few tens of tests, i.e., $n=2$ and $m<50$.
\begin{algorithm}
\caption{An algorithm to compute $|\mathcal{R}_{\sigma,n,m}(c)|$ in time $T(\sigma \circ \gamma)*\Theta\left(\left|\mathcal{C}_{m, n^2}\right|\right)$.
%The logarithmic term %$\Theta\left(\log_2{\binom{m+k-1}{m}}\right)$
%is due to the logarithmic cost criterion.
}\label{alg:R_computation}
\small
\begin{algorithmic}[1]
\Require $m \in \mathbb{N}$, $n \in \mathbb{N}_{>0}$, $c \in \mathbb{R}$, and $\sigma: \mathcal{M}_{n,m} \rightarrow \mathbb{R}$.
\Ensure Returns $|\mathcal{R}_{\sigma,n,m}(c)|$.
\Function{aux\_C}{$\sigma, m, k, c, x$}\Comment{An auxiliary function}
\If{$k=1$}
\State $x[k] \gets m$\Comment{$x$ is now belong to $\mathcal{C}_{m, n^2}$}
\State \Return $(1 \textbf{ if } \sigma(\gamma(x))< c \textbf{ else } 0)$%\Comment{Evaluate $\mathbf{1}_{\mathcal{R}_{\sigma,n,m}(c)}(\gamma(x))$}
\EndIf
\State $c \gets 0$%\Comment{$c$ is a counter for the elements in $\mathcal{R}_{\sigma,n,m}(c)$}
\For{$v \gets 0, \dots, m$}
\State $x[k] \gets v$
\State $c \gets counter+$\Call{aux\_C}{$\sigma, m-v, k-1, c, x$} %\Comment{This sum costs $\Theta\left(\log_2{\binom{m+k-1}{m}}\right)$ according to the logarithmic cost criterion}
\EndFor
\State \Return $c$
\EndFunction
\Statex
\Function{R\_Cardinality}{$\sigma, n, m, c$}
\State $x \gets $\Call{array}{$n^2, 0$} \Comment{$x$ is initialized to $\vec{0}\in \mathbb{N}^{n^2}$}
\State \Call{aux\_C}{$\sigma, m, n^2, c, x$}
\EndFunction
\end{algorithmic}
\end{algorithm}
The asymptotic time cost~\cite{cormen2022introduction} of evaluating the function $\gamma$ is $O(n^2)$.
Thus, the complexity of the presented
approach to compute $\varrho_{\sigma,n,m}(c)$ is $(T(\sigma) + O(n^2))*\Theta(\binom{m+n^2-1}{m})$, where
$T(f)$ is the time complexity of the function $f$.
It follows that the complexity of the approach is $\Theta(n^2 \binom{m+n^2-1}{m})$ when
$T(\sigma) \in \Theta(n^2)$ as in the cases of Cohen's $\kappa$ or $\IA$.
%
%\begin{example}
%Let $M_C$ be the matrix
%\[
%M_C=\begin{pmatrix}\num{19804}&\num{172}\\\num{1}&\num{23}\end{pmatrix}.
%\]
%Cohen's $\kappa$ and $\IA$ for $M_C$ are $\kappa(M_C)\approx 0.20835$ and $\IA(M_C)\approx 0.55963$, respectively.
%The matrix $M_C$ summarises the results of $m=\num{19804}+\num{172}+\num{1}+\num{23}=\num{20000}$ tests.
%The $\kappa$-significativity of $\kappa(M_C)$ in $\mathcal{M}_{2,\num{20000}}$ is $\varrho_{\kappa,2,\num{20000}}(\kappa(M_C))\approx 0.7259$.
%Thus, less than $73\%$ of the $2\times 2$-confusion matrices over $\num{20000}$ tests have a $\kappa$ value smaller than that of $M_C$ and
%the probability of choosing by chance a confusion matrix $M_C' \in \mathcal{M}_{2,\num{20000}}$ with $\kappa(M_C') \leq \kappa(M_C)$ is
%$0.7259$.
%
%Instead, the $\IA$-significativity of $\IA(M_C)$ in $\mathcal{M}_{2,\num{20000}}$ is $\varrho_{\IA,2,20}(\IA(M_C))\approx 0.9611$.
%Hence, more than $96\%$ of the $2\times 2$-confusion matrices over $\num{20000}$ tests have an $\IA$ value smaller than that of $M_C$ and
%the probability of choosing by chance a confusion matrix $M_C' \in \mathcal{M}_{2,\num{20000}}$ with $\IA(M_C') \leq \IA(M_C)$ is
%$0.9611$.
%\end{example}
\begin{example}\label{ex:varrho}
Let $M_C$ be the matrix
\[
M_C=\begin{pmatrix}\num{8}&\num{3}\\\num{0}&\num{9}\end{pmatrix}.
\]
Cohen's $\kappa$ and $\IA$ for $M_C$ are $\kappa(M_C)\approx 0.70588$ and $\IA(M_C)\approx 0.52115$, respectively.
The matrix $M_C$ summarises the results of $m=\num{8}+\num{3}+\num{0}+\num{9}=\num{20}$ tests.
The $\kappa$-significativity of $\kappa(M_C)$ in $\mathcal{M}_{2,\num{20}}$ is $\varrho_{\kappa,2,\num{20}}(\kappa(M_C))\approx 0.8866$.
Thus, more than $88\%$ of the $2\times 2$-confusion matrices over $\num{20}$ tests have a $\kappa$ value lower than that of $M_C$ and
the probability of choosing by chance a confusion matrix $M_C' \in \mathcal{M}_{2,\num{20}}$ with $\kappa(M_C') < \kappa(M_C)$ is
$0.8866$.
Instead, the $\IA$-significativity of $\IA(M_C)$ in $\mathcal{M}_{2,\num{20}}$ is $\varrho_{\IA,2,20}(\IA(M_C))\approx 0.7628$.
Hence, more than $76\%$ of the $2\times 2$-confusion matrices over $\num{20}$ tests have an $\IA$ value lower than that of $M_C$. Moreover,
the probability of choosing by chance a confusion matrix $M_C' \in \mathcal{M}_{2,\num{20}}$ with $\IA(M_C') < \IA(M_C)$ is
$0.2324$.
\end{example}
Figure~\ref{fig:varrho-kappa} clarifies the relation between $n$, $m$, and $\varrho_{\kappa, n, m}(c)$
for $n$ in ${2,3}$ as $m$ grows.
It is worth remarking that Cohen's $\kappa$, as many other agreement measures, assumes values in $[-1,1]$ where the negative part
of the interval is usually associated with confusion matrices exhibiting ``disagreement'' among
the classifiers. The $\kappa$-significativity does not distinguish between agreement and ``disagreement''.
Hence, when, in Example~\ref{ex:varrho}, we wrote that more than $95\%$ of the matrices in
$\mathcal{M}_{2,\num{20}}$ have a $\kappa$ value smaller than that of $M_C$, we referred to the full
set of matrices in $\mathcal{M}_{2,\num{20}}$ with no reference to sign of their $\kappa$-images.
\subsection{Numerical Evaluation}
Due of its time complexity, the approach sketched in Section~\ref{sec:significativity_confusion} fails to scale up to trials dealing
with hundreds of tests or involving non-dichotomic outcomes. For instance, an evaluation of the probability for a $2\times 2$
confusion matrix $M$ summarising $200$ test results that $\sigma(M) < c$, i.e., $\varrho_{\sigma,2,200}(c)$, iterates over
$\binom{200+2^2-1}{200} = \num{1373701}$ weak compositions, which are less than half of the $\binom{20+3^2-1}{20} = \num{3108105}$ weak compositions
required to evaluate the probability for a $3\times 3$
confusion matrix $M$ collecting $20$ test results of satisfying the same inequality, i.e., $\varrho_{\sigma,3,20}(c)$.
%For instance, $\iota_{k,m}$ can enumerate
%all the weak compositions of $m$ into $k$ in lexicographic order, i.e., if $a,b \in \left[0, \binom{m+k-1}{m}-1\right]$ and $a<b$,
%then there exists $j \in [1, k]$ such that
%$a_i = b_i$ for all $i \in [1,j-1]$ and $a_j < b_j$ where $\iota_{k,m}'(a)=\langle a_1, \ldots, a_k \rangle$ and
%$\iota_{k,m}'(b)=\langle b_1, \ldots, b_k \rangle$.
Nevertheless, $\varrho_{\sigma,n,m}(c)$ can be numerically estimated by using the Monte Carlo method~\cite{montecarlo} as
\begin{equation}\label{eq:monte_carlo_varrho}
\varrho_{\sigma,n,m}(c) \approx
%\frac{\frac{\left|\mathcal{C}_{m,n^2}\right|}{N} \sum_{i=1}^{N} \mathbf{1}_{\mathcal{R}_{\sigma,n,m}(c)}(x_i)}{\left|\mathcal{C}_{m,n^2}\right|} =
\frac{1}{N} \sum_{i=1}^{N} \mathbf{1}_{\mathcal{R}_{\sigma,n,m}(c)}(x_i)
\end{equation}
where $x_1, \ldots, x_N$ are $N$ weak compositions uniformly distributed in $\mathcal{C}_{m,n^2}$.
Since $\mathcal{C}_{m,k}$ is discrete, there is a bijective function
%$\iota_{m,k}: \left[0, \binom{m+k-1}{m}-1\right] \rightarrow \mathcal{C}_{m,k}$
$\iota_{m,k}: \left[0, \left|\mathcal{C}_{m,k}\right|-1\right] \rightarrow \mathcal{C}_{m,k}$
mapping each natural number lower than
$\left|\mathcal{C}_{m,k}\right|$ to a weak composition of $m$ into $k$ parts -- actually, there
are $\left|\mathcal{C}_{m,k}\right|!$ of them because $\mathcal{C}_{m,k}$ is finite --.
If we apply $\iota_{m,n^2}$ to uniform samples over the integers in $\left[0, \left|\mathcal{C}_{m,n^2}\right|-1\right]$,
we get uniform samples over $\mathcal{C}_{m,n^2}$, and we can approximate $\varrho_{\sigma,n,m}(c)$ as suggested by Eq.~\ref{eq:monte_carlo_varrho}
with an error proportional to $1/\sqrt{N}$ (e.g., see~{\cite{Mackay1998}).
Figure~\ref{fig:montecarlo_error} represents the average error of 100 Monte Carlo-based estimations
of $\varrho_{\kappa, n, m}(c)$ using
$\left\lceil\sqrt{\left|\mathcal{C}_{m,n^2}\right|}\right\rceil$ samples.
\begin{figure*}[!ht]
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[xmin=-1,xmax=1,
ymin=-6e-2,
xlabel={$c$},
ylabel={Average errors},
legend pos=south west,
legend style={cells={align=left}}]
%\addplot table [x=kappa,y=mean_delta_R_kappa_2_2,col sep=space] {kappa_2.csv};
%\addlegendentry{$m=2$}
\addplot table [x=kappa,y=mean_delta_R_kappa_2_4,col sep=space] {kappa_2.csv};
\addlegendentry{$m=4$ $(N=6)$}
\addplot table [x=kappa,y=mean_delta_R_kappa_2_8,col sep=space] {kappa_2.csv};
\addlegendentry{$m=8$ $(N=13)$}
\addplot table [x=kappa,y=mean_delta_R_kappa_2_16,col sep=space] {kappa_2.csv};
\addlegendentry{$m=16$ $(N=32)$}
\end{axis}
\end{tikzpicture}}
\caption{$n=2$}
\end{subfigure}
%\hfill
% \begin{subfigure}[b]{0.31\textwidth}
%\resizebox{\textwidth}{!}{
%\begin{tikzpicture}
% \begin{axis}[xmin=0,xmax=1,
% ymin=0,ymax=1,
% xlabel={$x$},
% ylabel={\% of $2\times 2$-matrices (Probability)},
% legend pos=south east,
% legend style={cells={align=left}}]
% \addplot table [x=ir,y=p,col sep=space] {IR_rho.csv};
% \addlegendentry{$\rho_{\IR}(x)$}
%\end{axis}
%\end{tikzpicture}}
%\caption{The function $\rho_{\IR}(x)$.}
%\end{subfigure}
\hfill
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[xmin=-1,xmax=1,
ymin=-2.5e-2,
xlabel={$c$},
ylabel={Average errors},
legend pos=south west,
legend style={cells={align=left}}]
%\addplot table [x=kappa,y=mean_delta_R_kappa_3_2,col sep=space] {kappa_3.csv};
%\addlegendentry{$m=2$}
\addplot table [x=kappa,y=mean_delta_R_kappa_3_4,col sep=space] {kappa_3.csv};
\addlegendentry{$m=4$ $(N=23)$}
\addplot table [x=kappa,y=mean_delta_R_kappa_3_8,col sep=space] {kappa_3.csv};
\addlegendentry{$m=8$ $(N=114)$}
\addplot table [x=kappa,y=mean_delta_R_kappa_3_16,col sep=space] {kappa_3.csv};
\addlegendentry{$m=16$ $(N=858)$}
\end{axis}
\end{tikzpicture}}
\caption{$n=3$}
\end{subfigure}
\caption{The average difference between $\varrho_{\kappa,n,m}(c)$ and $100$ of its Monte Carlo estimation when
$N= \left\lceil\sqrt{\left|\mathcal{C}_{m,n^2}\right|}\right\rceil$.}\label{fig:montecarlo_error}
\end{figure*}
In this case, the time complexity is $N * (T(\sigma) +T(\gamma) + T(\iota_{m, n^2}) + T(\mathcal{N}_{n,m})) $ where $\mathcal{N}_{n,m}$
is the function that uniformly samples the natural numbers in $\left[0, \left|\mathcal{C}_{m,n^2}\right|-1\right]$.
The most used pseudo-random number generators, such as Mersenne Twister~\cite{10.1145/272991.272995}, WELL~\cite{10.1145/1132973.1132974},
and \texttt{xoroshiro128++}~\cite{10.1145/3460772}, takes linear time on the number of output bits per generation, i.e., $\Theta\left(\log_2{\left|\mathcal{C}_{m,n^2}\right|}\right)$.
However, if %we assume to exclusively deal with $n$s and $m$s such that $\log_2{\left|\mathcal{C}_{m,n^2}\right|} < c_\text{bits}$ for some
%constant $c_\text{bits}$.
we assume the number of bits in the representation of both $n$ and $m$ to be upper-bound, then $T(\mathcal{N}_{n,m}) \in \Theta(1)$.
This constrain may be reasonable in the investigated domain where the matrix size, $n$, and the number of
tests used to build the confusion matrices, $m$, are usually upper bounded by $5$
(e.g., PI-RADS~\cite{PIRADS2}, BI-RADS~\cite{birads2013}) and $\num{1000000}$ (for AI applications), respectively.
Under these conditions, the Monte Carlo method takes time
$N * (T(\sigma) + T(\iota_{m, n^2}))$ to evaluate $\varrho_{\sigma,n,m}(c)$
using $N$ samples. If $T(\sigma) \in \Theta(n^2)$, as in the case of Cohen's $\kappa$, Fleiss's $\kappa$, $\IA$, and $\IR$, the time complexity belongs to
$N * (T(\iota_{m, n^2}) + \Theta(n^2))$.
When the bits in the number representation cannot be upper-bounded, a more refined analysis is required to access the asymptotic complexity of the proposed method. In particular, the logarithmic cost criterion~\cite{cormen2022introduction} should be used to take into account that the complexity of each arithmetic operation is affected by the sizes of the operator representations.
The following section presents the lexicographic order enumerator for the set $\mathcal{C}_{m,k}$ and
proves that $T(\iota_{m, k}) \in O(k+m)$.
%Thus, when $\iota_{m, n^2}$ is the lexicographic order enumerator for $\mathcal{C}_{m,n^2}$,
%the estimating $\varrho_{\sigma,n,m}(c)$ by Monte Carlo method takes time $O(N(n^2+m))$.
\subsection{The Lexicographic Order Enumeration of $\mathcal{C}_{m,k}$}
The relation $<_l \subseteq \mathcal{C}_{m,k} \times \mathcal{C}_{m,k}$ is the \emph{lexicographic order} among the weak compositions in $\mathcal{C}_{m,k}$
when for any $a, b \in\mathcal{C}_{m,k}$, $a <_l b$ implies that there exists $j \in [1, k]$ such that $a[i] = b[i]$ for all $i \in [1,j-1]$ and $a[j] < b[j]$.
The \emph{lexicographic order enumerator of set $\mathcal{C}_{m,k}$} is
the bijective function $\iota_{m,k}: \left[0, \left|\mathcal{C}_{m,k}\right|-1\right] \rightarrow \mathcal{C}_{m,k}$ such that $i<j$ if and only if $\iota_{m,k}(i) <_l \iota_{m,k}(j)$ for any $i,j \in \left[0, \left|\mathcal{C}_{m,k}\right|-1\right]$.
Thus, $\iota_{m,k}(i)=\langle\ell_1, \ldots, \ell_k\rangle$ is the $i$-th element in the lexicographic order among the weak compositions in $\mathcal{C}_{m,k}$.
This section presents a time-efficient algorithm to evaluate $\iota_{m,k}$.
Let us consider the weak compositions in $\mathcal{C}_{m,k}$ having $j \in [0, m]$ as the first component.
Since they belong to $\mathcal{C}_{m,k}$,
the sum of their components is $m$. Thus, there are as many as the weak compositions of $m-j$ in $k-1$ parts because their first component is $j$, i.e., there are
$\left|\mathcal{C}_{m-j,k-1}\right|$ weak compositions in $\mathcal{C}_{m,k}$ whose first component is $j$.
Because of the definition of lexicographic order, the first weak compositions in the lexicographic order are those having $0$ as the first component.
Then there are those having $1$ as
the first component, and so on up to the weak compositions whose first component is $m$. It follows that $i\geq\left|\mathcal{C}_{m,k-1}\right|$ if and only if
$\iota_{m,k}(i)$ follows all the weak compositions whose first component is $0$ in lexicographic order.
Analogously, $i\geq\left|\mathcal{C}_{m,k-1}\right|+\left|\mathcal{C}_{m-1,k-1}\right|$ if and only if $\iota_{m,k}(i)$ follows
all the weak compositions whose first component is $1$ in the lexicographic order. In general, $i\geq \sum_{j=0}^{l}\left|\mathcal{C}_{m-j,k-1}\right|$ if and only if
$\iota_{m,k}(i)$ follows all the weak compositions whose first component is $l$. As a consequence, the first component of $\iota_{m,k}(i)$, i.e., $\ell_1$, is such that
\begin{equation}
\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right| \leq i < \left|\mathcal{C}_{m-\ell_1,k-1}\right| + \sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right|
\end{equation}
or, equivalently,
\begin{equation}\label{eq:i_succession}
0\leq i -\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right| < \left|\mathcal{C}_{m-\ell_1,k-1}\right|.
\end{equation}
The following theorem suggests a strategy to identify the remaining components of $\iota_{m,k}(i)$, i.e., $\ell_2, \ldots, \ell_k$.
\begin{theorem}\label{theorem:iota1}
For any $m \in \mathbb{N}$, for any $k \in \mathbb{N}$, and for any $i \in \mathbb{N}$, if $k>1$ and $\iota_{m,k}(i)=\langle \ell_1, \ell_2, \ldots, \ell_{k}\rangle$, then
\begin{equation}
\iota_{m', k-1}(i')=\langle \ell_2, \ldots, \ell_{k} \rangle
\end{equation}
where $m' = m - \ell_1$ and $i' = i -\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right|$.
\end{theorem}
\begin{proof}
Let $<_l'$ be the lexicographic order among weak compositions in $\mathcal{C}_{m-\ell_1,k-1}$. According to the definition of lexicographic order,
$\langle a_1, \ldots, a_{k-1} \rangle <_l' \langle b_1, \ldots, b_{k-1} \rangle$ if and only if there exists $j \in [1, k-1]$ such that for all $i \in [1, j-1]$
$a_i = b_i$ and $a_j < b_j$. Thus, $\langle a_1, \ldots, a_{k-1} \rangle <_l' \langle b_1, \ldots, b_{k-1} \rangle$ if and only if
$\langle \ell_1, a_1, \ldots, a_{k-1} \rangle <_l' \langle \ell_1, b_1, \ldots, b_{k-1} \rangle$ and the lexicographic order among the weak compositions of $m$
in $k$ parts whose first component is $\ell_1$ and $<_l'$ are consistent.
Hence, the components $\ell_2', \dots, \ell_k'$ of $i'$-th weak composition with respect to the $<_l'$
must be the $i'$-th weak composition having $\ell_1$ as the first component with respect to the $<_l$, i.e.,
$\iota_{m-\ell_1, k-1}(i')=\langle \ell_2', \ldots, \ell_{k}' \rangle$ if and only if $\langle \ell_1, \ell_2', \ldots, \ell_{k}' \rangle$ is the
$i'$-th weak composition having $\ell_1$ as the first component with respect to the $<_l$.
Because of the definition of lexicographic order,
for all $a, b \in \mathcal{C}_{m,k}$, if the first components of $a$ and $b$ are $a_1$ and $b_1$ and $a_1 \neq b_1$, then
$a <_l b$ if and only if $a_1 < b_1$. Hence, all the weak compositions in $\mathcal{C}_{m,k}$ whose first component is lower than
$\ell_1$ come before the weak compositions in $\mathcal{C}_{m,k}$ whose first component is $\ell_1$ according to $<_l$.
Analogously, all the weak compositions in $\mathcal{C}_{m,k}$ whose first component is greater than
$\ell_1$ come after the weak compositions in $\mathcal{C}_{m,k}$ whose first component is $\ell_1$ according to $<_l$.
There are $\left|\mathcal{C}_{m-j,k-1}\right|$ weak compositions in $\mathcal{C}_{m,k}$ whose first component is $j$.
Thus, the $i'$-th weak composition in $\mathcal{C}_{m,k}$ that has $\ell_1$ as the first component with respect to the $<_l$
is in position $i'+\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right|$ of the overall lexicographic order among all
the weak compositions in $\mathcal{C}_{m,k}$. Since $i'=i-\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right|$ by hypothesis,
$i'+\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right|=(i-\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right|)+\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right|=i$
and
the $i'$-th weak composition in $\mathcal{C}_{m,k}$ that has $\ell_1$ as the first component with respect to the $<_l$
is in position $i$ of the overall lexicographic order among all
the weak compositions in $\mathcal{C}_{m,k}$. It follows that
$\iota_{m-\ell_1, k-1}(i')=\langle \ell_2', \ldots, \ell_{k}' \rangle$ if and only if $\langle \ell_1, \ell_2', \ldots, \ell_{k}' \rangle$ is the
$i$-th weak composition having $\ell_1$ as the first component with respect to the $<_l$, i.e.,
$\iota_{m-\ell_1, k-1}(i')=\langle \ell_2', \ldots, \ell_{k}' \rangle$ if and only if $\iota_{m, k}(i)=\langle \ell_1, \ell_2', \ldots, \ell_{k}' \rangle$.
\end{proof}
By exploiting Theorem~\ref{theorem:iota1}, we can build an algorithm that iteratively identifies all the components of $\iota_{m,k}(i)$
in the order of the components themselves.
The central expression's index $j$ in Eq.~\ref{eq:i_succession} ranges from $0$ up to $\ell_1-1$. The same expression can be re-written
by using an index from $m$ down to $m-\ell_1+1$
as follows
\begin{equation}
i -\sum_{j=0}^{\ell_1-1}\left|\mathcal{C}_{m-j,k-1}\right| = i -\sum_{j=m-\ell_1+1}^{m}\left|\mathcal{C}_{j,k-1}\right|.
\end{equation}
Hence, we can use a variable $m_l$ to store the initial value of $m$ and
decrease $i$ and $m$ by $\left|\mathcal{C}_{m,k-1}\right|$ and $1$, respectively, as long as $m>0$ and
$i\geq \left|\mathcal{C}_{m,k-1}\right|$.
The first component of $\iota_{m,k}(i)$ will be the difference among $m_l$ and the first $m$ such that $i < \left|\mathcal{C}_{m,k-1}\right|$,
i.e., $\ell_1=m_l-m$. The following components can be identified by decreasing $k$ by $1$ and repeating the previous steps.
\begin{algorithm}
\caption{A lexicographic order enumerator for the set $\mathcal{C}_{m,k}$.
The time complexity of this algorithm is $O(mk)$.
}\label{alg:wc_enumerator1}
\small
\begin{algorithmic}[1]
\Require $m \in \mathbb{N}$, $k\in \mathbb{N}_{>0}$, and $i \in [0, \left|\mathcal{C}_{m, k}\right|-1]$.
\Ensure Returns the $i$-th element in the lexicographic order of the weak compositions of $m$ in $k$ parts.
\Function{LexicographicOrder}{$m, k, i$}
\State $\ell \gets $\Call{array}{$k, 0$}\Comment{$\ell$ is $\vec{0}\in \mathbb{N}^{k}$}
\While{$m>0$ and $k>1$}
\State $m_l \gets m$
\State $loop\_cond \gets \left(i >= \binom{m+k-2}{m}\right)$\label{line:loop_1} %\Comment{$\left|\mathcal{C}_{m,k-1}\right|=\binom{m+k-2}{m}$}
\While{$loop\_cond$}
\State $i \gets i - \binom{m+k-2}{m}$\label{line:i_decrease}
\State $loop\_cond \gets \left(m>0 \land i >= \binom{m+k-3}{m-1}\right)$\label{line:loop_2} %\Comment{$\left|\mathcal{C}_{m-1,k-1}\right|=\binom{m+k-3}{m-1}$}
\If{$loop\_cond$}
\State $m \gets m -1$
\EndIf
\EndWhile
\State $k \gets k - 1$
\State $\ell[\ell.size()-k] \gets m_l - m$
\EndWhile
\State $\ell[\ell.size()] \gets m$
\State \Return $\ell$
\EndFunction
\end{algorithmic}
\end{algorithm}
Algorithm~\ref{alg:wc_enumerator1} describes the proposed algorithm.
The outer while-loop can be executed $k$ times at most
because each iteration decreases $k$ by one, $k$ never increases, and the loop ends when $k\leq 1$.
Analogously, the most nested loop is repeated at most $m+k$ times in total
because all iterations except the last one decrease $m$ by one, $m$ never increases,
and the two loops end when $m\leq 0$.
The lines~\ref{line:loop_1}, \ref{line:i_decrease}, and \ref{line:loop_2} of Algorithm~\ref{alg:wc_enumerator1} require
the evaluation of the binomial coefficients $\binom{m+k-2}{m}$, $\binom{m+k-2}{m}$, and $\binom{m+k-3}{m-1}$.
Since computing $\binom{a}{b}$ requires $\min\{a-b, b\}$ multiplications and
divisions, the time complexity of Algorithm~\ref{alg:wc_enumerator1} belongs to $O\left(k\min\{k,m\}+(m+k)\min\{k,m\}\right)=O(mk)$
according to the uniform cost criterion~\cite{cormen2022introduction}.
To improve complexity, we can observe that $\left|\mathcal{C}_{m, k}\right|$, $\left|\mathcal{C}_{m-1, k}\right|$, and $\left|\mathcal{C}_{m, k-1}\right|$ are related, and once $\left|\mathcal{C}_{m, k}\right|$ has been computed, we can evaluate both $\left|\mathcal{C}_{m-1, k}\right|$ and $\left|\mathcal{C}_{m, k-1}\right|$ from its value in constant time according to the uniform cost criterion.
\begin{lemma}\label{lemma:binom_relation}
It holds that $\left|\mathcal{C}_{m-1, k}\right| = \frac{m}{m+k-1} \left|\mathcal{C}_{m, k}\right|$, and
$\left|\mathcal{C}_{m, k-1}\right| = \frac{k-1}{m+k-1} \left|\mathcal{C}_{m, k}\right|$
for any $m, k \in \mathbb{N}_{>0}$.
\end{lemma}
\begin{proof}
As far $\left|\mathcal{C}_{m-1, k}\right|$ concerns,
\begin{align*}
\left|\mathcal{C}_{m-1, k}\right| &= \binom{(m-1)+k-1}{m-1} = \frac{(m+k-2)!}{(m-1)!(k-1)!}\\
&= \frac{m}{m+k-1}\frac{m+k-1}{m}\frac{(m+k-2)!}{(m-1)!(k-1)!}\\
&= \frac{m}{m+k-1}\frac{(m+k-1)!}{m!(k-1)!}= \frac{m}{m+k-1}\left|\mathcal{C}_{m-1, k}\right|.
\end{align*}
Analogously,
\begin{align*}
\left|\mathcal{C}_{m, k-1}\right| &= \binom{m+(k-1)-1}{m} = \frac{(m+k-2)!}{m!(k-2)!}\\
&= \frac{k-1}{m+k-1}\frac{m+k-1}{k-1}\frac{(m+k-2)!}{m!(k-2)!}\\
&= \frac{k-1}{m+k-1}\frac{(m+k-1)!}{m!(k-1)!} = \frac{k-1}{m+k-1}\left|\mathcal{C}_{m, k-1}\right|.
\end{align*}
\end{proof}
Algorithm~\ref{alg:wc_enumerator2} mimes the steps of Algorithm~\ref{alg:wc_enumerator1}
but, thanks to Lemma~\ref{lemma:binom_relation}, avoids the computation of the binomial coefficient at each while-loop iteration.
The time complexity of Algorithm~\ref{alg:wc_enumerator2} according to the uniform const criterion
belongs to $O(\min\{k,m\} + k + m)=O(k + m)$.
\begin{algorithm}
\caption{A lexicographic order enumerator for the set $\mathcal{C}_{m,k}$
%This is the fast version of the Algorithm~\ref{alg:wc_enumerator1}. It
that avoids the computation of binomial coefficients during the while-loop iterations.
The time complexity of this algorithm is $O(m+k)$.}\label{alg:wc_enumerator2}
\small
\begin{algorithmic}[1]
\Require $m \in \mathbb{N}$, $k\in \mathbb{N}_{>0}$, and $i \in [0, \left|\mathcal{C}_{m, k}\right|-1]$.
\Ensure Returns the $i$-th element in the lexicographic order of the weak compositions of $m$ in $k$ parts.
\Function{FastLexicographicOrder}{$m, k, i$}
\State $\ell \gets $\Call{array}{$k, 0$}\Comment{$\ell$ is $\vec{0}\in \mathbb{N}^{k}$}
\State $C \gets \binom{m+k-2}{m}$\Comment{$C=\left|\mathcal{C}_{m,k-1}\right|$}
\While{$m>0$ and $k>1$}
\State $m_l \gets m$
\State $loop\_cond \gets \left(i >= C\right)$
\While{$loop\_cond$}
\State $i \gets i - C$
\State $loop\_cond \gets m>0 \land i >= \frac{m}{m+k-2} C$%\Comment{$\left|\mathcal{C}_{m-1,k-1}\right|=\frac{m}{m+k-2} C$}
\If{$loop\_cond$}
\State $C \gets \frac{m}{m+k-2} C$\Comment{$C=\left|\mathcal{C}_{m-1,k-1}\right|$}
\State $m \gets m-1$\Comment{$C=\left|\mathcal{C}_{m,k-1}\right|$}
\EndIf
\EndWhile
\State $k \gets k - 1$\Comment{$C=\left|\mathcal{C}_{m,k}\right|$}
\State $\ell[\ell.size()-k] \gets m_l - m$
\State $C \gets \frac{k-1}{m+k-1} C$\Comment{$C=\left|\mathcal{C}_{m,k-1}\right|$}
\EndWhile
\State $\ell[\ell.size()] \gets m$
\State \Return $\ell$
\EndFunction
\end{algorithmic}
\end{algorithm}
%By exploiting Lemma~\ref{lemma:iota1}, we can define three successions of numbers $i_\alpha$, $m_\alpha$, and $\ell_\alpha$ as follows.
%
%\begin{equation}
%i_\alpha \defeq \begin{cases}
%i & \text{if $\alpha=0$}\\
%i_{\alpha-1}-\sum_{j=0}^{\ell_\alpha-1}\left|\mathcal{C}_{m_{\alpha-1}-j,k-\alpha}\right|& \text{if $\alpha>0$}
%\end{cases}
%\end{equation}
%
%\begin{equation}
%m_\alpha \defeq \begin{cases}
%m & \text{if $\alpha=0$}\\
%m_{\alpha-1}-\ell_\alpha & \text{if $\alpha>0$}
%\end{cases}
%\end{equation}
%
%\begin{equation}
%\ell_\alpha \defeq \max\left\{ \ell \in \mathbb{N} \,|\, i_\alpha > \sum_{j=0}^{\ell-1}\left|\mathcal{C}_{m_{\alpha-1}-j,k-\alpha}\right| \right\}
%\end{equation}
When $T(\sigma) \in \Theta(n^2)$ and
$\iota_{m, n^2}$ is implemented by Algorithm~\ref{alg:wc_enumerator2},
the Monte Carlo method with $N$ samples evaluates
$\varrho_{\sigma,n,m}(c)$ in time $O(N (n^2+m))$.
%\begin{algorithm}
%\caption{The lexicographic order enumerator for the set $\mathcal{C}_{m,k}$.
%This function maps any natural value in $\left[0, \left|\mathcal{C}_{m,n^2}\right|-1\right]$ to
%a weak composition in $\mathcal{C}_{m,k}$ preserving the lexicographic order, i.e., if $a,b \in \left[0, \binom{m+k-1}{m}-1\right]$ and $a<b$,
%then there exists $j \in [1, k]$ such that $a_i = b_i$ for all $i \in [1,j-1]$ and $a_j < b_j$ where $\iota_{k,m}(a)=\langle a_1, \ldots, a_k \rangle$ and
%$\iota_{k,m}(b)=\langle b_1, \ldots, b_k \rangle$.
%The algorithm counts how many weak compositions of $m$ in $k$ parts have the value $j$ as the first components
%$n_{m,k,j}=\binom{(m-j)+(k-1)-1}{m-j}=\binom{(m-j)+k-2}{m-j}$. This is repeated up to the largest $l$ such that
%$i \geq \sum_{j=0}^{l} n_{m,k,j}$. Thus, the first component of the $i$-th element of $\mathcal{C}_{m,k}$ in the lexicographic order is
%$l+1$, and the algorithm proceeds by searching the first component of the $(i-\sum_{j=0}^{l} n_{m,k,j})$-th element of
%$\mathcal{C}_{m-l,k-1}$ until $k>1$ or $m>0$. Computing $\binom{n}{m}$ requires $\Theta(\min\{m, n-m\})$ multiplications and
%divisions.
%%divisions between numbers smaller than or equal to $n$. Since two $l$-bit numbers can be multiplied in time $O(l \log{l} 4^{\log^*l})$~\cite{multiplication},
%%$T(\binom{n}{m}) \in O(\log{n^{\min\{m, n-m\}^2}} \log{\log{n^{\min\{m, n-m\}}}} 4^{\log^*\log{n^{\min\{m, n-m\}}}})$.
%The most nested loop is repeated at most $m$ times because each iteration decreases $m$ by one, $m$ is never increased,
%and the two loops end when $m\leq 0$. Analogously, the outer while-loop can be executed $k$ times at most,
%each iteration decreases $k$ by one, $k$ is never increased, and the loop ends when $k\leq 1$.
%%Hence, according to the logarithmic cost criterion,
%%a course upper bound for the time complexity of this algorithm is
%%$O((m+k)\log{(m+k)^{\min\{k, m\}^2}} \log{\log{(m+k)^{\min\{k, m\}}}} 4^{\log^*\log{(m+k)^{\min\{k, m\}}}})$, while
%%the constant cost criterion time complexity is $O((m+k)\min\{m, n-m\})$.
%Hence, according to the uniform cost criterion~\cite{cormen2022introduction},
%the time complexity of this algorithm is $O(\min\{m, k\}+m+k)=O(m + k)$.
%}\label{alg:wc_enumerator}
%\begin{algorithmic}[1]
%\Require $m \in \mathbb{N}$, $k\in \mathbb{N}_{>0}$, and $i \in [0, \left|\mathcal{C}_{m, k}\right|-1]$.
%\Ensure Returns the $i$-th element in the lexicographic order of the weak compositions of $m$ in $k$ parts.
%\Function{WeakCompositionEnumerator}{$m, k, i$}
%\State $x \gets \langle 0, \ldots, 0 \rangle$\Comment{$x$ is initialised to an array of size $k$}
%\State $Cs \gets \binom{k+m-2}{m}$\Comment{$Cs$ is $\left|\mathcal{C}_{m, k-1}\right|=\binom{(k-1)+m-1}{m}$}
%\While{$m>0$ and $k>1$}
%\State $prev\_m \gets m$
%\State $(Cs', m') \gets (Cs, m)$
%\While{$i >= Cs'$ and $m'>0$}
%\State $(Cs, m) \gets (Cs', m')$\Comment{$Cs$ is $\left|\mathcal{C}_{m, k-1}\right|$}
%%\State $m \gets m'$
%%\State $Cs \gets Cs'$\Comment{$Cs'$ is $\left|\mathcal{C}_{m, k-1}\right|$}
%\State $i \gets i-Cs$
%\State $Cs' \gets \frac{Cs'*m}{k+m-2}$\Comment{update $Cs'$ to be $\left|\mathcal{C}_{m-1, k-1}\right|$}%=\frac{(k+m-3)!}{(m-1)!(k-2)!}=\frac{(k+m-3)\ldots(k-1)}{(m-1)!}=\left|\mathcal{C}_{m, k-1}\right| \frac{m}{k+m-2}$}
%\State $m' \gets m'-1$
%\EndWhile
%\State $x[x.size()+1-k] \gets prev\_m - m$
%\State $Cs \gets \frac{Cs*(k-2)}{k+m-2}$\Comment{update $Cs$ to be $\left|\mathcal{C}_{m, k-2}\right|$}
%\State $k \gets k - 1$
%\EndWhile
%\State $x[x.size()] \gets m$
%\State \Return $x$
%\EndFunction
%\end{algorithmic}
%\end{algorithm}
\section{Significativity over Probability Matrices}\label{sec:significativity_probability}
%To compute the $\sigma$-significativity of $c$ in $\mathcal{M}_{n,m}$ we need to represent the number $\left|\mathcal{C}_{m,n^2}\right|$
%and this may require more bits than those available in the registers of the most common consumer
%hardware (i.e., $64$ bit). For instance, when $n=3$ and $m=1000$, $\left|\mathcal{C}_{m,n^2}\right|$ requires
%$\lceil\log_2\left(\left|\mathcal{C}_{m,n^2}\right|\right)\rceil = 65$ bits. The same number of bits
%are required when $n=5$ and $m=51$. Even though the asymptotic time complexity is not affected by $\mathcal{C}_{m,n^2}$ cardinality
%when we assume $\left|\mathcal{C}_{m,n^2}\right|$ to be upper bounded by a constant, there are computational drawbacks
%in using software arithmetic in place of the hardware arithmetic because the hardware registers cannot represent the involved numbers.
%
%Moreover, the
The $\sigma$-significativity of $c$ in $\mathcal{M}_{n,m}$ depends on the number $m$ of the tests accounted by the matrices
in $\mathcal{M}_{n,m}$.
In this section, we introduce a different measure of significativity
%whose goal is
%to gauge the $\sigma$-significativity of $c$ in $\mathcal{M}_{n,m}$ as $m$ tends to infinity.
%As a consequence, this new measure can estimate the maximum
%theoretical $\sigma$-significativity in $\mathcal{M}_{n,m}$ for any $\sigma$-value $c$.
for agreement values to avoid dependency on the data set size. With this aim, we
focus on probability matrices in place of confusion matrices.
The \emph{$\sigma$-significativity of $c$ in $\mathcal{P}_n$} is the ratio between the number of probability
matrices $M_P \in \mathcal{P}_n$ such that $\sigma(M_P)< c$ and the overall number of matrices in $\mathcal{P}_n$.
These two numbers are infinite, but if $\mathcal{P}_n$ and the set of the probability matrices $M_P \in \mathcal{P}_n$ such that $\sigma(M_P) < c$
are Lebesgue-measurable (e.g., see~\cite{Apostol:105425}) in an opportune space and non-null, we can evaluate the ratio between their cardinalities as the ratio
between their Lebesgue measures.
% To evaluate this ratio, we first need a way to assess the number of probability matrices. Of course, the matrices are infinite, but we can estimate it as a ratio.
%Dealing with tests having $n$ possible outcomes, the size of the probability matrices is $n \times n$. Thus,
According to the definition of probability matrix, a $n \times n$ probability matrix consists of $n^2$ values in the interval $[0,1]$ such that their sum
equals $1$. Thus, we can map any $n \times n$ probability matrix into a point of the $(n^2)$-dimensional hypercube $[0,1]^{n^2}$ by using $\gamma^{-1}$.
However, not all the points in $[0,1]^{n^2}$ correspond to a probability matrix because their components must always sum up to $1$.
%Moreover, since all the values in a probability matrix are probabilities, they must be values in the interval $[0,1]$.
The $(k-1)$-dimensional \emph{probability simplex} (e.g., see~\cite{boyd2004convex}) is defined as
%\begin{equation}\label{def:simplex}
%\Delta^{(k-1)} \defeq \left\{\langle x_1, \ldots, x_{k}\rangle \in \mathbb{R}^{k} \big| \sum_{ i=1}^{k} %x_{i} = 1 \land \bigwedge_{i = 1}^{k} x_{i}\geq 0\right\}.
%\end{equation}
\begin{equation}\label{def:simplex}
\Delta^{(k-1)} \defeq \left\{x \in \mathbb{R}_{\geq 0}^{k} \bigg| \sum_{ i=1}^{k} x_i = 1 \right\}
\end{equation}
where $x_i$ is the $i$-th component of vector $x$.
%\begin{equation}\label{def:simplex}
%\begin{aligned} \Delta^{(k-1)} \defeq &\left\{\langle x_1, \ldots, x_{k}\rangle \in \mathbb{R}_{\geq 0}^{k} \bigg|\phantom{\sum_{ i=1}^{k}}\right.\\
%&\left.\phantom{\langle x_1, \ldots, x_{k}\rangle \in}\sum_{ i=1}^{k} x_{i} = 1\right\}.\end{aligned}
%\end{equation}
The function $\gamma$ maps any point in $\Delta^{(n^2-1)}$ into a $n\times n$ probability matrix and every probability matrix is the image of
a point of the same simplex, i.e., $\gamma(\Delta^{(n^2-1)})=\mathcal{P}_n$ and $\Delta^{(n^2-1)}=\gamma^{-1}(\mathcal{P}_n)$.
Hence, the set of the
matrices $M_P \in \mathcal{P}_n$ such that $\sigma(M_P) < c$ corresponds
to the set of points $x \in \Delta^{(n^2-1)}$ such that $\sigma(\gamma(x)) < c$, i.e.,
\begin{equation}
\mathcal{S}_{\sigma, n}(c) \defeq \left\{x \in \Delta^{(n^2-1)} \, \big|\, \sigma(\gamma(x)) < c \right\}.
\end{equation}
\begin{figure*}[!ht]
\begin{center}
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\tikzset{every mark/.append style={scale=0.5}}
\begin{tikzpicture}
\begin{axis}
[ view={-45}{60},
xmin=0,xmax=1,
ymin=0,ymax=1,
zmin=0, zmax=1,
% xlabel = {$p(\text{TP})$},
% ylabel = {$p(\text{FP})$},
% zlabel = {$p(\text{FN})$}
xlabel = {$x_1$},
ylabel = {$x_2$},
zlabel = {$x_3$}
]
\addplot3[red, only marks] file{./out_set.txt};
\addlegendentry{$\Delta^3 \setminus \mathcal{S}_{\IA,2}(0.3)$}
\addplot3[blue, only marks] file{./in_set.txt};
\addlegendentry{$\mathcal{S}_{\IA,2}(0.3)$}
\end{axis}
\end{tikzpicture}}
\end{subfigure}
\hfill
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\tikzset{every mark/.append style={scale=0.5}}
\begin{tikzpicture}
\begin{axis}
[ view={0}{0},
xmin=0,xmax=1,
ymin=0,ymax=1,
zmin=0, zmax=1,
% xlabel = {$p(\text{TP})$},
% ylabel = {$p(\text{FP})$},
% zlabel = {$p(\text{FN})$}
xlabel = {$x_1$},
ylabel = {$x_2$},
zlabel = {$x_3$}
]
\addplot3[red, only marks, mark=square*] file{./out_set.txt};
\addlegendentry{$\Delta^3 \setminus \mathcal{S}_{\IA,2}(0.3)$}
\addplot3[blue, only marks] file{./in_set.txt};
\addlegendentry{$\mathcal{S}_{\IA,2}(0.3)$}
\end{axis}
\end{tikzpicture}}
\end{subfigure}
\caption{A 3D graphical representation of the set $\mathcal{S}_{\IA, 2}(0.3)$.
We uniformly sampled 2000 points in the cube $\Delta^{3}$ and plotted them. The blue-coloured
points belong to $\mathcal{S}_{\IA, 2}(0.3)$, while
the red ones lays in $\Delta^{3} \setminus \mathcal{S}_{\IA, 2}(0.3)$.}\label{fig:S}
\end{center}
\end{figure*}
For example, Figure~\ref{fig:S} represents the set $\mathcal{S}_{\IA, 2}(0.3)$.
For any $\langle x_1, \ldots, x_{n^2} \rangle \in \Delta^{(n^2-1)}$, $x_{n^2} = 1 - \sum_{ i=1}^{n^2-1} x_i$ according to Eq.~\ref{def:simplex}.
As a consequence, the function $\pi(\langle x_1, \ldots, x_{n^2} \rangle) \defeq \langle x_1, \ldots, x_{n^2-1} \rangle$ is bijective on $\Delta^{(n^2-1)}$ and any point in
$\Delta^{(n^2-1)}$ corresponds to a point in $\pi(\Delta^{(n^2-1)}) \subset \mathbb{R}^{n^2-1}$.
%and there also exists a bijective function $\pi$ from $\Delta^{(n^2-1)}$ to a subset of $\mathbb{R}^{(n^2-1)}$, e.g.,
%$\pi(\langle x_1, \ldots, x_{n^2} \rangle) \defeq \langle x_1, \ldots, x_{n^2-1} \rangle$. Hence, $\rho_\sigma(c)$ is the ratio between the volumes of $\pi(\mathcal{S}_{\sigma}(c))$ and
%$\pi(\Delta^{(n^2-1)})$ where $\pi(S) \defeq \{\pi(x)\,|\, x \in S\}$.
The Lebesgue measure of $\Delta^{(k-1)}$, i.e., its volume, on dimension $k-1$ is $\frac{1}{(k-1)!}$~\cite{simplex}. Hence, if $\mathcal{S}_{\sigma,n}(c)$ is Lebesgue-measurable, then
\begin{equation}\label{def:rho}
\rho_{\sigma,n}(c) = \frac{V(\mathcal{S}_{\sigma,n}(c))}{V(\Delta^{(n^2-1)})}={(n^2-1)!}\, \int_{\Delta^{(n^2-1)}} \mathbf{1}_{\mathcal{S}_{\sigma,n}(c)}(x) d\, x
\end{equation}
where $V(S)$ denotes the Lebesgue measure of $S$ on dimension $(n^2-1)$. %$\pi(S) \defeq \{\pi(x)\,|\, x \in S\}$.
The value $\rho_{\sigma,n}(c)$ is the $\sigma$-significativity of $c$ in $\mathcal{P}_n$.
O-minimal sets are Lebesgue-measurable~\cite{Dries_1998}. Thus, when $\mathcal{S}_{\sigma,n}(c))$ is o-minimal, $\rho_{\sigma,n}(c)$ is well defined.
This is the case for $\sigma$ among Cohen's $\kappa$, Scott's $\pi$, Yule's $Y$,
Fleiss's $\kappa$, $\IR$, and $\IA$. Even though their definitions include a division, $\mathcal{S}_{\sigma, n}(c)$, where $\sigma$ is any of the cited agreement measures,
is definable in $(\mathbb{R}, \{0,1\}, +, $ $*, e^{x}, >)$ and is an o-minimal set.
\subsection{Evaluating $\rho_{\sigma, n}(c)$}
The analytic evaluation of the integral in Eq.~\ref{def:rho} is not always possible and relies on the form of $\sigma(\cdot)$.
In any case, we can numerically estimate it using the Monte Carlo integration method.
This method approximates the integral of a function $f: D \subseteq \mathbb{R}^n \rightarrow \mathbb{R}$ over $\Omega \subseteq D$ as
\begin{equation}
\int_{\Omega} f(x) d\, x = \lim_{N \rightarrow +\infty} \frac{V(\Omega)}{N} \sum_{i=1}^{N} f(x_i)
\end{equation}
where $x_1, \ldots x_N$ are uniformly sampled point in $\Omega$
(e.g., see~\cite{Mackay1998}). As in the discrete case, the approximation error is proportional to $1/\sqrt{N}$.
The following theorem suggests how to uniformly sample $\Delta^{(n^2-1)}$.
\begin{theorem}\cite[Ch.~5, Theorem~2.2]{devroye:1986}\label{theo:sampling_simplex}
If $E_1, \ldots, E_{k}$ are independent and identically distributed exponential random variables, then $\left\langle \frac{E_1}{\sum_{i=1}^{k}E_i}, \ldots, \frac{E_k}{\sum_{i=1}^{k}E_i} \right\rangle$
is uniformly distributed over $\Delta^{(k-1)}$.
\end{theorem}
Hence, we can approximate $V(\mathcal{S}_{\sigma}(c))$ as
\begin{equation}
V(\mathcal{S}_{\sigma,n}(c))= \int_{\Delta^{(n^2-1)}} \mathbf{1}_{\mathcal{S}_{\sigma,n}(c)}(x) d\, x \approx
\frac{V(\Delta^{(n^2-1)})}{N} \sum_{i=1}^{N} \mathbf{1}_{\mathcal{S}_{\sigma,n}(c)}(y_i),
\end{equation}
where $y_1, \ldots, y_N$ are uniformly distributed samples of $\Delta^{(n^2-1)}$.
From Eq.~\ref{def:rho}, it follows that:
\begin{equation}
\rho_{\sigma,n}(c) %= \frac{V(\mathcal{S}_{\sigma,n}(c))}{V(\Delta^{(n^2-1)})}
\approx \frac{1}{N} \sum_{i=1}^{N} \mathbf{1}_{\mathcal{S}_{\sigma,n}(c)}(y_i).
\end{equation}
\begin{figure*}[!ht]
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[xmin=0,xmax=1,
ymin=0,ymax=1,
xlabel={$c$},
ylabel={$\IA$–Significativity},
legend pos=south east,
legend style={cells={align=left}}]
\addplot table [x=IA,y=rho2,col sep=space] {IA_rho.csv};
\addlegendentry{$\rho_{\IA,2}(c)$}
\addplot table [x=IA,y=rho3,col sep=space] {IA_rho.csv};
\addlegendentry{$\rho_{\IA,3}(c)$}
\addplot table [x=IA,y=rho4,col sep=space] {IA_rho.csv};
\addlegendentry{$\rho_{\IA,4}(c)$}
\addplot table [x=IA,y=rho5,col sep=space] {IA_rho.csv};
\addlegendentry{$\rho_{\IA,5}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The functions $\rho_{\IA,n}(c)$.}
\end{subfigure}
%\hfill
% \begin{subfigure}[b]{0.31\textwidth}
%\resizebox{\textwidth}{!}{
%\begin{tikzpicture}
% \begin{axis}[xmin=0,xmax=1,
% ymin=0,ymax=1,
% xlabel={$x$},
% ylabel={\% of $2\times 2$-matrices (Probability)},
% legend pos=south east,
% legend style={cells={align=left}}]
% \addplot table [x=ir,y=p,col sep=space] {IR_rho.csv};
% \addlegendentry{$\rho_{\IR}(x)$}
%\end{axis}
%\end{tikzpicture}}
%\caption{The function $\rho_{\IR}(x)$.}
%\end{subfigure}
\hfill
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[xmin=-1,xmax=1,
ymin=0,ymax=1,
xlabel={$c$},
ylabel={$\kappa$-Significativity},
legend pos=south east,
legend style={cells={align=left}}]
\addplot table [x=kappa,y=rho2,col sep=space] {kappa_rho.csv};
\addlegendentry{$\rho_{\kappa,2}(c)$}
\addplot table [x=kappa,y=rho3,col sep=space] {kappa_rho.csv};
\addlegendentry{$\rho_{\kappa,3}(c)$}
\addplot table [x=kappa,y=rho4,col sep=space] {kappa_rho.csv};
\addlegendentry{$\rho_{\kappa,4}(c)$}
\addplot table [x=kappa,y=rho5,col sep=space] {kappa_rho.csv};
\addlegendentry{$\rho_{\kappa,5}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The functions $\rho_{\kappa,n}(c)$.}
\end{subfigure}
\caption{The $\IA$-significativity and Cohen's $\kappa$-significativity. When $\sigma$ is a agreement measure between classifiers having $n$ possible outcomes,
the function
$\rho_{\sigma,n}(x)$ is the ratio between the number of $n\times n$-probability matrices $M \in \mathcal{P}_n$ such that $\sigma(M)< c$ and the
overall number of $n\times n$-probability matrices.}\label{fig:rho}
\end{figure*}
Sampling $\Delta^{(n^2-1)}$ according to Theorem~\ref{theo:sampling_simplex} takes time $\Theta(k)$ per
sample. Thus, when $T(\sigma) \in \Theta(n^2)$, the
Monte Carlo method can estimate $\rho_{\sigma,n}(c)$ in time $\Theta(n^2N)$.
\begin{example}
Let $M_C$ be the matrix as defined in Example~\ref{ex:varrho}.
Cohen's $\kappa$ and $\IA$ for $M_P\defeq T_P(M_C)$ are $\kappa(M_P)=\kappa(M_C)\approx 0.70588$ and $\IA(M_P)=\IA(M_C)\approx 0.52115$, respectively.
The $\kappa$-significativity of $\kappa(M_P)$ in $\mathcal{P}_{2}$ is $\rho_{\kappa,2}(\kappa(M_P))\approx 0.9642$.
Thus, more than $96\%$ of the $2\times 2$-probability matrices have a $\kappa$ value lower than that of $M_P$ and
the probability of choosing by chance a probability matrix $M_P' \in \mathcal{P}_{2}$ with $\kappa(M_P') < \kappa(M_P)$ is
$0.9642$.
Instead, the $\IA$-significativity of $\IA(M_P)$ in $\mathcal{P}_{2}$ is $\rho_{\IA,2}(\IA(M_P))\approx 0.9507$.
Hence, less than $5\%$ of the $2\times 2$-probability matrices have an $\IA$ value greater than or equal to that of $M_P$ and
the probability of choosing by chance a probability matrix $M_P' \in \mathcal{P}_{2}$ with $\IA(M_P') \leq \IA(M_P)$ is
$0.9507$.
\end{example}
Figure~\ref{fig:rho} shows $\IA$ and Cohen's $\kappa$-significativity estimations as
$n$ ranges between $2$ and $5$.
\subsection{Significativity Relation}
The following theorem relates the $\sigma$-significativities of $c$ in $\mathcal{P}_n$ and
$\mathcal{M}_{n, m}$.
\begin{theorem}\label{theo:riemann}
If $\mathcal{S}_{\sigma,n}(c)$ is Riemann-measurable and $\sigma(M_C) = \sigma(T_P(M_C))$ for all $M_C \in \mathcal{M}_{n,m}$, then
\begin{equation}
\lim_{m\rightarrow +\infty}\varrho_{\sigma,n,m}(c) =\rho_{\sigma, n}(c).
\end{equation}
\end{theorem}
\begin{proof}
See Appendix~\ref{sec:proofs_riemann}.
\end{proof}
Since Riemann-measurability implies Lebesgue-measurability~\cite{Apostol:105425}, $\rho_{\sigma, n}(c)$ is well defined
when $\mathcal{S}_{\sigma,n}(c)$ is Riemann-measurable.
It follows that, when $\mathcal{S}_{\sigma,n}(c)$ is Riemann-measurable and
$\sigma(M_C) = \sigma(T_P(M_C))$,
$\rho_{\sigma, n}(c)$ can be used as an approximation of $\varrho_{\sigma,n,m}(c)$.
Although the Riemann-integrability may seem a restrictive condition, the following results prove that it is quite commonly satisfied, and every bounded o-minimal set is integrable by
Riemann.
\begin{lemma}\label{lemma:ominimal}
Every bounded o-minimal subset of $\mathbb{R}^n$ is Riemann-measurable.
\end{lemma}
\begin{proof}
A bounded set $S$ is Riemann-measurable if and only if its indicator function $\mathbf{1}_{S}$ is Riemann integrable.
By Lebesgue's criterion for Riemann integrability (e.g., see~\cite[Theorem~14.5]{Apostol:105425}), any function $f$ definable and bounded on a compact interval
$I\subseteq \mathbb{R}^n$ is
Riemann-integrable on $I$ if and only if the set of discontinuities of $f$ in $I$ has $n$-dimensional Lebesgue measure $0$.
However, the set of discontinuities of $\mathbf{1}_{S}$ matches the topological boundary of $S$.
If $S \subseteq \mathbb{R}^n$ is o-minimal, then the dimension of the boundary of $S$ is lower than $m$ in $\mathbb{R}^n$~\cite[Ch.~4, Corollary 1.10]{Dries_1998}.
Thus, the boundary of $S$ has $m$-dimensional Lebesgue measure $0$, and $S$ is Riemann-measurable.
\end{proof}
The following corollary is a direct consequence of Theorem~\ref{theo:riemann} and Lemma~\ref{lemma:ominimal}.
\begin{corollary}\label{cor:ominimal}
If $\mathcal{S}_{\sigma, n}(c)$ is definable in an o-minimal theory over $\mathbb{R}$ and $\sigma(M_C) = \sigma(T_P(M_C))$ for all $M_C \in \mathcal{M}_{n,m}$, then
\begin{equation}
\lim_{m\rightarrow +\infty} \varrho_{\sigma,n,m}(c) =\rho_{\sigma, n}(c).
\end{equation}
\end{corollary}
It is easy to verify that $\sigma(M_C) = \sigma(T_P(M_C))$ holds for Cohen's $\kappa$ and $\IA$. Since, as pointed out in this section,
$\mathcal{S}_{\sigma,n}(c)$ is an o-minimal set for the same agreement measures, Corollary~\ref{cor:ominimal}'s thesis holds for them.
Figure~\ref{fig:error} shows the estimated errors in approximating $\varrho_{\sigma,n,m}(c)$ by $\rho_{\sigma, n}(c)$ as $m$ changes for $\sigma$ among Cohen's $\kappa$ and $\IA$.
As expected, the difference between the two indices decreases as $m$ increases.
However, when the data set consists of 10 entries, it exceeds 0.8 and 0.3 for $\IA$ and Cohen's $\kappa$, respectively. Thus, if the data set is not large enough, then $\rho_{\sigma, n}(c)$ can not effectively approximate $\varrho_{\sigma,n,m}(c)$.
%
%The semi-algebraic theory of the polynomials on $\mathbb{R}$, i.e., $(\mathbb{R}, \{0,1\}, +, $ $*, >)$ and
%its Pfaffian extensions~\cite{speissegger1999pfaffian}, e.g., $(\mathbb{R}, \{0,1\}, +, $ $*,e^{x}, >)$ or $(\mathbb{R}, \{0,1\}, +, $ $*,\ln{x}, >)$, are o-minimal theories.
%Thus, the above results hold for Cohen's $\kappa$, $\IR$, $\IA$, and all statistic coefficients definable as rational functions on $\mathbb{R}$.
%If $\rho_{f}(c)$ is computable, then we can define a quality scale for $\sigma(\cdot)$ by sampling over all the admissible values of $c$, i.e., $[0,1]$, or by using a dichotomic search~\cite{Cormen:2009}. For instance, we may identify the value $c_0, c_1, c_2, c_3$ such that
%$\rho_{f}(c_0)=0.2$, $\rho_{f}(c_1)=0.4$, $\rho_{f}(c_2)=0.6$, and $\rho_{f}(c_3)=0.8$, and, miming the McHugh's scale for Cohen's $\kappa$~\cite{McHugh:2012},
%use them as boundaries of the scale classes, e.g.,
%$[0, c_0)$ (``none to slight''), $[c_0, c_1)$ (``fair''), $[c_1, c_2)$ (``moderate''),
%$[c_2, c_3)$ (``substantial''), and $[c_3, 1.0)$ (``perfect or almost perfect agreement'').
\begin{figure*}[!ht]
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[ xmin=0,xmax=1,
ymin=0,ymax=1,
xlabel={$c$},
ylabel={Significativity difference},
legend pos=north east,
legend style={cells={align=left}}]
%\addplot table [x=v,y=delta2_2,col sep=space] {IA_delta.csv};
%\addlegendentry{$\rho_{\IA,2}(x)-\varrho_{\IA, 2, 2}(x)$}
%\addplot table [x=IA,y=delta2_4,col sep=space] {IA_delta.csv};
%\addlegendentry{$\rho_{\IA,2}(x)-\varrho_{\IA, 2, 4}(x)$}
%\addplot table [x=IA,y=delta2_8,col sep=space] {IA_delta.csv};
%\addlegendentry{$\rho_{\IA,2}(x)-\varrho_{\IA, 2, 8}(x)$}
\addplot table [x=IA,y=delta2_10,col sep=space] {IA_delta.csv};
\addlegendentry{$\rho_{\IA,2}(c)-\varrho_{\IA, 2, 10}(c)$}
\addplot table [x=IA,y=delta2_100,col sep=space] {IA_delta.csv};
\addlegendentry{$\rho_{\IA,2}(c)-\varrho_{\IA, 2, 100}(c)$}
\addplot table [x=IA,y=delta2_1000,col sep=space] {IA_delta.csv};
\addlegendentry{$\rho_{\IA,2}(c)-\varrho_{\IA, 2, 1000}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The difference between $\rho_{\IA, 2}(c)$ and
$\varrho_{\IA, 2, m}(c)$.}
\end{subfigure}
%\hfill
% \begin{subfigure}[b]{0.31\textwidth}
%\resizebox{\textwidth}{!}{
%\begin{tikzpicture}
% \begin{axis}[xmin=0,xmax=1,
% ymin=0,ymax=1,
% xlabel={$x$},
% ylabel={\% of $2\times 2$-matrices (Probability)},
% legend pos=south east,
% legend style={cells={align=left}}]
% \addplot table [x=ir,y=p,col sep=space] {IR_rho.csv};
% \addlegendentry{$\rho_{\IR}(x)$}
%\end{axis}
%\end{tikzpicture}}
%\caption{The function $\rho_{\IR}(x)$.}
%\end{subfigure}
\hfill
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[ xmin=-1,xmax=1,
ymin=0,ymax=1,
xlabel={$c$},
ylabel={Significativity difference},
legend pos=north west,
legend style={cells={align=left}}]
\addplot table [x=kappa,y=delta2_10,col sep=space] {kappa_delta.csv};
\addlegendentry{$\rho_{\kappa,2}(c)-\varrho_{\kappa, 2, 10}(c)$}
\addplot table [x=kappa,y=delta2_100,col sep=space] {kappa_delta.csv};
\addlegendentry{$\rho_{\kappa,2}(c)-\varrho_{\kappa, 2, 100}(c)$}
\addplot table [x=kappa,y=delta2_1000,col sep=space] {kappa_delta.csv};
\addlegendentry{$\rho_{\kappa,2}(c)-\varrho_{\kappa, 2, 1000}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The difference between $\rho_{\kappa, 2}(c)$ and
$\varrho_{\kappa, 2, m}(c)$.}
\end{subfigure}
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[ xmin=0,xmax=1,
ymin=0,ymax=1.3,
xlabel={$c$},
ylabel={Significativity difference},
legend pos=north east,
legend style={cells={align=left}}]
%\addplot table [x=v,y=delta3_2,col sep=space] {IA_delta.csv};
%\addlegendentry{$\rho_{\IA,3}(x)-\varrho_{\IA, 3, 2}(x)$}
%\addplot table [x=IA,y=delta3_4,col sep=space] {IA_delta.csv};
%\addlegendentry{$\rho_{\IA,3}(x)-\varrho_{\IA, 3, 4}(x)$}
%\addplot table [x=IA,y=delta3_8,col sep=space] {IA_delta.csv};
%\addlegendentry{$\rho_{\IA,3}(x)-\varrho_{\IA, 3, 8}(x)$}
\addplot table [x=IA,y=delta3_10,col sep=space] {IA_delta.csv};
\addlegendentry{$\rho_{\IA,3}(c)-\varrho_{\IA, 3, 10}(c)$}
\addplot table [x=IA,y=delta3_100,col sep=space] {IA_delta.csv};
\addlegendentry{$\rho_{\IA,3}(c)-\varrho_{\IA, 3, 100}(c)$}
\addplot table [x=IA,y=delta3_1000,col sep=space] {IA_delta.csv};
\addlegendentry{$\rho_{\IA,3}(c)-\varrho_{\IA, 3, 1000}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The difference between $\rho_{\IA, 3}(c)$ and
$\varrho_{\IA, 3, m}(c)$.}
\end{subfigure}
%\hfill
% \begin{subfigure}[b]{0.31\textwidth}
%\resizebox{\textwidth}{!}{
%\begin{tikzpicture}
% \begin{axis}[xmin=0,xmax=1,
% ymin=0,ymax=1,
% xlabel={$x$},
% ylabel={\% of $2\times 2$-matrices (Probability)},
% legend pos=south east,
% legend style={cells={align=left}}]
% \addplot table [x=ir,y=p,col sep=space] {IR_rho.csv};
% \addlegendentry{$\rho_{\IR}(x)$}
%\end{axis}
%\end{tikzpicture}}
%\caption{The function $\rho_{\IR}(x)$.}
%\end{subfigure}
\hfill
\begin{subfigure}[b]{0.49\textwidth}
\resizebox{\textwidth}{!}{
\begin{tikzpicture}
\begin{axis}[ xmin=-1,xmax=1,
ymin=0,ymax=1,
xlabel={$c$},
ylabel={Significativity difference},
legend pos=north west,
legend style={cells={align=left}}]
\addplot table [x=kappa,y=delta3_10,col sep=space] {kappa_delta.csv};
\addlegendentry{$\rho_{\kappa,3}(c)-\varrho_{\kappa, 3, 10}(c)$}
\addplot table [x=kappa,y=delta3_100,col sep=space] {kappa_delta.csv};
\addlegendentry{$\rho_{\kappa,3}(c)-\varrho_{\kappa, 3, 100}(c)$}
\addplot table [x=kappa,y=delta3_1000,col sep=space] {kappa_delta.csv};
\addlegendentry{$\rho_{\kappa,3}(c)-\varrho_{\kappa, 3, 1000}(c)$}
\end{axis}
\end{tikzpicture}}
\caption{The difference between $\rho_{\kappa, 3}(c)$ and
$\varrho_{\kappa, 3, m}(c)$.}
\end{subfigure}
\caption{The difference between $\rho_{\sigma, n}(c)$ and
$\varrho_{\sigma, n, m}(c)$ for $m \in \{10, 100, 1000\}$ when $\sigma$ is
Cohen's $\kappa$ and $\IA$.}\label{fig:error}
\end{figure*}
\section{Conclusion}\label{sec:conclusions}
%Rating an agreement measure can be challenging, especially when dealing with various effect sizes.
%Moreover, we have shown that the absence of any absolute qualitative reference scale for the agreement measures plays a role in the interpretation of their statistical significance.
This work introduces a general technique to evaluate the statistical relevance of agreement values.
Our proposal does not gauge the meaningfulness of the data set used to build a confusion matrix. It instead evaluates the significativity of an agreement value over a data set once the data set size has been set.
We introduced the $\sigma$-significativity of $c$
over $n\times n$ confusion matrices that collect $m$
classifications, $\varrho_{\sigma,n,m}(c)$, as the probability of choosing by chance a
confusion matrix having an agreement value lower than $c$.
This measure is parametetric in the agreement measure $\sigma$, the number of classes $n$, and
the size of the data set $m$.
We also define the $\sigma$-significativity of $c$
over $n\times n$ probability matrices, $\rho_{\sigma,n}(c)$, as the probability of
choosing by chance a probability matrix having an agreement value lower than $c$.
As long as the set of the probability matrices whose agreement value is lower than $c$, $\mathcal{S}_{\sigma, n}(c)$, is definable in an o-minimal theory, the two $\sigma$-significativity
converge as $m$ tends to infinity.
%The computability of the two significativity measures is also addressed.
The $\sigma$-significativity over confusion matrices is computable. However, the asymptotic time complexity of its exact evaluation is so high that it discourages its use. Hence, we suggested a Monte Carlo numerical estimator for it whose complexity is linear in $m$ and quadratic in $n$.
On the other hand, the $\sigma$-significativity over probability matrices depends on $\sigma$ and, in some cases, it is not analytically computable. We proposed a numerical method, with quadratic time complexity in $n$, to estimate this index too. The algorithms have been implemented in \texttt{R} package, named \texttt{rSignificativity}, which is available on \href{https://albertocasagrande.github.io/rSignificativity}{GitHub}. This package was used in combination with PGF/TikZ~\cite{tantau2018pgf} to produce the figures in this manuscript.
%The package is also available to the public as an online tool on \href{https://albertocasagrande.github.io/significativity/}. The online tool will make it easier for researchers who do not have a programming background to produce publishable results.
The notion of $\sigma$-significativity is meant to provide a
statistical significance to the agreement values and not to replace
them. In this spirit, we plan to investigate the relation
between agreement scales, such as the one proposed by Landis and Kock~\cite{Landies1977}, and the $\sigma$-significativity.
We also plan to use $\sigma$-significativity in both AI and clinical domain.
For example, in training AI algorithms for medical image classifications, a parametric scale for any agreement value could assess the consistency between human annotators and algorithmic predictions.
\bibliographystyle{plain}
\bibliography{IT_biblio}
\appendix
\section{Proof of Theorem~\ref{theo:riemann}}\label{sec:proofs_riemann}
While the function $T_P$ maps any confusion matrix into a probability matrix, some probability matrices do not correspond to any confusion matrix because $T_P$ is not bijective.
It is easy to see that, for any confusion matrix $M_C \in \mathbb{N}^{n \times n}$, $T_P(M_C) \in \mathbb{Q}^{n \times n}$. Thus,
the probability matrices with some irrational components are not images of any confusion matrix.
Let $\mathcal{P}_{n, m}$ be the set of probability matrices corresponding to a $n\times n$ confusion matrix whose components
sum up to $m$, i.e.,
\begin{equation}\label{eq:prob_n_m}
\mathcal{P}_{n, m} \defeq T_P\left(\mathcal{M}_{n,m} \right).
\end{equation}
Since $\gamma: \Delta^{(n^2-1)} \rightarrow \mathcal{P}_{n}$ is bijective, we can also define the set
\begin{equation}\label{eq:pC_n_m}
\mathcal{C}_{m,n^2}^*\defeq \gamma^{-1}\left(\mathcal{P}_{n, m} \right).
\end{equation}
By construction, $\mathcal{P}_{n, m}$ is a set of $n\times n$ rational probability matrices (see Eq.~\ref{def:T_P}),
i.e., $\mathcal{P}_{n, m} \subset \mathcal{P}_{n} \cap \mathbb{Q}^{n \times n}$. Hence,
$\mathcal{C}_{m,n^2}^* = \gamma^{-1}\left(\mathcal{P}_{n, m} \right) \subset \gamma^{-1}\left(\mathcal{P}_{n} \right) = \Delta^{(n^2-1)}$
and $\pi(\mathcal{C}_{m,n^2}^*) = \pi(\Delta^{(n^2-1)})$. It is worth to notice that both $\mathcal{P}_{n,m}$ and $\mathcal{C}_{m,n^2}^*$
have finite cardinalities because
$T_P: \mathcal{M}_{n,m} \rightarrow \mathcal{P}_{n,m}$ and $\gamma: \mathcal{P}_{n} \rightarrow \mathcal{C}_{m,n^2}$
are bijective. In particular, $|\mathcal{C}_{m,n^2}^*| = |\mathcal{P}_{n,m}| = |\mathcal{M}_{n,m}| = |\mathcal{C}_{m,n^2}|=\binom{m+n^2-1}{m}$.
The matrices in $\mathcal{P}_{n, m}$ correspond to evenly spread points in $\pi\left( \Delta^{(n^2-1)} \right)$.
\begin{figure}[!ht]
\begin{center}
\tikzset{every mark/.append style={scale=0.5}}
\resizebox{0.45\textwidth}{!}{%
\begin{tikzpicture}
\begin{axis}
[ view={0}{90},
xmin=0,xmax=1,
ymin=0,ymax=1,
zmin=0, zmax=1,
xlabel = {$x_1$},
ylabel = {$x_2$},
zlabel = {$x_3$}
]
\addplot3[blue, only marks] file{./confusion_matrix_2x2_proj.txt};
\end{axis}
\end{tikzpicture}}
\caption{A $2$-dimensional projection of the points in $\pi(\mathcal{C}_{2, 50}^*)$ (the blue set). The points are evenly spread inside $\pi(\Delta^{3})$.
The minimal distance between two distinct points $x, x' \in \pi(\mathcal{C}_{2, 50}^*)$ is $1/50$.}\label{fig:confusion_matrices}
\end{center}
\end{figure}
%
These points
induce a uniform grid with cell side length $1/m$ such that each of the cells included in $\pi\left( \Delta^{(n^2-1)} \right)$ contains one of the points.
\begin{lemma}\label{lemma:distance}
The smallest distance between two distinct vectors in $\pi\left(\mathcal{C}_{m,n^2}^* \right)$ is $1/m$. %, i.e.,
%\begin{equation*}
%\min_{y,y' \in \pi\left(\gamma^{-1}\left(\mathcal{P}_{n, m} \right) \right)\setminus \text{Id}}\Mdist{y}{y'} = \frac{1}{m}.
%\end{equation*}
%
Moreover, if $y,y' \in \pi\left(\mathcal{C}_{m,n^2}^* \right)$ and $\|y-y'\|=1/m$, then all the components
of $y-y'$, but one equal $0$. %, i.e., if $\langle x_1, \ldots, x_{n^2-1}\rangle =y -y'$, then there exists $i \in [1, n^2-1]$ such that
%$|x_i| = 1/m$ and $x_j=0$ for all $j \neq i \in [1, n^2-1]$.
\end{lemma}
\begin{proof}
By construction, any point $\gamma\left(\mathcal{C}_{n,m}\right)$ has the form $\left\langle \frac{c_1}{m}, \ldots \frac{c_{n^2-1}}{m} \right\rangle$ with $c_i \in \mathbb{N}$ and $\sum_{i=1}^{n^2-1}c_i\leq m$. It follows that $c_i \in [1, m]$ for any $i\in [1, n^2-1]$.
If $y=\left\langle \frac{c_1}{m}, \ldots \frac{c_{n^2-1}}{m} \right\rangle$ and $y'=\left\langle \frac{c_1'}{m}, \ldots \frac{c_{n^2-1}'}{m} \right\rangle$, then
\begin{align*}
\Mdist{y}{y'} &= \sqrt{\sum_{j=1}^{n^2-1} \left(\frac{c_j}{m}-\frac{c_j'}{m}\right)^2}= \sqrt{\frac{1}{m^2}\sum_{j=1}^{n^2-1} \left(c_j-c_j'\right)^2}
=\frac{1}{m}\sqrt{\sum_{j=1}^{n^2-1} \left(c_j-c_j'\right)^2}
\end{align*}
Hence, $y \neq y'$ if and only if $c_j \neq c_j'$ for some $j \in [1, n^2-1]$.
Since both $c_j$ and $c_j'$ are natural numbers, if $c_j \neq c_j'$, then $ \left|c_j-c_j'\right| \geq 1$ and $\left(c_j-c_j'\right)^2 \geq 1$.
Thus,
\begin{align*}
\Mdist{y}{y'} &=\frac{1}{m}\sqrt{\sum_{j=1}^{n^2-1} \left(c_j-c_j'\right)^2} \geq \frac{1}{m}\sqrt{k},
\end{align*}
where $k$ is the number of indices $j \in [1, n^2-1]$ such that $c_j \neq c_j'$, i.e., $k \defeq |D|$ where $D \defeq \{j \in [1, n^2-1]\, |\, c_j \neq c_j' \}$.
It follows that the minimum of $\Mdist{y}{y'}$ for $y\neq y'$ equals $1/m$. Moreover,
if $\Mdist{y}{y'}=1/m$, then $k=1$, i.e., $1=|D|$. Hence, $|[1, n^2-1] \setminus D| = n^2-2$ and $c_j = c_j'$ for all $j \in [1, n^2-1] \setminus D$.
As a consequence, $c_j - c_j' = 0$ for $n^2-2$ different indices $j \in [1, n^2-1] \setminus D$ and $|c_j - c_j'| = 1$ for the only $j \in D$.
\end{proof}
Let $\mathcal{R}_{\sigma,n,m}^*(c)$ be the subset of the probability matrices in $\mathcal{S}_{\sigma,n}(c)$ that correspond to a
$n\times n$ confusion matrix with $m$ tests, i.e.,
\begin{equation}\label{eq:pR_n_m}
\mathcal{R}_{\sigma,n,m}^*(c) \defeq \mathcal{S}_{\sigma,n}(c) \cap \mathcal{C}_{m,n^2}^*.
\end{equation}
\begin{lemma}\label{lemma:sigma}
Let $\sigma$ such that $\sigma(M_C) = \sigma(T_P(M_C))$ for all $M_C \in \mathcal{M}_{n,m}$. The sets $\mathcal{R}_{\sigma,n,m}^*(c)$ and $\mathcal{R}_{\sigma,n,m}(c)$ have the same cardinality.
\end{lemma}
\begin{proof}
Since $\mathcal{R}_{\sigma,n,m}^*(c) \defeq \mathcal{S}_{\sigma,n}(c) \cap \mathcal{C}_{m,n^2}^*$,
\begin{align*}
\left|\mathcal{R}_{\sigma,n,m}^*(c)\right| &= \left|\mathcal{S}_{\sigma,n}(c) \cap \mathcal{C}_{m,n^2}^* \right|\\
&= \left|\left\{x \in \Delta^{(n^2-1)} \,|\, \sigma(\gamma(x)) < c \right\} \cap \mathcal{C}_{m,n^2}^* \right|\\
&= \left|\left\{x \in \mathcal{C}_{m,n^2}^* \,|\, \sigma(\gamma(x)) < c \right\}\right|.
\end{align*}
Moreover
\begin{align*}
\left|\mathcal{R}_{\sigma,n,m}^*(c)\right| &= \left|\left\{x \in \mathcal{C}_{m,n^2}^* \,|\, \sigma(\gamma(x)) < c \right\}\right|\\
&= \left|\left\{x \in \gamma^{-1}\left(\mathcal{P}_{n,m}\right) \,|\, \sigma(\gamma(x)) < c \right\}\right|
\end{align*}
because $\mathcal{C}_{m,n^2}^* \defeq \gamma^{-1}\left(\mathcal{P}_{n,m}\right)$.
Hence,
\begin{align*}
\left|\mathcal{R}_{\sigma,n,m}^*(c)\right| &= \left|\left\{x \in \gamma^{-1}\left(\mathcal{P}_{n,m}\right) \,|\, \sigma(\gamma(x)) < c \right\}\right|\\
&= \left|\left\{M_P \in \mathcal{P}_{n,m}\,|\, \sigma(M_P) < c \right\}\right|\\
&= \left|\left\{M_P \in T_P\left(\mathcal{M}_{n,m}\right)\,|\, \sigma(M_P) < c \right\}\right|
\end{align*}
because $\gamma: P_n \rightarrow \Delta^{(n^2-1)}$ is bijective and $\mathcal{P}_{n,m} \defeq T_P\left(\mathcal{M}_{n,m}\right)$.
Since $T_P: \mathcal{M}_{n,m} \rightarrow \mathcal{P}_{n,m}$ is bijective,
\begin{align*}
\left|\mathcal{R}_{\sigma,n,m}^*(c)\right|
&= \left|\left\{M_P \in T_P\left(\mathcal{M}_{n,m}\right)\,|\, \sigma(M_P) < c \right\}\right|\\
&= \left|\left\{M_C \in \mathcal{M}_{n,m}\,|\, \sigma(T_P(M_C)) < c \right\}\right|.
\end{align*}
However,
\begin{align*}
\left|\mathcal{R}_{\sigma,n,m}^*(c)\right|
&= \left|\left\{M_C \in \mathcal{M}_{n,m}\,|\, \sigma(T_P(M_C)) < c \right\}\right|\\
&= \left|\left\{M_C \in \mathcal{M}_{n,m}\,|\, \sigma(M) < c \right\}\right|
\end{align*}
because $\sigma(T_P(M))=\sigma(M)$ for any $M_C \in \mathcal{M}_{n,m}$.
Finally,
\begin{align*}
\left|\mathcal{R}_{\sigma,n,m}^*(c)\right|
&= \left|\left\{M_C \in \mathcal{M}_{n,m}\,|\, \sigma(M) < c \right\}\right|\\
&= \left|\left\{M_C \in \gamma^{-1}\left(\mathcal{C}_{n,m}\right) \,|\, \sigma(M_C) < c \right\}\right|\\
&= \left|\left\{x \in \mathcal{C}_{n,m} \,|\, \sigma(\gamma(x)) < c \right\}\right|\\
&= \left|\mathcal{R}_{\sigma,n,m}(c)\right|.
\end{align*}
Thus, the thesis holds.
\end{proof}
%Since both $\mathcal{P}_{n,m}$ is a finite set and $|\mathcal{P}_{n,m}|=|\mathcal{C}_{m,n^2}^*|$, we can define the ratio between
%the number of probability matrices $M \in \mathcal{P}_{n,m}$ such that $\sigma(\gamma(M)) < c$ and $|\mathcal{P}_{n,m}|$ as:
%\begin{equation}
%\varrho_{\sigma,n,m}(c)^*=\frac{\left|\mathcal{R}_{\sigma,n,m}^*(c)\right|}{\left|\mathcal{C}_{m,n^2}^*\right|}
%\end{equation}
\begin{lemma}\label{lemma:riemann}
If $A \subseteq \Delta^{(n^2-1)}$ is Riemann-measurable in dimension $n^2-1$, then
\begin{equation}
V(A) = \lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left|A \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|
\end{equation}
\end{lemma}
\begin{proof}
Let us consider the $(n^2-1)$-dimensional grid of $[0,1]^{n^2-1}$ having the cells
\begin{equation}\label{eq:grid}
Q_{c_1, \ldots, c_{n^2-1}} \defeq \prod_{i=1}^{n^2-1} \left[ \frac{2 c_i-1}{2m}, \frac{2 c_i + 1}{2m} \right]
\end{equation}
where $c_i \in [0, m]$ for any $i \in [1, n^2-1]$. Every cell $Q_{c_1, \ldots, c_{n^2-1}}$ has Lebesgue measure $1/m^{n^2-1}$ and
its interior, $\interior{Q_{c_1, \ldots, c_{n^2-1}}}$, contains
$\langle c_1/m, \ldots, c_{n^2-1}/m\rangle$.
Since $T_P\left(\mathcal{M}_{n,m}\right)=\mathcal{P}_{n,m}$, %because of Eq.~\ref{eq:prob_n_m}
$\gamma\left(\mathcal{C}_{m,n^2}\right)=\mathcal{M}_{n,m}$, and $\mathcal{C}_{m,n^2}^*=\gamma^{-1}\left(\mathcal{P}_{n,m}\right)$,
the vector $\langle c_1, \ldots, c_{n^2-1}\rangle$ belongs to $\pi\left(\mathcal{C}_{m,n^2}\right)$ if and only if
$\pi\left(\mathcal{C}_{m,n^2}^*\right)$ contains $\langle c_1/m, \ldots, c_{n^2-1}/m\rangle$.
Moreover, the maximal distance between the vector $\langle c_1/m, \ldots, c_{n^2-1}/m\rangle$ and any point on the border of
$Q_{c_1, \ldots, c_{n^2-1}}$ is $1/(\sqrt{2} m)$, and Lemma~\ref{lemma:distance} proves that the minimal distance
between two distinct vectors in $\pi\left(\mathcal{C}_{m,n^2}^*\right)$ is $1/m$.
Hence, $\langle c_1, \ldots, c_{n^2-1}\rangle \in \pi\left(\mathcal{C}_{m,n^2}\right)$ if and only if the set
$Q_{c_1, \ldots, c_{n^2-1}} \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)$ is the singleton $\{\langle c_1/m, \ldots, c_{n^2-1}/m\rangle\}$.
Moreover, $\langle c_1, \ldots, c_{n^2-1}\rangle \not\in \pi\left(\mathcal{C}_{m,n^2}\right)$ if and only if
$Q_{c_1, \ldots, c_{n^2-1}} \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)=\emptyset$.
For any set Riemann-measurable set $A \subseteq \pi(\Delta^{(n^2-1)})$, we can define the union, $\overline{J}(A,m)$,
of the cells $Q_{c_1, \ldots, c_{n^2-1}}$ that are not disjoint from $A$, i.e,
$$
\overline{J}(A,m) \defeq \bigcup_{ A \cap Q_{c_1, \ldots, c_{n^2-1}} \neq 0} Q_{c_1, \ldots, c_{n^2-1}}.
$$
The interior of $\overline{J}(A,m)$ over-approximates $A$, i.e., $\interior{\overline{J}(A,m)} \supseteq A$.
Since every cell $Q_{c_1, \ldots, c_{n^2-1}}$ has Lebesgue measure $1/m^{n^2-1}$ and contains exactly one
of the vectors in $\pi\left(\mathcal{C}_{m,n^2}^*\right)$, the Lebesgue measure of $\overline{J}(A,m)$
is $1/m^{n^2-1}$ multiplied by the number of vectors in $\pi\left(\mathcal{C}_{m,n^2}^*\right) \cap \overline{J}(A,m)$, i.e.,
\begin{align*}
V(\overline{J}(A,m)) &= \frac{1}{m^{n^2-1}}\sum_{ A \cap Q_{c_1, \ldots, c_{n^2-1}} \neq 0} 1= \frac{1}{m^{n^2-1}} \left|\overline{J}(A,m) \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|.
\end{align*}
Since $\overline{J}(A,m) \supseteq A$, $A \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)$ is a subset of $\overline{J}(A,m) \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)$ and
$V(\overline{J}(A,m)) \geq 1/m^{n^2-1} \left|A \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|$.
Analogously,
the union, $\underline{J}(A,m)$, of the cells $Q_{c_1, \ldots, c_{n^2-1}}$ that are subsets of the interior of $A$, i.e,
$$
\underline{J}(A,m) \defeq \bigcup_{ \interior{A} \subseteq Q_{c_1, \ldots, c_{n^2-1}}} Q_{c_1, \ldots, c_{n^2-1}},
$$
then $\underline{J}(A,m)$ under-approximates $A$, i.e., $\underline{J}(A,m) \subseteq A$.
Since $\interior{Q_{c_1, \ldots, c_{n^2-1}}}$ contains
$\langle c_1/m, \ldots, c_{n^2-1}/m\rangle$,
the Lebesgue measure of $\underline{J}(A,m)$
is $1/m^{n^2-1}$ multiplied by the number of vectors in $\pi\left(\mathcal{C}_{m,n^2}^*\right)$
that also belong to $\underline{J}(A,m)$, i.e.,
\begin{align*}
V(\underline{J}(A,m)) &= \frac{1}{m^{n^2-1}}\sum_{ \interior{A} \subseteq Q_{c_1, \ldots, c_{n^2-1}}} 1 = \frac{1}{m^{n^2-1}} \left|\underline{J}(A,m) \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|.
\end{align*}
Since $\underline{J}(A,m) \subseteq A$, $\underline{J}(A,m) \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)$ is a subset of $A \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)$ and
$V(\underline{J}(A,m)) \leq 1/m^{n^2-1} \left|A \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|$.
We know that $\underline{J}(A,m) \subseteq A \subseteq \interior{\overline{J}(A,m)}$. Thus, $V(\underline{J}(A,m)) \leq V(A) \leq V(\overline{J}(A,m))$. As a consequence,
\begin{equation}
V(\underline{J}(A,m)) \leq \frac{1}{m^{n^2-1}} \left|A \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right| \leq V(\overline{J}(A,m)).
\end{equation}
However, $\lim_{m \rightarrow +\infty}V(\underline{J}(A,m)) = \lim_{m \rightarrow +\infty} V(\overline{J}(A,m))$ because $A$ is Riemann-measurable.
From the squeeze theorem, it follows that
\begin{equation}
V(A) = \lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left|A \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|.
\end{equation}
Hence, the thesis holds.
\end{proof}
%\begin{theorem}
%If $\mathcal{S}_{\sigma,n}(c)$ is Riemann-measurable in dimension $n^2-1$, then
%\begin{equation}
%\lim_{m\rightarrow +\infty}\frac{|\mathcal{R}_{\sigma,n,m}^*(c)|}{|\mathcal{C}_{m,n^2}^*|}=\rho_{\sigma, n}(c).
%\end{equation}
%\end{theorem}
We are now ready to prove Theorem~\ref{theo:riemann}.
\begin{proof}
Since $\mathcal{S}_{\sigma, n}(c)$ and $\mathcal{C}_{m,n^2}^*$ are subsets of
$\Delta^{(n^2-1)}$, and since $\pi$ is bijective,
\begin{align*}
\pi\left(\mathcal{S}_{\sigma, n}(c)\right) \cap \pi\left(\mathcal{C}_{m,n^2}^*\right) = \pi\left(\mathcal{S}_{\sigma, n}(c) \cap \mathcal{C}_{m,n^2}^*\right)
\end{align*}
Hence,
\begin{align*}
V(\pi\left(\mathcal{S}_{\sigma, n}(c)\right)) &=\lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \pi\left(\mathcal{S}_{\sigma, n}(c)\right) \cap \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|\\
&= \lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \pi\left(\mathcal{S}_{\sigma, n}(c) \cap \mathcal{C}_{m,n^2}^*\right)\right|\\
&= \lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \pi\left(\mathcal{R}_{\sigma,n,m}^*(c)\right)\right|\\
&= \lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \mathcal{R}_{\sigma,n,m}^*(c)\right|
\end{align*}
because of Lemma~\ref{lemma:riemann} and Eq.~\ref{eq:pR_n_m}.
However,
\begin{align*}
\pi\left(\Delta^{(n^2-1)}\right) \cap \pi\left(\mathcal{C}_{m,n^2}^*\right) = \pi\left(\mathcal{C}_{m,n^2}^*\right)
\end{align*}
because $\mathcal{C}_{m,n^2}^* \subseteq \Delta^{(n^2-1)}$ and because $\pi$ is bijective.
Thus,
\begin{align*}
V(\pi\left(\Delta^{(n^2-1)}\right)) = \lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \pi\left(\mathcal{C}_{m,n^2}^*\right)\right|
= \lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \mathcal{C}_{m,n^2}^*\right|
\end{align*}
because of Lemma~\ref{lemma:riemann}. It follows that
\begin{align*}
\rho_{\sigma, n}(c)=\frac{V(\mathcal{S}_{\sigma, n}(c))}{V(\Delta^{(n^2-1)})}=\frac{\lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \mathcal{R}_{\sigma,n,m}^*(c)\right|}{\lim_{m \rightarrow +\infty} \frac{1}{m^{n^2-1}} \left| \mathcal{C}_{m,n^2}^*\right|}=\lim_{m \rightarrow +\infty}\frac{\left| \mathcal{R}_{\sigma,n,m}^*(c)\right|}{\left| \mathcal{C}_{m,n^2}^*\right|}.
\end{align*}
Since $\left| \mathcal{C}_{m,n^2}^*\right|=\left| \mathcal{C}_{m,n^2}\right|$,
\begin{align*}
\rho_{\sigma, n}(c)=\lim_{m \rightarrow +\infty}\frac{\left| \mathcal{R}_{\sigma,n,m}^*(c)\right|}{\left| \mathcal{C}_{m,n^2}^*\right|}
=\lim_{m \rightarrow +\infty}\frac{\left| \mathcal{R}_{\sigma,n,m}(c)\right|}{\left| \mathcal{C}_{m,n^2}\right|}
=\lim_{m \rightarrow +\infty} \varrho_{\sigma, n, m}(c).
\end{align*}
because of Lemma~\ref{lemma:sigma} and Eq.~\ref{eq:varrho}.
\end{proof}
\end{document}
\begin{thebibliography}{10}
\bibitem{Apostol:105425}
Tom~M Apostol.
\newblock {\em Mathematical analysis; 2nd ed.}
\newblock Addison-Wesley series in mathematics. Addison-Wesley, Reading, MA,
1974.
\bibitem{10.1145/3460772}
David Blackman and Sebastiano Vigna.
\newblock Scrambled linear pseudorandom number generators.
\newblock {\em ACM Trans. Math. Softw.}, 47(4), September 2021.
\bibitem{bona2016}
Mikl{\'o}s B{\'o}na.
\newblock {\em A walk through combinatorics: An introduction to enumeration and
graph theory, second edition}.
\newblock World Scientific Publishing Company, Chennai, India, 4th edition,
2016.
\bibitem{boyd2004convex}
Stephen Boyd and Lieven Vandenberghe.
\newblock {\em Convex optimization}.
\newblock Cambridge university press, Cambridge, UK, 2004.
\bibitem{Casagrande:2020a}
Alberto Casagrande, Francesco Fabris, and Rossano Girometti.
\newblock Beyond kappa: an informational index for diagnostic agreement in
dichotomous and multivalue ordered-categorical ratings.
\newblock {\em Medical and Biological Engineering and Computing},
58:3089--3099, 2020.
\bibitem{Casagrande:2020b}
Alberto Casagrande, Francesco Fabris, and Rossano Girometti.
\newblock Extending information agreement by continuity.
\newblock In {\em 2020 IEEE International Conference on Bioinformatics and
Biomedicine (BIBM)}, pages 1432--1439, 2020.
\bibitem{Casagrande:2022}
Alberto Casagrande, Francesco Fabris, and Rossano Girometti.
\newblock Fifty years of shannon information theory in assessing the accuracy
and agreement of diagnostic tests.
\newblock {\em Medical \& Biological Engineering \& Computing}, 60(4):941--955,
2022.
\bibitem{Casagrande:2023}
Alberto Casagrande, Francesco Fabris, and Rossano Girometti.
\newblock An information-oriented paradigm in evaluating accuracy and agreement
in radiology.
\newblock {\em European Radiology Experimental}, 7(1):1--7, 2023.
\bibitem{Casagrande:2024aa}
Alberto Casagrande, Francesco Fabris, and Rossano Girometti.
\newblock A prevalence-robust measure of diagnostic test performance.
\newblock {\em Network Modeling Analysis in Health Informatics and
Bioinformatics}, 14(1):12, 2024.
\bibitem{Cohen:1960}
Jacob Cohen.
\newblock A coefficient of agreement for nominal scales.
\newblock {\em Educational and Psychological Measurement}, 20:37--46, 1960.
\bibitem{Cohen:1968}
Jacob Cohen.
\newblock {Weighted kappa: Nominal scale agreement with provision for scaled
disagreement or partial credit}.
\newblock {\em Psychological bulletin}, 70(4):213--220, October 1968.
\bibitem{cormen2022introduction}
Tomas~H. Cormen, Charles~E. Leiserson, Ronald~L. Rivest, and Clifford Stein.
\newblock {\em Introduction to Algorithms}.
\newblock MIT Press, Cambridge, MA, USA, 4th edition, 2022.
\bibitem{devroye:1986}
Luc Devroye.
\newblock {\em Non-Uniform Random Variate Generation}.
\newblock Springer-Verlag, New York, NY, USA, 1986.
\bibitem{birads2013}
Carl~J. D'Orsi, Edward~A. Sickles, Ellen~B. Mendelson, and Elizabeth~A. Morris.
\newblock {\em 2013 ACR BI-RADS Atlas: Breast Imaging Reporting and Data
System}.
\newblock American College of Radiology, Reston, VA, USA, 2014.
\bibitem{Elamin2024}
Aisha Elamin and Suneeta Teckchandani.
\newblock {\em Hypoglycemia}, pages 329--338.
\newblock Springer Nature Singapore, Singapore, 2024.
\bibitem{dda9a3b2-7e66-38cd-8ce8-7d36cd60b878}
R.~A. Fisher.
\newblock On the interpretation of $\chi^2$ from contingency tables, and the
calculation of p.
\newblock {\em Journal of the Royal Statistical Society}, 85(1):87--94, 1922.
\bibitem{fisher1925statistical}
Ronald~Aylmer Fisher.
\newblock {\em Statistical methods for research workers}.
\newblock Oliver \& Boyd, Edinburgh, UK, 1925.
\bibitem{fleiss2013statistical}
J.L. Fleiss, B.~Levin, and M.C. Paik.
\newblock {\em Statistical Methods for Rates and Proportions}.
\newblock Wiley Series in Probability and Statistics. Wiley\&Sons, Hoboken, NJ,
USA, 2013.
\bibitem{Fleiss71}
Joseph Fleiss.
\newblock Measuring nominal scale agreement among many raters.
\newblock {\em Psychological Bulletin}, 76:378--382, 11 1971.
\bibitem{Fraenkel1958}
Abraham~Adolf Fraenkel and Yehoshua Bar{-}Hillel.
\newblock {\em Foundations of Set Theory}.
\newblock North-Holland Pub.~Co, Amsterdam, 1958.
\bibitem{7780459}
Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun.
\newblock Deep residual learning for image recognition.
\newblock In {\em 2016 IEEE Conference on Computer Vision and Pattern
Recognition (CVPR)}, pages 770--778, 2016.
\bibitem{hinton2015distillingknowledgeneuralnetwork}
Geoffrey Hinton, Oriol Vinyals, and Jeff Dean.
\newblock Distilling the knowledge in a neural network, 2015.
\bibitem{hubert1985comparing}
Lawrence Hubert and Phipps Arabie.
\newblock Comparing partitions.
\newblock {\em Journal of Classification}, 2(1):193--218, 1985.
\bibitem{krippendorff2004reliability}
Klaus Krippendorff.
\newblock Reliability in content analysis: Some common misconceptions and
recommendations.
\newblock {\em Human Communication Research}, 30(3):411--433, 2004.
\bibitem{Landies1977}
J.~Richard Landis and Gary~G. Koch.
\newblock The measurement of observer agreement for categorical data.
\newblock {\em Biometrics}, 33(1):159--174, 1977.
\bibitem{726791}
Y.~Lecun, L.~Bottou, Y.~Bengio, and P.~Haffner.
\newblock Gradient-based learning applied to document recognition.
\newblock {\em Proceedings of the IEEE}, 86(11):2278--2324, 1998.
\bibitem{MA2025110852}
Baoqiang Ma, Jiapan Guo, Lisanne~V. {van Dijk}, Johannes~A. Langendijk,
Peter~M.A. {van Ooijen}, Stefan Both, and Nanna~M. Sijtsema.
\newblock {PET and CT based DenseNet outperforms advanced deep learning models
for outcome prediction of oropharyngeal cancer}.
\newblock {\em Radiotherapy and Oncology}, 207:110852, 2025.
\bibitem{Mackay1998}
D.~J.~C. Mackay.
\newblock {\em Introduction to Monte Carlo Methods}, pages 175--204.
\newblock Springer Netherlands, Dordrecht, 1998.
\bibitem{10.1145/272991.272995}
Makoto Matsumoto and Takuji Nishimura.
\newblock Mersenne twister: a 623-dimensionally equidistributed uniform
pseudo-random number generator.
\newblock {\em ACM Trans. Model. Comput. Simul.}, 8(1):3--30, January 1998.
\bibitem{McNemar:1947}
Quinn McNemar.
\newblock Note on the sampling error of the difference between correlated
proportions or percentages.
\newblock {\em Psychometrika}, 12(2):153--157, Jun 1947.
\bibitem{montecarlo}
Nicholas Metropolis and Stanis\l{}aw Ulam.
\newblock {The Monte Carlo Method}.
\newblock {\em {Journal of the American Statistical Association}},
44(247):335--341, 1949.
\bibitem{monti2003consensus}
Stefano Monti, Pablo Tamayo, Jill Mesirov, and Todd Golub.
\newblock Consensus clustering: A resampling-based method for class discovery
and visualization of gene expression microarray data.
\newblock {\em Machine Learning}, 52(1):91--118, 2003.
\bibitem{10.1145/1132973.1132974}
Fran\c{c}ois Panneton, Pierre L'Ecuyer, and Makoto Matsumoto.
\newblock Improved long-period generators based on linear recurrences modulo 2.
\newblock {\em ACM Trans. Math. Softw.}, 32(1):1--16, March 2006.
\bibitem{presburger1929}
Moj\.zesz Presburger.
\newblock {\"Uber die Vollst\"andigkeit eines gewissen Systems der Arithmetik
ganzer Zahlen, in welchem die Addition als einzige Operation hervortritt}.
\newblock In {\em Comptes Rendus du I congr\`es de Math\'ematiciens des Pays
Slaves, Warszawa}, pages 92--101, 1929.
\bibitem{10.1086/266577}
William~A. Scott.
\newblock Reliability of content analysis:the case of nominal scale coding.
\newblock {\em Public Opinion Quarterly}, 19(3):321--325, 01 1955.
\bibitem{Shrout:1979}
{Patrick E.} Shrout and {Joseph L.} Fleiss.
\newblock {Intraclass correlations: Uses in assessing rater reliability}.
\newblock {\em Psychological Bulletin}, 86(2):420--428, 3 1979.
\bibitem{speissegger1999pfaffian}
Patrick Speissegger.
\newblock The {Pfaffian} closure of an o-minimal structure.
\newblock {\em Journal f\"ur die reine und angewandte Mathematik},
1999(508):189--211, 1999.
\bibitem{simplex}
P.~Stein.
\newblock A note on the volume of a simplex.
\newblock {\em The American Mathematical Monthly}, 73(3):299--301, 1966.
\bibitem{tantau2018pgf}
Till Tantau.
\newblock {\em The PGF and TikZ package: Manual for version 3.1.5}.
\newblock Institute for Theoretical Computer Science, Universität zu Lübeck,
2018.
\bibitem{Tarski51}
Alfred Tarski.
\newblock {\em A Decision Method for Elementary Algebra and Geometry}.
\newblock University of California Press, Oakland, CA, USA, 1951.
\bibitem{Dries_1998}
Lou P.~D. {van den Dries}.
\newblock {\em Tame Topology and O-minimal Structures}.
\newblock London Mathematical Society Lecture Note Series. Cambridge University
Press, Cambridge, UK, 1998.
\bibitem{PIRADS2}
Jeffrey~C. Weinreb, Jelle~O. Barentsz, Peter~L. Choyke, Francois Cornud,
Masoom~A. Haider, Katarzyna~J. Macura, Daniel Margolis, Mitchell~D. Schnall,
Faina Shtern, Clare~M. Tempany, Harriet~C. Thoeny, and Sadna Verma.
\newblock {PI-RADS} {Prostate} {Imaging} -- {Reporting} and {Data System}:
2015, version 2.
\newblock {\em European Urology}, 69(1):16--40, 2016.
\bibitem{Yule12}
G.~Udny Yule.
\newblock On the methods of measuring association between two attributes.
\newblock {\em Journal of the Royal Statistical Society}, 75(6):579--652, 1912.
\end{thebibliography}