generate_data.py

import numpy as np

from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
from sklearn.utils import shuffle

# get shuffled iris data
X, y = load_iris(return_X_y=True)
X, y = shuffle(X, y, random_state=0)

spl = 10

y_true = y[spl:]
clf = LogisticRegression(C=1)
clf.fit(X[:spl], y[:spl])
# clf.fit(X, y)
df_pred = clf.decision_function(X[spl:])
probas_pred = clf.predict_proba(X[spl:])
np.savetxt("y_true.txt", y_true)
np.savetxt("df.txt", df_pred)
np.savetxt("probas.txt", probas_pred)

manual.py

"""
multiclass AUC sanity check

Unlike the binary setting, where AUC can use any score and doesn't require
probability estimates, in the multi-output case this might not be the case.

I wrote this naive implementation, following:
  
  An experimental comparison of performance measures for classification
  C. Ferri, , J. Hernández-Orallo , R. Modroiu 

and things are not clear at all...
"""
import numpy as np

y_true = np.loadtxt("y_true.txt").astype(np.int)
df_pred = np.loadtxt("df.txt")
probas_pred = np.loadtxt("probas.txt")

# define helpers

def auc_two_classes(y_true, probas, a, b):
    n_samples = len(y_true)

    auc = 0
    n_a = np.sum(y_true == a)
    n_b = np.sum(y_true == b)

    for i in range(n_samples):
        for j in range(n_samples):

            if not (y_true[i] == a and y_true[j] == b):
                continue

            if probas[i, a] > probas[j, a]:
                auc += 1

            # below is just to match paper exactly
            # in practice in real life this shouldn't happen
            elif probas[i, a] == probas[j, a]:
                auc += 0.5

    return auc / (n_a * n_b)


def auc_ovr_one(y_true, probas, a):
    n_samples = len(y_true)
    n_a = np.sum(y_true == a)
    n_not_a = n_samples - n_a

    auc = 0

    for i in range(n_samples):
        for j in range(n_samples):

            if not (y_true[i] == a and y_true[j] != a):
                continue

            if probas[i, a] > probas[j, a]:
                auc += 1
            elif probas[i, a] == probas[j, a]:
                auc += 0.5

    return auc / (n_a * n_not_a)


print("Checking the primitives in isolation:")
print("=====================================")
print("ovo proba", auc_two_classes(y_true, probas_pred, 2, 1))
print("ovo df   ", auc_two_classes(y_true, df_pred, 2, 1))
print()
print("ovr proba", auc_ovr_one(y_true, probas_pred, 2))
print("ovr df   ", auc_ovr_one(y_true, df_pred, 2))


print("Aggregated metrics")
print("==================")

def mc_auc_ovo(y_true, probas, weighted=False, distinct=False):

    classes = np.unique(y_true)
    scores = []

    for pos_cls in classes:
        for neg_cls in classes:

            if pos_cls == neg_cls:
                continue
            if distinct and pos_cls > neg_cls:
                continue

            auc = auc_two_classes(y_true, probas, pos_cls, neg_cls)
            if weighted:
                auc *= np.mean(y_true == pos_cls)

            scores.append(auc)

    return np.mean(scores)


def mc_auc_ovr(y_true, probas, weighted=False):

    classes = np.unique(y_true)
    scores = []

    for pos_cls in classes:
        auc = auc_ovr_one(y_true, probas, pos_cls)
        if weighted:
            auc *= np.mean(y_true == pos_cls)

        scores.append(auc)

    return np.mean(scores)


print("not weighted")
print("ovo proba         ", mc_auc_ovo(y_true, probas_pred))
print("ovo df            ", mc_auc_ovo(y_true, df_pred))
print("ovo proba distinct", mc_auc_ovo(y_true, probas_pred, distinct=True))
print("ovo df    distinct", mc_auc_ovo(y_true, df_pred, distinct=True))
print("\nweighted")
print("ovo proba         ", mc_auc_ovo(y_true, probas_pred, weighted=True))
print("ovo df            ", mc_auc_ovo(y_true, df_pred, weighted=True))
print("ovo proba distinct", mc_auc_ovo(y_true, probas_pred, weighted=True, distinct=True))
print("ovo df    distinct", mc_auc_ovo(y_true, df_pred, weighted=True, distinct=True))

print("\nnot weighted")
print("ovr proba", mc_auc_ovr(y_true, probas_pred))
print("ovr df   ", mc_auc_ovr(y_true, df_pred))
print("\nweighted")
print("ovr proba", mc_auc_ovr(y_true, probas_pred, weighted=True))
print("ovr df   ", mc_auc_ovr(y_true, df_pred, weighted=True))

manual_stdout

Checking the primitives in isolation:
=====================================
ovo proba 0.959759481961
ovo df    0.984736355227

ovr proba 0.980096087852
ovr df    0.99245024022
Aggregated metrics
==================
not weighted
ovo proba          0.968393462843
ovo df             0.919904409497
ovo proba distinct 0.950508788159
ovo df    distinct 0.850138760407

weighted
ovo proba          0.322900422889
ovo df             0.306998039734
ovo proba distinct 0.317071935157
ovo df    distinct 0.284093211753

not weighted
ovr proba 0.968465578467
ovr df    0.919931763699

weighted
ovr proba 0.322924633134
ovr df    0.30700722293

probas.txt

1.803132918688412842e-02 4.606640351438364700e-01 5.213046356692794259e-01
1.164864338681035477e-01 5.139747122814198876e-01 3.695388538504766340e-01
6.962455532898063904e-02 5.373395974678950582e-01 3.930358472031243999e-01
7.204021495267802500e-02 5.730744881104643529e-01 3.548852969368575527e-01
6.991651423936211251e-02 5.016346670847556810e-01 4.284488186758823036e-01
6.796311209392739139e-01 2.914934081008391242e-01 2.887547095988688556e-02
8.262824295603955760e-02 4.880215830381079423e-01 4.293501740058524585e-01
7.398501883572305982e-02 4.879481018123815739e-01 4.380668793518953663e-01
6.760734510633202854e-01 2.901311842779138450e-01 3.379536465876599449e-02
7.964708632149816170e-01 1.866745367851723181e-01 1.685459999984592616e-02
3.474691957121457675e-02 3.583703606139978515e-01 6.068827198147875857e-01
8.509435872920290134e-02 4.105753551444020211e-01 5.043302861263950776e-01
6.999722292834740234e-01 2.523641388666046370e-01 4.766363184992129798e-02
6.724835245769099634e-01 2.833093367447748201e-01 4.420713867831525118e-02
4.742284428346629777e-02 4.720497609986373244e-01 4.805273947178964056e-01
7.972846354438156702e-01 1.805030425757565560e-01 2.221232198042776343e-02
7.576134338199155271e-01 2.024869354006992028e-01 3.989963077938533248e-02
1.153947575765059547e-01 5.566877405532142387e-01 3.279175018702797928e-01
1.803781001584012500e-01 5.243874717740818481e-01 2.952344280675168742e-01
7.203726347619267001e-01 2.446797506298435121e-01 3.494761460822982252e-02
2.840122241793625879e-02 4.141106231978258223e-01 5.574881543842380438e-01
8.218635408520580388e-02 3.799898958740794974e-01 5.378237500407146987e-01
7.333655213140608531e-01 2.441271300549988921e-01 2.250734863094028607e-02
5.032928040812036791e-02 4.254589432527108861e-01 5.242117763391687113e-01
1.550184801583563178e-02 3.993522229867868534e-01 5.851459289973774958e-01
1.227940242209528549e-01 4.352132217817513138e-01 4.419927539972958730e-01
7.476631547670579137e-01 2.338983238461015135e-01 1.843852138684076020e-02
3.216204087711193454e-02 4.529936242371294663e-01 5.148443348857587587e-01
1.245258750085084798e-01 4.765967938070554411e-01 3.988773311844361347e-01
1.192026185569960983e-01 5.591987924594459258e-01 3.215985889835579203e-01
2.366219632948434945e-02 4.481823387465238562e-01 5.281554649239917909e-01
7.196541135895072605e-01 2.507540978848702418e-01 2.959178852562241793e-02
3.909239341050988609e-02 4.617561189318676318e-01 4.991514876576225168e-01
7.375121314542560924e-01 2.225991539329158220e-01 3.988871461282808561e-02
7.550244151239472457e-01 2.266560410284501892e-01 1.831954384760245066e-02
1.153823581648319579e-01 5.574713726861941776e-01 3.271462691489737673e-01
2.616602157927563552e-02 4.394664357747015693e-01 5.343675426460229305e-01
2.011962733497532629e-02 3.437465036022248621e-01 6.361338690627998949e-01
2.356743774802111738e-02 5.160735303377029792e-01 4.603590319142759624e-01
1.160875520226575981e-02 4.882289855278042490e-01 5.001622592699300762e-01
1.228251583333377084e-01 5.567369750652966243e-01 3.204378666013656396e-01
1.189080161622243191e-02 5.015879418532329659e-01 4.865212565305445658e-01
9.673743258885576790e-02 4.577343850440076523e-01 4.455281823671367047e-01
1.230344888459725949e-01 5.588584609303267614e-01 3.181070502237006714e-01
3.882201672850698476e-02 4.897136809266268154e-01 4.714643023448661374e-01
3.902165918924874882e-02 4.985770376442009155e-01 4.624013031665503703e-01
4.332021270366276378e-02 3.567048002602143386e-01 5.999749870361230641e-01
2.703642418326698479e-02 4.889071742289862188e-01 4.840564015877466542e-01
1.049056515358237629e-01 6.119707730405120616e-01 2.831235754236641755e-01
3.404631036061477739e-02 4.395454632239478854e-01 5.264082264154372748e-01
1.263449658738045323e-01 4.042605744652454924e-01 4.693944596609500586e-01
6.755259861647301411e-01 2.869937589526130317e-01 3.748025488265698674e-02
2.698986331964879984e-02 3.945006469308862118e-01 5.785094897494650334e-01
9.852202960248160335e-02 4.929655905239373470e-01 4.085123798735810774e-01
2.148561558116891257e-01 4.836801409144520503e-01 3.014637032738588518e-01
1.004025625920916404e-01 5.146851053538687992e-01 3.849123320540395743e-01
8.627849163417691813e-02 4.967682219648476005e-01 4.169532864009755091e-01
1.942164122078753052e-02 4.796703442854525212e-01 5.009080144937599899e-01
7.593499392344295051e-01 2.207428107255730299e-01 1.990725003999744766e-02
7.158870334752654996e-01 2.516257734679558933e-01 3.248719305677871122e-02
2.327791126635296581e-02 4.602934436966450438e-01 5.164286450370019210e-01
1.373234955340776886e-01 4.783319859752112602e-01 3.843445184907110512e-01
6.949518286056532990e-01 2.812964698926803675e-01 2.375170150166635089e-02
8.386990148111839183e-01 1.469149798064622037e-01 1.438600538235389366e-02
7.239047106926639608e-02 5.820036726150982220e-01 3.456058563156353403e-01
6.913473444191029493e-01 2.687328522557725363e-01 3.991980332512447965e-02
3.836663326384131406e-02 3.826504868278536309e-01 5.789828799083049926e-01
2.652124409239845493e-01 4.946029764302026233e-01 2.401845826458129107e-01
7.481985532669748196e-01 2.150897137032159179e-01 3.671173302980923481e-02
4.370094537430890824e-02 5.878059939970771142e-01 3.684930606286138111e-01
1.797272630225753981e-02 3.483135002479387787e-01 6.337137734498037300e-01
1.202147681098546944e-01 4.817180954296959583e-01 3.980671364604493889e-01
6.746797528617258788e-01 2.963890837008006374e-01 2.893116343747341099e-02
1.526562037062645025e-02 4.584245961983302031e-01 5.263097834310432876e-01
2.079394001137311113e-02 4.028698258530724052e-01 5.763362341355544594e-01
1.502435564067106967e-02 3.783102481392631722e-01 6.066653962200657668e-01
1.626315747998946862e-02 4.089565242581552629e-01 5.747803182618552720e-01
7.762666082726898953e-01 1.983548098660834758e-01 2.537858186122670875e-02
7.261697897944627833e-01 2.385650036252726924e-01 3.526520658026453120e-02
4.677088296615945079e-02 4.236227174528874317e-01 5.296063995809531244e-01
2.453827455919246545e-02 3.855208782716869065e-01 5.899408471691206524e-01
5.715629488226829924e-01 3.837877882800596607e-01 4.464926289725740932e-02
3.230205132690781855e-02 3.036272648966615306e-01 6.640706837764307480e-01
6.715056164107540138e-01 2.997111703544373951e-01 2.878321323480862587e-02
2.358631823371434497e-02 4.003090789408973449e-01 5.761046028253882234e-01
2.998708757246026924e-02 4.607686049174872789e-01 5.092443075100523409e-01
7.303400359193905622e-01 2.318879401177657562e-01 3.777202396284366770e-02
7.624869669390765825e-01 2.139322263696668835e-01 2.358080669125644027e-02
2.245792159692630227e-02 5.404152010740254086e-01 4.371268773290483689e-01
7.451806082909940843e-01 2.309005916578291961e-01 2.391880005117665367e-02
6.981590009497026239e-01 2.625549804370630458e-01 3.928601861323431643e-02
7.764860541986938758e-01 1.985096699780618812e-01 2.500427582324414241e-02
8.355714851781941011e-02 5.377600408242523278e-01 3.786828106579281927e-01
1.608140647399109585e-02 3.768688975449537848e-01 6.070496959810550708e-01
7.063604796253617098e-03 4.691656480897980486e-01 5.237707471139482207e-01
7.435703249141542015e-01 2.342818422327498928e-01 2.214783285309588137e-02
6.796311209392739139e-01 2.914934081008391242e-01 2.887547095988688556e-02
7.238888479282165056e-01 2.553334601627747502e-01 2.077769190900861929e-02
7.860238556066762028e-02 4.763868758507894818e-01 4.450107385885428424e-01
1.144717912273251342e-01 5.904013825268694671e-01 2.951268262458054403e-01
7.135859277280380830e-01 2.654970204057355287e-01 2.091705186622647844e-02
6.914647933001973890e-01 2.692050316155247813e-01 3.933017508427792686e-02
1.179757127979049330e-01 5.877980807778677708e-01 2.942262064242273101e-01
7.118464820644757607e-01 2.654501169045323516e-01 2.270340103099196746e-02
3.142128142669954518e-03 4.723001727300921715e-01 5.245576991272379308e-01
1.935708133190611469e-01 4.948850765297188037e-01 3.115441101512199662e-01
2.206630371984923816e-02 4.651919075845807883e-01 5.127417886955699666e-01
1.269155035665654740e-01 5.926115384477195214e-01 2.804729579857149768e-01
8.156427694822248053e-01 1.659142244615746664e-01 1.844300605620056641e-02
5.570311084946552582e-02 4.173122735148269702e-01 5.269846156357075317e-01
7.906302203177683063e-01 2.001241058613527080e-01 9.245673820878940602e-03
5.522587127117083161e-03 4.974044505264512139e-01 4.970729623464317237e-01
7.831987170821274047e-01 1.919870759929453441e-01 2.481420692492741428e-02
6.963104650901753567e-01 2.638321835090208789e-01 3.985735140080382682e-02
1.140651521889802025e-02 5.201064466160607580e-01 4.684870381650412252e-01
7.167259676539401436e-01 2.582584461710290546e-01 2.501558617503088500e-02
2.462340225667120769e-02 2.972676581801071971e-01 6.781089395632217132e-01
1.220206366293549727e-01 5.010642582181247251e-01 3.769151051525201912e-01
1.442917844637662539e-01 5.498457171810918753e-01 3.058624983551418985e-01
2.041677276263436092e-01 5.722719975236930656e-01 2.235602748499632697e-01
3.109565189060246176e-02 3.604294076084284537e-01 6.084749405009690637e-01
3.032743113431232707e-02 4.339812365625388968e-01 5.356913323031488039e-01
3.697888010017095356e-02 5.434694019181388613e-01 4.195517179816902753e-01
5.174085484340928270e-02 5.256420386181476223e-01 4.226171065384431436e-01
6.504260591452436513e-01 3.171650724233314733e-01 3.240886843142484763e-02
1.413411402691047147e-01 5.722472947335150995e-01 2.864115649973801303e-01
1.963071038771168134e-02 3.834932606623591700e-01 5.968760289499290828e-01
2.698986331964879984e-02 3.945006469308862118e-01 5.785094897494650334e-01
7.218361806195532360e-01 2.532213169947387099e-01 2.494250238570809575e-02
8.758472490679078404e-02 6.015348418822745735e-01 3.108804332109345592e-01
1.441428298153322740e-01 4.553673517839969542e-01 4.004898184006707162e-01
6.630777554384020833e-02 3.757010314196880252e-01 5.579911930364717110e-01
5.981581445547437537e-02 6.259610733789102710e-01 3.142231121656153814e-01
7.308387705716505867e-01 2.547961886625306760e-01 1.436504076581881020e-02
7.703052763725595220e-01 2.023404933075096113e-01 2.735423031993089102e-02
6.796311209392739139e-01 2.914934081008391242e-01 2.887547095988688556e-02
2.041102329845258592e-02 4.158233785793876147e-01 5.637655981221597612e-01
1.274556604224261758e-01 5.625703745530452604e-01 3.099739650245283973e-01
1.324231202258381764e-02 3.849080167692057741e-01 6.018496712082103128e-01
7.161083713068323586e-01 2.484518717523986342e-01 3.543975694076910432e-02

probas_r.txt

0 1 2
1.803132918688412842e-02 4.606640351438364700e-01 5.213046356692794259e-01
1.164864338681035477e-01 5.139747122814198876e-01 3.695388538504766340e-01
6.962455532898063904e-02 5.373395974678950582e-01 3.930358472031243999e-01
7.204021495267802500e-02 5.730744881104643529e-01 3.548852969368575527e-01
6.991651423936211251e-02 5.016346670847556810e-01 4.284488186758823036e-01
6.796311209392739139e-01 2.914934081008391242e-01 2.887547095988688556e-02
8.262824295603955760e-02 4.880215830381079423e-01 4.293501740058524585e-01
7.398501883572305982e-02 4.879481018123815739e-01 4.380668793518953663e-01
6.760734510633202854e-01 2.901311842779138450e-01 3.379536465876599449e-02
7.964708632149816170e-01 1.866745367851723181e-01 1.685459999984592616e-02
3.474691957121457675e-02 3.583703606139978515e-01 6.068827198147875857e-01
8.509435872920290134e-02 4.105753551444020211e-01 5.043302861263950776e-01
6.999722292834740234e-01 2.523641388666046370e-01 4.766363184992129798e-02
6.724835245769099634e-01 2.833093367447748201e-01 4.420713867831525118e-02
4.742284428346629777e-02 4.720497609986373244e-01 4.805273947178964056e-01
7.972846354438156702e-01 1.805030425757565560e-01 2.221232198042776343e-02
7.576134338199155271e-01 2.024869354006992028e-01 3.989963077938533248e-02
1.153947575765059547e-01 5.566877405532142387e-01 3.279175018702797928e-01
1.803781001584012500e-01 5.243874717740818481e-01 2.952344280675168742e-01
7.203726347619267001e-01 2.446797506298435121e-01 3.494761460822982252e-02
2.840122241793625879e-02 4.141106231978258223e-01 5.574881543842380438e-01
8.218635408520580388e-02 3.799898958740794974e-01 5.378237500407146987e-01
7.333655213140608531e-01 2.441271300549988921e-01 2.250734863094028607e-02
5.032928040812036791e-02 4.254589432527108861e-01 5.242117763391687113e-01
1.550184801583563178e-02 3.993522229867868534e-01 5.851459289973774958e-01
1.227940242209528549e-01 4.352132217817513138e-01 4.419927539972958730e-01
7.476631547670579137e-01 2.338983238461015135e-01 1.843852138684076020e-02
3.216204087711193454e-02 4.529936242371294663e-01 5.148443348857587587e-01
1.245258750085084798e-01 4.765967938070554411e-01 3.988773311844361347e-01
1.192026185569960983e-01 5.591987924594459258e-01 3.215985889835579203e-01
2.366219632948434945e-02 4.481823387465238562e-01 5.281554649239917909e-01
7.196541135895072605e-01 2.507540978848702418e-01 2.959178852562241793e-02
3.909239341050988609e-02 4.617561189318676318e-01 4.991514876576225168e-01
7.375121314542560924e-01 2.225991539329158220e-01 3.988871461282808561e-02
7.550244151239472457e-01 2.266560410284501892e-01 1.831954384760245066e-02
1.153823581648319579e-01 5.574713726861941776e-01 3.271462691489737673e-01
2.616602157927563552e-02 4.394664357747015693e-01 5.343675426460229305e-01
2.011962733497532629e-02 3.437465036022248621e-01 6.361338690627998949e-01
2.356743774802111738e-02 5.160735303377029792e-01 4.603590319142759624e-01
1.160875520226575981e-02 4.882289855278042490e-01 5.001622592699300762e-01
1.228251583333377084e-01 5.567369750652966243e-01 3.204378666013656396e-01
1.189080161622243191e-02 5.015879418532329659e-01 4.865212565305445658e-01
9.673743258885576790e-02 4.577343850440076523e-01 4.455281823671367047e-01
1.230344888459725949e-01 5.588584609303267614e-01 3.181070502237006714e-01
3.882201672850698476e-02 4.897136809266268154e-01 4.714643023448661374e-01
3.902165918924874882e-02 4.985770376442009155e-01 4.624013031665503703e-01
4.332021270366276378e-02 3.567048002602143386e-01 5.999749870361230641e-01
2.703642418326698479e-02 4.889071742289862188e-01 4.840564015877466542e-01
1.049056515358237629e-01 6.119707730405120616e-01 2.831235754236641755e-01
3.404631036061477739e-02 4.395454632239478854e-01 5.264082264154372748e-01
1.263449658738045323e-01 4.042605744652454924e-01 4.693944596609500586e-01
6.755259861647301411e-01 2.869937589526130317e-01 3.748025488265698674e-02
2.698986331964879984e-02 3.945006469308862118e-01 5.785094897494650334e-01
9.852202960248160335e-02 4.929655905239373470e-01 4.085123798735810774e-01
2.148561558116891257e-01 4.836801409144520503e-01 3.014637032738588518e-01
1.004025625920916404e-01 5.146851053538687992e-01 3.849123320540395743e-01
8.627849163417691813e-02 4.967682219648476005e-01 4.169532864009755091e-01
1.942164122078753052e-02 4.796703442854525212e-01 5.009080144937599899e-01
7.593499392344295051e-01 2.207428107255730299e-01 1.990725003999744766e-02
7.158870334752654996e-01 2.516257734679558933e-01 3.248719305677871122e-02
2.327791126635296581e-02 4.602934436966450438e-01 5.164286450370019210e-01
1.373234955340776886e-01 4.783319859752112602e-01 3.843445184907110512e-01
6.949518286056532990e-01 2.812964698926803675e-01 2.375170150166635089e-02
8.386990148111839183e-01 1.469149798064622037e-01 1.438600538235389366e-02
7.239047106926639608e-02 5.820036726150982220e-01 3.456058563156353403e-01
6.913473444191029493e-01 2.687328522557725363e-01 3.991980332512447965e-02
3.836663326384131406e-02 3.826504868278536309e-01 5.789828799083049926e-01
2.652124409239845493e-01 4.946029764302026233e-01 2.401845826458129107e-01
7.481985532669748196e-01 2.150897137032159179e-01 3.671173302980923481e-02
4.370094537430890824e-02 5.878059939970771142e-01 3.684930606286138111e-01
1.797272630225753981e-02 3.483135002479387787e-01 6.337137734498037300e-01
1.202147681098546944e-01 4.817180954296959583e-01 3.980671364604493889e-01
6.746797528617258788e-01 2.963890837008006374e-01 2.893116343747341099e-02
1.526562037062645025e-02 4.584245961983302031e-01 5.263097834310432876e-01
2.079394001137311113e-02 4.028698258530724052e-01 5.763362341355544594e-01
1.502435564067106967e-02 3.783102481392631722e-01 6.066653962200657668e-01
1.626315747998946862e-02 4.089565242581552629e-01 5.747803182618552720e-01
7.762666082726898953e-01 1.983548098660834758e-01 2.537858186122670875e-02
7.261697897944627833e-01 2.385650036252726924e-01 3.526520658026453120e-02
4.677088296615945079e-02 4.236227174528874317e-01 5.296063995809531244e-01
2.453827455919246545e-02 3.855208782716869065e-01 5.899408471691206524e-01
5.715629488226829924e-01 3.837877882800596607e-01 4.464926289725740932e-02
3.230205132690781855e-02 3.036272648966615306e-01 6.640706837764307480e-01
6.715056164107540138e-01 2.997111703544373951e-01 2.878321323480862587e-02
2.358631823371434497e-02 4.003090789408973449e-01 5.761046028253882234e-01
2.998708757246026924e-02 4.607686049174872789e-01 5.092443075100523409e-01
7.303400359193905622e-01 2.318879401177657562e-01 3.777202396284366770e-02
7.624869669390765825e-01 2.139322263696668835e-01 2.358080669125644027e-02
2.245792159692630227e-02 5.404152010740254086e-01 4.371268773290483689e-01
7.451806082909940843e-01 2.309005916578291961e-01 2.391880005117665367e-02
6.981590009497026239e-01 2.625549804370630458e-01 3.928601861323431643e-02
7.764860541986938758e-01 1.985096699780618812e-01 2.500427582324414241e-02
8.355714851781941011e-02 5.377600408242523278e-01 3.786828106579281927e-01
1.608140647399109585e-02 3.768688975449537848e-01 6.070496959810550708e-01
7.063604796253617098e-03 4.691656480897980486e-01 5.237707471139482207e-01
7.435703249141542015e-01 2.342818422327498928e-01 2.214783285309588137e-02
6.796311209392739139e-01 2.914934081008391242e-01 2.887547095988688556e-02
7.238888479282165056e-01 2.553334601627747502e-01 2.077769190900861929e-02
7.860238556066762028e-02 4.763868758507894818e-01 4.450107385885428424e-01
1.144717912273251342e-01 5.904013825268694671e-01 2.951268262458054403e-01
7.135859277280380830e-01 2.654970204057355287e-01 2.091705186622647844e-02
6.914647933001973890e-01 2.692050316155247813e-01 3.933017508427792686e-02
1.179757127979049330e-01 5.877980807778677708e-01 2.942262064242273101e-01
7.118464820644757607e-01 2.654501169045323516e-01 2.270340103099196746e-02
3.142128142669954518e-03 4.723001727300921715e-01 5.245576991272379308e-01
1.935708133190611469e-01 4.948850765297188037e-01 3.115441101512199662e-01
2.206630371984923816e-02 4.651919075845807883e-01 5.127417886955699666e-01
1.269155035665654740e-01 5.926115384477195214e-01 2.804729579857149768e-01
8.156427694822248053e-01 1.659142244615746664e-01 1.844300605620056641e-02
5.570311084946552582e-02 4.173122735148269702e-01 5.269846156357075317e-01
7.906302203177683063e-01 2.001241058613527080e-01 9.245673820878940602e-03
5.522587127117083161e-03 4.974044505264512139e-01 4.970729623464317237e-01
7.831987170821274047e-01 1.919870759929453441e-01 2.481420692492741428e-02
6.963104650901753567e-01 2.638321835090208789e-01 3.985735140080382682e-02
1.140651521889802025e-02 5.201064466160607580e-01 4.684870381650412252e-01
7.167259676539401436e-01 2.582584461710290546e-01 2.501558617503088500e-02
2.462340225667120769e-02 2.972676581801071971e-01 6.781089395632217132e-01
1.220206366293549727e-01 5.010642582181247251e-01 3.769151051525201912e-01
1.442917844637662539e-01 5.498457171810918753e-01 3.058624983551418985e-01
2.041677276263436092e-01 5.722719975236930656e-01 2.235602748499632697e-01
3.109565189060246176e-02 3.604294076084284537e-01 6.084749405009690637e-01
3.032743113431232707e-02 4.339812365625388968e-01 5.356913323031488039e-01
3.697888010017095356e-02 5.434694019181388613e-01 4.195517179816902753e-01
5.174085484340928270e-02 5.256420386181476223e-01 4.226171065384431436e-01
6.504260591452436513e-01 3.171650724233314733e-01 3.240886843142484763e-02
1.413411402691047147e-01 5.722472947335150995e-01 2.864115649973801303e-01
1.963071038771168134e-02 3.834932606623591700e-01 5.968760289499290828e-01
2.698986331964879984e-02 3.945006469308862118e-01 5.785094897494650334e-01
7.218361806195532360e-01 2.532213169947387099e-01 2.494250238570809575e-02
8.758472490679078404e-02 6.015348418822745735e-01 3.108804332109345592e-01
1.441428298153322740e-01 4.553673517839969542e-01 4.004898184006707162e-01
6.630777554384020833e-02 3.757010314196880252e-01 5.579911930364717110e-01
5.981581445547437537e-02 6.259610733789102710e-01 3.142231121656153814e-01
7.308387705716505867e-01 2.547961886625306760e-01 1.436504076581881020e-02
7.703052763725595220e-01 2.023404933075096113e-01 2.735423031993089102e-02
6.796311209392739139e-01 2.914934081008391242e-01 2.887547095988688556e-02
2.041102329845258592e-02 4.158233785793876147e-01 5.637655981221597612e-01
1.274556604224261758e-01 5.625703745530452604e-01 3.099739650245283973e-01
1.324231202258381764e-02 3.849080167692057741e-01 6.018496712082103128e-01
7.161083713068323586e-01 2.484518717523986342e-01 3.543975694076910432e-02

wtf.r

library('pROC')

y <- read.csv("y_true_r.txt")$target 
probas <- read.csv("probas_r.txt", sep=" ")

auc1 <- multiclass.roc(y, probas$X0)$auc
auc2 <- multiclass.roc(y, probas$X1)$auc 
auc3 <- multiclass.roc(y, probas$X2)$auc

print(auc1)
print(auc2)
print(auc3)
print((auc1 + auc2 + auc3) / 3)

y_true.txt

2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00

y_true_r.txt

target
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
1.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
0.000000000000000000e+00
2.000000000000000000e+00
1.000000000000000000e+00
2.000000000000000000e+00
0.000000000000000000e+00