import numpy as np
import pandas as pd
from scipy.stats import kendalltau
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path
from collections import OrderedDict
import os
import skimage.io as skio
dict_metrics = {
'tpr': 'TPR, Recall, Sensitivity',
'tnr': 'TNR, Specificity, Selectivity',
'fpr': 'FPR',
'fnr': 'FNR, Miss rate',
'mpr': 'May positive rate (MPR)',
'mnr': 'May negative rate (MNR)',
'f1': 'F1',
'precision': 'Precision',
'edge_coherence': 'Edge coherence',
'accuracy_must_may': 'Accuracy (ignoring cannot)'
}
dict_models = OrderedDict([
('e84a8b06', 'm'),
('b0a3ff6a', 'ms'),
('5a53032f', 'msd'),
('0575609e', 'pseudo, msd'),
('90fa5734', 'msd, dada, pseudo'),
('34e84fe9', 'pseudo, msd, dada'),
('ae98d8ee', 'dada, pseudo, msd'),
('23ad5382', 'pseudo, dada, msd'),
('c4721ec4', 'dada, msd'),
('d7118205', 'msd_spade'),
('0da9669d', 'msd_spade, pseudo'),
('9a8fcb11', 'msd_spade, pseudo'),
('8051ea3a', 'dada, msd_spade'),
('49167587', 'trash, dada, msd_spade, pseudo'),
])
def boxplot_metric(df, metric, do_stripplot=False, **snskwargs):
f = plt.figure(dpi=300)
if do_stripplot:
ax = sns.boxplot(x='model', y=metric, data=df, fliersize=0., **snskwargs)
ax = sns.stripplot(x='model', y=metric, data=df, size=2., color='gray', **snskwargs)
else:
ax = sns.boxplot(x='model', y=metric, data=df, **snskwargs)
# Set X-label
ax.set_xlabel('Models', rotation=0, fontsize='medium');
# Set Y-label
ax.set_ylabel(dict_metrics[metric], rotation=90, fontsize='medium');
# Change spines
sns.despine(left=True, bottom=True)
# Change X-Tick labels
xticklabels = [dict_models[t.get_text()] for t in ax.get_xticklabels()]
ax.set_xticklabels(xticklabels,
rotation=20,
verticalalignment='top',
horizontalalignment='right',
fontsize='xx-small');
# # Remove legend
# ax.get_legend().remove()
# # Set x-axis limits
# set_xaxis(ax, x_min=-0.5, x_max=19.5, step=1., fontsize='small')
# # Vertical grid lines
# ax.grid(b=True, axis='x', which='major')
# ax.tick_params(axis='x', which='both',length=0)
return ax
def heatmap_kendall(data_dict, metric, models, **snskwargs):
f = plt.figure(dpi=300)
ax = sns.heatmap(dict_kendall_mat[metric], linewidths=.5)
# Set axis labels
ax.set_xlabel(None);
ax.set_ylabel(None);
# Change X-Tick labels
ax.set_xticklabels(models,
rotation=20,
verticalalignment='top',
horizontalalignment='right',
fontsize='xx-small');
ax.set_yticklabels(models,
rotation=0,
fontsize='xx-small');
ax.set_title(dict_metrics[metric])
# # Remove legend
# ax.get_legend().remove()
# # Set x-axis limits
# set_xaxis(ax, x_min=-0.5, x_max=19.5, step=1., fontsize='small')
# # Vertical grid lines
# ax.grid(b=True, axis='x', which='major')
# ax.tick_params(axis='x', which='both',length=0)
return ax
models_paths = list(Path('./data').glob('*'))
models_df = {m.name.split('--')[1]: pd.read_csv(m.joinpath('eval_masker.csv'), index_col=False)
for m in models_paths}
for k, v in models_df.items():
v['model'] = [k] * len(v)
df = pd.concat(list(models_df.values()), ignore_index=True)
models = df.model.unique()
remove_keys = set(dict_models.keys()).difference(set(models))
for m in remove_keys:
dict_models.pop(m)
tpr_th = 0.95
fpr_th = 0.05
acc_th = 0.5
edgec_th = 0.02
idx_good_in_all = []
for idx in df.idx.unique():
df_th = df.loc[(df.tpr >= tpr_th) &\
(df.fpr <= fpr_th) &\
(df.accuracy_must_may >= acc_th) &\
(df.idx == idx) &\
(df.model.isin(models))]
if len(df_th) == len(models):
idx_good_in_all.append(idx)
print(len(idx_good_in_all))
idx_not_good_in_all = list(set(df.idx.unique()).difference(idx_good_in_all))
82
tpr_th = 0.95
fpr_th = 0.05
edgec_th = 0.05
idx_not_good_in_any = []
for idx in df.idx.unique():
df_th = df.loc[((df.tpr <= tpr_th) |\
(df.fpr >= fpr_th) |\
(df.edge_coherence >= edgec_th)) &\
((df.idx == idx) &\
(df.model.isin(models)))]
if len(df_th) > 0:
idx_not_good_in_any.append(idx)
print(len(idx_not_good_in_any))
25
data_path = '/home/alex/Dropbox/ccai/data/'
imgs_orig_path = os.path.join(data_path, 'floodmasks_eval/imgs')
n_cols = 5
n_rows = len(idx_failed) // n_cols
f, axes = plt.subplots(n_rows, n_cols, dpi=800)
idx = 0
for r in range(n_rows):
for c in range(n_cols):
filename = df.loc[df.idx == idx_not_good_in_any[idx], 'filename'].values[0]
axes[r, c].imshow(skio.imread(os.path.join(imgs_orig_path, filename)))
axes[r, c].axis('off');
axes[r, c].set_title(filename[:15], fontsize='xx-small')
idx += 1
