31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166 | def evaluate_fitting(train_history,
out_path,
monitor=["loss"],
prefix_split=".",
suffix=None,
show=False
):
""" Function for automatic plot generation providing a training history dictionary.
!!! info "Preview"
Created filename in directory of `out_path`:
- without suffix "plot.fitting_course.png"
- with suffix "plot.fitting_course.SUFFIX.png"
???+ example
```python
# Initialize and train a model
model = NeuralNetwork(n_labels=8, channels=3, architecture="2D.ResNet50")
history = model.train(datagen_train, datagen_validation, epochs=100)
# Pass history dict to evaluation function
evaluate_fitting(history, out_path="./")
# Figure will be created at: "./plot.fitting_course.png"
```
Args:
train_history (dict): A history dictionary from a Keras history object which contains several logs.
out_path (str): Path to directory in which plotted figure is stored.
monitor (list of str): List of metrics which should be visualized in the fitting plot.
prefix_split (str): Split prefix for keys in the history dictionary. Used for Bagging and Stacking.
suffix (str): Special suffix to add in the created figure filename.
show (bool): Option, whether to also display the generated chart.
"""
# Convert to pandas dataframe
hist_prepared = dict([ (k,pd.Series(v)) for k,v in train_history.items() ])
dt = pd.DataFrame.from_dict(hist_prepared, orient="columns")
# Identify all selected columns
selected_cols = []
for key in train_history:
for m in monitor:
if m in key:
selected_cols.append(key)
break
# Add epoch column
dt["epoch"] = dt.index + 1
# Melt dataframe
dt_melted = dt.melt(id_vars=["epoch"],
value_vars=selected_cols,
var_name="metric",
value_name="score")
# Handle special prefix tags (if split-able by '.')
if prefix_split is not None:
for c in selected_cols:
valid_split = True
if prefix_split not in c:
valid_split = False
break
if valid_split:
dt_melted[["prefix", "metric"]] = dt_melted["metric"].str.split(".",
expand=True)
# Remove NaN tags
dt_melted = dt_melted.dropna(axis=0)
# Preprocess transfer learning tag
if dt_melted["metric"].str.startswith("tl_").any():
# filter prefix groups
filter = dt_melted[dt_melted["metric"].str.startswith("ft_")]
# if prefix available -> add epochs for each prefix group
if valid_split:
# identify number of epochs for each prefix
filter_tl = dt_melted[dt_melted["metric"].str.startswith("tl_")]
tl_epochs = filter_tl.groupby(["prefix"])["epoch"].max()
# compute fine tune epoch update
group_repeats = filter.groupby(["prefix"]).size()
if group_repeats.empty : ft_update = 0
else : ft_update = tl_epochs.repeat(group_repeats).to_numpy()
# if no prefix available -> add epochs to all ft phases
else:
# identify number of epochs global
filter_tl = dt_melted[dt_melted["metric"].str.startswith("tl_")]
tl_epochs = filter_tl["epoch"].max()
# compute fine tune epoch update
ft_update = tl_epochs
# update number of epochs
dt_melted.loc[dt_melted["metric"].str.startswith("ft_"), "epoch"] =\
filter["epoch"].to_numpy() + ft_update
else : tl_epochs = None
# Remove preprocessed transfer learning tag from metric column
dt_melted["metric"] = dt_melted["metric"].apply(remove_tag, tag="tl_")
dt_melted["metric"] = dt_melted["metric"].apply(remove_tag, tag="ft_")
# Preprocess validation tag
dt_melted["subset"] = np.where(dt_melted["metric"].str.startswith("val_"),
"validation", "training")
dt_melted["metric"] = dt_melted["metric"].apply(remove_tag, tag="val_")
# Plot results via plotnine
fig = (ggplot(dt_melted, aes("epoch", "score", color="subset"))
+ geom_line(size=1)
+ ggtitle("Fitting Curve during Training Process")
+ xlab("Epoch")
+ ylab("Score")
+ scale_colour_discrete(name="Subset")
+ theme_bw()
+ theme(subplots_adjust={'wspace':0.2}))
if prefix_split is not None and valid_split:
fig += facet_grid("prefix ~ metric")
else : fig += facet_wrap("metric", scales="free_y")
if tl_epochs is not None and valid_split:
tle_df = tl_epochs.to_frame().reset_index()
fig += geom_vline(tle_df, aes(xintercept="epoch"))
elif tl_epochs is not None and not valid_split:
fig += geom_vline(xintercept=tl_epochs)
# Store figure to disk
filename = "plot.fitting_course"
if suffix is not None : filename += "." + str(suffix)
filename += ".png"
fig.save(filename=filename,
path=out_path, dpi=200, limitsize=False)
if show : print(fig)
|
