Training_and_Evaluation_Overview error

[akiva-skolnik]

Training_and_Evaluation_Overview error:

Successfully loaded training round 1 for worker 0: Iterations: 1, Training episodes: 19, Evaluation episodes: 0
Successfully loaded training round 1 for worker 1: Iterations: 1, Training episodes: 19, Evaluation episodes: 0
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
File /usr/local/lib/python3.8/dist-packages/matplotlib/axis.py:1770, in Axis.convert_units(self, x)
   1769 try:
-> 1770     ret = self.converter.convert(x, self.units, self)
   1771 except Exception as e:

File /usr/local/lib/python3.8/dist-packages/matplotlib/category.py:57, in StrCategoryConverter.convert(value, unit, axis)
     56 # force an update so it also does type checking
---> 57 unit.update(values)
     58 return np.vectorize(unit._mapping.__getitem__, otypes=[float])(values)

File /usr/local/lib/python3.8/dist-packages/matplotlib/category.py:216, in UnitData.update(self, data)
    214 for val in OrderedDict.fromkeys(data):
    215     # OrderedDict just iterates over unique values in data.
--> 216     _api.check_isinstance((str, bytes), value=val)
    217     if convertible:
    218         # this will only be called so long as convertible is True.

File /usr/local/lib/python3.8/dist-packages/matplotlib/_api/__init__.py:93, in check_isinstance(_types, **kwargs)
     92     names.append("None")
---> 93 raise TypeError(
     94     "{!r} must be an instance of {}, not a {}".format(
     95         k,
     96         ", ".join(names[:-1]) + " or " + names[-1]
     97         if len(names) > 1 else names[0],
     98         type_name(type(v))))

TypeError: 'value' must be an instance of str or bytes, not a float

The above exception was the direct cause of the following exception:

ConversionError                           Traceback (most recent call last)
File /usr/local/lib/python3.8/dist-packages/IPython/core/formatters.py:340, in BaseFormatter.__call__(self, obj)
    338     pass
    339 else:
--> 340     return printer(obj)
    341 # Finally look for special method names
    342 method = get_real_method(obj, self.print_method)

File /usr/local/lib/python3.8/dist-packages/IPython/core/pylabtools.py:152, in print_figure(fig, fmt, bbox_inches, base64, **kwargs)
    149     from matplotlib.backend_bases import FigureCanvasBase
    150     FigureCanvasBase(fig)
--> 152 fig.canvas.print_figure(bytes_io, **kw)
    153 data = bytes_io.getvalue()
    154 if fmt == 'svg':

File /usr/local/lib/python3.8/dist-packages/matplotlib/backend_bases.py:2342, in FigureCanvasBase.print_figure(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, pad_inches, bbox_extra_artists, backend, **kwargs)
   2336     renderer = _get_renderer(
   2337         self.figure,
   2338         functools.partial(
   2339             print_method, orientation=orientation)
   2340     )
   2341     with getattr(renderer, "_draw_disabled", nullcontext)():
-> 2342         self.figure.draw(renderer)
   2344 if bbox_inches:
   2345     if bbox_inches == "tight":

File /usr/local/lib/python3.8/dist-packages/matplotlib/artist.py:95, in _finalize_rasterization.<locals>.draw_wrapper(artist, renderer, *args, **kwargs)
     93 @wraps(draw)
     94 def draw_wrapper(artist, renderer, *args, **kwargs):
---> 95     result = draw(artist, renderer, *args, **kwargs)
     96     if renderer._rasterizing:
     97         renderer.stop_rasterizing()

File /usr/local/lib/python3.8/dist-packages/matplotlib/artist.py:72, in allow_rasterization.<locals>.draw_wrapper(artist, renderer)
     69     if artist.get_agg_filter() is not None:
     70         renderer.start_filter()
---> 72     return draw(artist, renderer)
     73 finally:
     74     if artist.get_agg_filter() is not None:

File /usr/local/lib/python3.8/dist-packages/matplotlib/figure.py:3175, in Figure.draw(self, renderer)
   3172         # ValueError can occur when resizing a window.
   3174 self.patch.draw(renderer)
-> 3175 mimage._draw_list_compositing_images(
   3176     renderer, self, artists, self.suppressComposite)
   3178 for sfig in self.subfigs:
   3179     sfig.draw(renderer)

File /usr/local/lib/python3.8/dist-packages/matplotlib/image.py:131, in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    129 if not_composite or not has_images:
    130     for a in artists:
--> 131         a.draw(renderer)
    132 else:
    133     # Composite any adjacent images together
    134     image_group = []

File /usr/local/lib/python3.8/dist-packages/matplotlib/artist.py:72, in allow_rasterization.<locals>.draw_wrapper(artist, renderer)
     69     if artist.get_agg_filter() is not None:
     70         renderer.start_filter()
---> 72     return draw(artist, renderer)
     73 finally:
     74     if artist.get_agg_filter() is not None:

File /usr/local/lib/python3.8/dist-packages/matplotlib/axes/_base.py:3064, in _AxesBase.draw(self, renderer)
   3061 if artists_rasterized:
   3062     _draw_rasterized(self.figure, artists_rasterized, renderer)
-> 3064 mimage._draw_list_compositing_images(
   3065     renderer, self, artists, self.figure.suppressComposite)
   3067 renderer.close_group('axes')
   3068 self.stale = False

File /usr/local/lib/python3.8/dist-packages/matplotlib/image.py:131, in _draw_list_compositing_images(renderer, parent, artists, suppress_composite)
    129 if not_composite or not has_images:
    130     for a in artists:
--> 131         a.draw(renderer)
    132 else:
    133     # Composite any adjacent images together
    134     image_group = []

File /usr/local/lib/python3.8/dist-packages/matplotlib/artist.py:72, in allow_rasterization.<locals>.draw_wrapper(artist, renderer)
     69     if artist.get_agg_filter() is not None:
     70         renderer.start_filter()
---> 72     return draw(artist, renderer)
     73 finally:
     74     if artist.get_agg_filter() is not None:

File /usr/local/lib/python3.8/dist-packages/matplotlib/collections.py:972, in _CollectionWithSizes.draw(self, renderer)
    969 @artist.allow_rasterization
    970 def draw(self, renderer):
    971     self.set_sizes(self._sizes, self.figure.dpi)
--> 972     super().draw(renderer)

File /usr/local/lib/python3.8/dist-packages/matplotlib/artist.py:72, in allow_rasterization.<locals>.draw_wrapper(artist, renderer)
     69     if artist.get_agg_filter() is not None:
     70         renderer.start_filter()
---> 72     return draw(artist, renderer)
     73 finally:
     74     if artist.get_agg_filter() is not None:

File /usr/local/lib/python3.8/dist-packages/matplotlib/collections.py:351, in Collection.draw(self, renderer)
    347 renderer.open_group(self.__class__.__name__, self.get_gid())
    349 self.update_scalarmappable()
--> 351 transform, offset_trf, offsets, paths = self._prepare_points()
    353 gc = renderer.new_gc()
    354 self._set_gc_clip(gc)

File /usr/local/lib/python3.8/dist-packages/matplotlib/collections.py:323, in Collection._prepare_points(self)
    321     ys = self.convert_yunits(ys)
    322     paths.append(mpath.Path(np.column_stack([xs, ys]), path.codes))
--> 323 xs = self.convert_xunits(offsets[:, 0])
    324 ys = self.convert_yunits(offsets[:, 1])
    325 offsets = np.ma.column_stack([xs, ys])

File /usr/local/lib/python3.8/dist-packages/matplotlib/artist.py:281, in Artist.convert_xunits(self, x)
    279 if ax is None or ax.xaxis is None:
    280     return x
--> 281 return ax.xaxis.convert_units(x)

File /usr/local/lib/python3.8/dist-packages/matplotlib/axis.py:1772, in Axis.convert_units(self, x)
   1770     ret = self.converter.convert(x, self.units, self)
   1771 except Exception as e:
-> 1772     raise munits.ConversionError('Failed to convert value(s) to axis '
   1773                                  f'units: {x!r}') from e
   1774 return ret

ConversionError: Failed to convert value(s) to axis units: masked_array(data=[--],
             mask=[ True],
       fill_value=1e+20,
            dtype=float64)

[MarkitectRoss]

I don't have this error myself, are you running the latest version of the code from main and can you provide screenshots?

[MarkitectRoss]

No response, assumed fixed, can reopen if not.

[akiva-skolnik]

sorry for the late response.
I am using the latest version

[akiva-skolnik]

BTW @MarkRoss-Eviden I cannot re-open.

[MarkitectRoss]

This seems to be related to when eval has 0 iterations, but training has a number of iterations, so only impacts the very beginning of training, and within say 20 minutes the graphs start to show correctly. Is that your expeirence too @akiva-skolnik ?

Oddly the code that is giving errors is wrapped in a try / except block which I'd assume would catch the errors and just state 'Logs are not yet available. It typically takes 25 minutes from the start of training for them to be available.', but for some reason the errors are getting through : -
try:
tm = metrics.TrainingMetrics("DEEPRACER_S3_URI")
tm.addRound("DR_LOCAL_S3_MODEL_PREFIX", training_round=1, workers=DEEPRACER_WORKERS)
results = tm.plotProgress(method=['median','mean'], rolling_average=1, figsize=(20,5))
except Exception:
print("Logs are not yet available. It typically takes 25 minutes from the start of training for them to be available.")