diff --git a/scripts/osg_stats.py b/scripts/osg_stats.py index f0e6ce4010..037aafea00 100755 --- a/scripts/osg_stats.py +++ b/scripts/osg_stats.py @@ -38,27 +38,41 @@ import termtables @click.option('--timeseries_sum', is_flag=True, help='Add a graph to timeseries for a sum per frame of all given timeseries metrics.') @click.option('--commulative_timeseries_sum', is_flag=True, - help='Add a graph to timeseries for a sum per frame of all given commulative timeseries.') + help='Add a graph to timeseries for a sum per frame of all given commulative timeseries.') @click.option('--stats_sum', is_flag=True, help='Add a row to stats table for a sum per frame of all given stats metrics.') @click.option('--begin_frame', type=int, default=0, help='Start processing from this frame.') @click.option('--end_frame', type=int, default=sys.maxsize, help='End processing at this frame.') +@click.option('--frame_number_name', type=str, default='FrameNumber', + help='Frame number metric name.') +@click.option('--hist_threshold', type=str, multiple=True, + help='Show a histogram for given metric only for frames with threshold_name metric over threshold_value.') +@click.option('--threshold_name', type=str, default='Frame duration', + help='Frame duration metric name.') +@click.option('--threshold_value', type=float, default=1.05/60, + help='Threshold for hist_over.') @click.argument('path', type=click.Path(), nargs=-1) def main(print_keys, timeseries, hist, hist_ratio, stdev_hist, plot, stats, timeseries_sum, stats_sum, begin_frame, end_frame, path, - commulative_timeseries, commulative_timeseries_sum): + commulative_timeseries, commulative_timeseries_sum, frame_number_name, + hist_threshold, threshold_name, threshold_value): sources = {v: list(read_data(v)) for v in path} if path else {'stdin': list(read_data(None))} keys = collect_unique_keys(sources) - frames = collect_per_frame(sources=sources, keys=keys, begin_frame=begin_frame, end_frame=end_frame) + frames, begin_frame, end_frame = collect_per_frame( + sources=sources, keys=keys, begin_frame=begin_frame, + end_frame=end_frame, frame_number_name=frame_number_name, + ) if print_keys: for v in keys: print(v) if timeseries: - draw_timeseries(sources=frames, keys=timeseries, add_sum=timeseries_sum) + draw_timeseries(sources=frames, keys=timeseries, add_sum=timeseries_sum, + begin_frame=begin_frame, end_frame=end_frame) if commulative_timeseries: - draw_commulative_timeseries(sources=frames, keys=commulative_timeseries, add_sum=commulative_timeseries_sum) + draw_commulative_timeseries(sources=frames, keys=commulative_timeseries, add_sum=commulative_timeseries_sum, + begin_frame=begin_frame, end_frame=end_frame) if hist: draw_hists(sources=frames, keys=hist) if hist_ratio: @@ -69,6 +83,9 @@ def main(print_keys, timeseries, hist, hist_ratio, stdev_hist, plot, stats, draw_plots(sources=frames, plots=plot) if stats: print_stats(sources=frames, keys=stats, stats_sum=stats_sum) + if hist_threshold: + draw_hist_threshold(sources=frames, keys=hist_threshold, begin_frame=begin_frame, + threshold_name=threshold_name, threshold_value=threshold_value) matplotlib.pyplot.show() @@ -92,19 +109,26 @@ def read_data(path): frame[key] = to_number(value) -def collect_per_frame(sources, keys, begin_frame, end_frame): +def collect_per_frame(sources, keys, begin_frame, end_frame, frame_number_name): + assert begin_frame < end_frame result = collections.defaultdict(lambda: collections.defaultdict(list)) + begin_frame = max(begin_frame, min(v[0][frame_number_name] for v in sources.values())) + end_frame = min(end_frame, begin_frame + max(len(v) for v in sources.values())) + for name in sources.keys(): + for key in keys: + result[name][key] = [0] * (end_frame - begin_frame) for name, frames in sources.items(): for frame in frames: - for key in keys: - if key in frame: - result[name][key].append(frame[key]) - else: - result[name][key].append(None) - for name, sources in result.items(): - for key, values in sources.items(): - result[name][key] = numpy.array(values[begin_frame:end_frame]) - return result + number = frame[frame_number_name] + if begin_frame <= number < end_frame: + index = number - begin_frame + for key in keys: + if key in frame: + result[name][key][index] = frame[key] + for name in result.keys(): + for key in keys: + result[name][key] = numpy.array(result[name][key]) + return result, begin_frame, end_frame def collect_unique_keys(sources): @@ -116,12 +140,11 @@ def collect_unique_keys(sources): return sorted(result) -def draw_timeseries(sources, keys, add_sum): +def draw_timeseries(sources, keys, add_sum, begin_frame, end_frame): fig, ax = matplotlib.pyplot.subplots() + x = numpy.array(range(begin_frame, end_frame)) for name, frames in sources.items(): - x = numpy.array(range(max(len(v) for k, v in frames.items() if k in keys))) for key in keys: - print(key, name) ax.plot(x, frames[key], label=f'{key}:{name}') if add_sum: ax.plot(x, numpy.sum(list(frames[k] for k in keys), axis=0), label=f'sum:{name}') @@ -130,10 +153,10 @@ def draw_timeseries(sources, keys, add_sum): fig.canvas.set_window_title('timeseries') -def draw_commulative_timeseries(sources, keys, add_sum): +def draw_commulative_timeseries(sources, keys, add_sum, begin_frame, end_frame): fig, ax = matplotlib.pyplot.subplots() + x = numpy.array(range(begin_frame, end_frame)) for name, frames in sources.items(): - x = numpy.array(range(max(len(v) for k, v in frames.items() if k in keys))) for key in keys: ax.plot(x, numpy.cumsum(frames[key]), label=f'{key}:{name}') if add_sum: @@ -227,7 +250,6 @@ def print_stats(sources, keys, stats_sum): if stats_sum: stats.append(make_stats(source=name, key='sum', values=sum_multiple(frames, keys))) metrics = list(stats[0].keys()) - max_key_size = max(len(tuple(v.values())[0]) for v in stats) termtables.print( [list(v.values()) for v in stats], header=metrics, @@ -235,6 +257,27 @@ def print_stats(sources, keys, stats_sum): ) +def draw_hist_threshold(sources, keys, begin_frame, threshold_name, threshold_value): + for name, frames in sources.items(): + indices = [n for n, v in enumerate(frames[threshold_name]) if v > threshold_value] + numbers = [v + begin_frame for v in indices] + x = [v for v in range(0, len(indices))] + fig, ax = matplotlib.pyplot.subplots() + ax.set_title(f'Frames with "{threshold_name}" > {threshold_value} ({len(indices)})') + ax.bar(x, [frames[threshold_name][v] for v in indices], label=threshold_name, color='black', alpha=0.2) + prev = 0 + for key in keys: + values = [frames[key][v] for v in indices] + ax.bar(x, values, bottom=prev, label=key) + prev = values + ax.hlines(threshold_value, x[0] - 1, x[-1] + 1, color='black', label='threshold', linestyles='dashed') + ax.xaxis.set_major_locator(matplotlib.pyplot.FixedLocator(x)) + ax.xaxis.set_major_formatter(matplotlib.pyplot.FixedFormatter(numbers)) + ax.grid(True) + ax.legend() + fig.canvas.set_window_title(f'hist_threshold:{name}') + + def filter_not_none(values): return [v for v in values if v is not None] @@ -269,5 +312,6 @@ def to_number(value): except ValueError: return float(value) + if __name__ == '__main__': main()