# SPDX-FileCopyrightText: 2021-2022 Blender Authors # # SPDX-License-Identifier: Apache-2.0 from . import TestQueue import json import pathlib class TestGraph: def __init__(self, json_filepaths: list[pathlib.Path]): # Initialize graph from JSON file. Note that this is implemented without # accessing any benchmark environment or configuration. This ways benchmarks # run on various machines can be aggregated and the graph generated on another # machine. # Gather entries for each device. devices = {} for json_filepath in json_filepaths: queue = TestQueue(json_filepath) for entry in queue.entries: if entry.status in {'done', 'outdated'}: device_name = f"{entry.device_name} ({entry.device_type})" if device_name in devices.keys(): devices[device_name].append(entry) else: devices[device_name] = [entry] data = [] # Sort devices alphabetically. sorted_devices = sorted(devices.items(), key=lambda item: item[0]) for device_name, device_entries in sorted_devices: # Gather used categories. categories = {} device_cpu = '' for entry in device_entries: category = entry.category if category in categories.keys(): categories[category].append(entry) else: categories[category] = [entry] if device_cpu == '': device_cpu = entry.device_cpu # Sort categories alphabetically. sorted_categories = sorted(categories.items(), key=lambda item: item[0]) # Generate one graph for every device x category x result key combination. for category, category_entries in sorted_categories: entries = sorted(category_entries, key=lambda entry: (entry.date, entry.revision, entry.test)) outputs = set() for entry in entries: for output in entry.output.keys(): outputs.add(output) chart_type = 'line' if entries[0].benchmark_type == 'time_series' else 'comparison' if chart_type == 'comparison': entries = sorted(entries, key=lambda entry: (entry.revision, entry.test)) for output in sorted(outputs, reverse=True): chart_name = f"{category} ({output})" data.append(self.chart(device_name, device_cpu, chart_name, entries, chart_type, output)) self.json = json.dumps(data, indent=2) def chart(self, device_name: str, device_cpu, chart_name: str, entries: list, chart_type: str, output: str) -> dict: # Gather used tests. tests = {} for entry in entries: test = entry.test if test not in tests.keys(): tests[test] = len(tests) # Gather used revisions. revisions = {} revision_dates = {} use_error_bars = False for entry in entries: revision = entry.revision if revision not in revisions.keys(): revisions[revision] = len(revisions) revision_dates[revision] = int(entry.date) output_values = entry.output_all_runs.get(output) if output_values and len(output_values) > 1: use_error_bars = True default_entry = { 'x': None, 'y': None, 'yMin': None, 'yMax': None, } # Convert to chart.js data layout. if chart_type == 'comparison': # For comparison, tests on the X axis and revisions as datasets. # Sort tests by index to ensure stable order for labels. sorted_tests = sorted(tests.items(), key=lambda item: item[1]) labels = [test for test, _ in sorted_tests] datasets = [] # Sort revisions by index. sorted_revisions = sorted(revisions.items(), key=lambda item: item[1]) for revision, index in sorted_revisions: datasets.append({ 'label': revision, 'data': [default_entry] * len(tests), }) for entry in entries: test_index = tests[entry.test] revision_index = revisions[entry.revision] output_values = entry.output_all_runs.get(output) if output_values: datasets[revision_index]['data'][test_index] = { 'x': test_index, 'y': sum(output_values) / len(output_values), 'yMin': min(output_values), 'yMax': max(output_values), } else: # For time series, dates on the X axis and tests as datasets. labels = [None] * len(revisions) for revision, index in revisions.items(): labels[index] = revision_dates[revision] * 1000 datasets = [] # Sort tests by index to ensure stable order. sorted_tests = sorted(tests.items(), key=lambda item: item[1]) for test, index in sorted_tests: datasets.append({ 'label': test, 'data': [default_entry] * len(revisions), 'tension': 0.1, }) for entry in entries: test_index = tests[entry.test] revision_index = revisions[entry.revision] output_values = entry.output_all_runs.get(output) if output_values: datasets[test_index]['data'][revision_index] = { 'x': revision_index, 'y': sum(output_values) / len(output_values), 'yMin': min(output_values), 'yMax': max(output_values), } data = {'labels': labels, 'datasets': datasets} return { 'device': device_name, 'device_cpu': device_cpu, 'name': chart_name, 'data': data, 'chart_type': chart_type, 'use_error_bars': use_error_bars} def write(self, filepath: pathlib.Path) -> None: # Write HTML page with JSON graph data embedded. template_dir = pathlib.Path(__file__).parent with open(template_dir / 'graph.template.html', 'r') as f: template = f.read() contents = template.replace('%JSON_DATA%', self.json) with open(filepath, "w") as f: f.write(contents)