diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b7192009190e2845d177fb876a41cc406ac080f..30e0f22dc590ddf00a27b152a408e4ac2f6c606d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2407,8 +2407,25 @@ coverage-test-on-main-scheduled: - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - ls - public_dir="$CI_JOB_NAME-public" + # if is needed to catch case when no artifact is there (first run), similarly as above - - if [[ -d $public_dir ]]; then mv $public_dir/* wmops/; fi + - if [[ -d $public_dir ]]; then + - mv $public_dir/* wmops/ + # check here if we have the split-by-levels files present - if not, fake them up with the existing global one + # this is needed for the first run with split graphs on a branch where the global version did run previously + # NOTE: checking only for level_1 file here as this should already be sufficient + # NOTE2: also not chechking for RAM for same reason + - wmops_all_global="wmops/log_wmops_all.txt" + - ram_all_global="wmops/log_ram_all.txt" + - if [ -f "${wmops_all_global}" ] && [ ! -f "wmops/log_wmops_all_level_1.txt" ]; then + - declare -a suffixes=("level_1" "level_2" "level_3" "rate_sw") + - for suffix in "${suffixes[@]}"; do + - cp ${wmops_all_global} wmops/log_wmops_all_${suffix}.txt + - cp ${ram_all_global} wmops/log_ram_all_${suffix}.txt + - done + - fi + - fi + - ls wmops - rm artifacts.zip - rm -rf $public_dir @@ -2417,7 +2434,7 @@ coverage-test-on-main-scheduled: &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory - public_dir="$CI_JOB_NAME-public" - mkdir $public_dir - - mv -f wmops/log_*_all.txt ./*.js ${public_dir}/ + - mv -f wmops/log_*_all*.txt ./*.js ${public_dir}/ # move logfiles for links - mkdir $public_dir/logs # first move logs diff --git a/ci/complexity_measurements/check_for_changes.py b/ci/complexity_measurements/check_for_changes.py index 2048feb04451f107a63afe02bb1932defb3a6e24..498d0d1a4d3aae4262b529b63d894ebd4833bc23 100644 --- a/ci/complexity_measurements/check_for_changes.py +++ b/ci/complexity_measurements/check_for_changes.py @@ -4,17 +4,20 @@ import sys THRESH = 0.01 -COLS = [ - [3, 5, 7, 9], # wmops_all - [3, 5, 7, 8, 10, 12, 13, 15, 17], # ram_all - [3, 5, 7, 9, 11, 13, 15, 17, 19], # rom_all -] +COLS_WMOPS = [3, 5, 7, 9] +COLS_RAM = [3, 5, 7, 8, 10, 12, 13, 15, 17] +COLS_ROM = [3, 5, 7, 9, 11, 13, 15, 17, 19] def main(args): - linewise_logfiles = [args.wmops_logfile, args.ram_logfile, args.rom_logfile] + linewise_logfiles = [*args.wmops_logfiles, *args.ram_logfiles, args.rom_logfile] + cols = ( + len(args.wmops_logfiles) * [COLS_WMOPS] + + len(args.ram_logfiles) * [COLS_RAM] + + [COLS_ROM] + ) changes_found_linewise = any( - [check_linewise_logfile(f, c) for f, c in zip(linewise_logfiles, COLS)] + [check_linewise_logfile(f, c) for f, c in zip(linewise_logfiles, cols)] ) if changes_found_linewise: @@ -59,9 +62,9 @@ def check_linewise_logfile(filepath, cols): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("wmops_logfile") - parser.add_argument("ram_logfile") - parser.add_argument("rom_logfile") + parser.add_argument("--wmops_logfiles", nargs="+") + parser.add_argument("--ram_logfiles", nargs="+") + parser.add_argument("--rom_logfile") args = parser.parse_args() sys.exit(main(args)) diff --git a/ci/complexity_measurements/genWebpageData.py b/ci/complexity_measurements/genWebpageData.py index 89247620e377c9ea46de2489bc8a0390ea2451f1..c3edad36d9972a765bf1ebd159843b343fef00da 100644 --- a/ci/complexity_measurements/genWebpageData.py +++ b/ci/complexity_measurements/genWebpageData.py @@ -90,16 +90,16 @@ RUNS_LINE_IDX = { DISPLAY_IDS = { "wmops": [ - "3xEVS", - "6xEVS", - "10xEVS", + # "3xEVS", + # "6xEVS", + # "10xEVS", "worst case codec", "worst case enc/dec", "worst case enc", "worst case dec", ], "rom": [ - "requirementRom", + # "requirementRom", "TotalRomCodecScore", "maxPROMEncScore", "maxPROMDecScore", @@ -111,7 +111,7 @@ DISPLAY_IDS = { "maxTROMRendScore", ], "ram": [ - "requirementRam", + # "requirementRam", "maxTotalRamCodecScore", "maxTotalRamEncScore", "maxTotalRamDecScore", @@ -129,9 +129,12 @@ DISPLAY_IDS = { } # the -1's are for reference lines DISPLAY_LINE_IDX = { - "wmops": [-1, -1, -1, 9, 3, 5, 7], - "rom": [-1, 3, 5, 7, 9, 11, 13, 15, 17, 19], - "ram": [-1, 3, 5, 7, 8, 10, 12, 13, 15, 17], + # "wmops": [-1, -1, -1, 9, 3, 5, 7], + # "rom": [-1, 3, 5, 7, 9, 11, 13, 15, 17, 19], + # "ram": [-1, 3, 5, 7, 8, 10, 12, 13, 15, 17], + "wmops": [9, 3, 5, 7], + "rom": [3, 5, 7, 9, 11, 13, 15, 17, 19], + "ram": [3, 5, 7, 8, 10, 12, 13, 15, 17], "wmops_per_op": [1, 2], } DISPLAY_LABELS = { @@ -144,16 +147,16 @@ REF_COLOR_FOR_COMP_BARS = dict( ) LINE_COLORS = { "wmops": [ - REF_COLORS[0], - REF_COLORS[1], - REF_COLORS[2], + # REF_COLORS[0], + # REF_COLORS[1], + # REF_COLORS[2], "#0080FF", "#FF8000", "#CF4B4B", "#008040", ], "rom": [ - REF_COLORS[0], + # REF_COLORS[0], "#FF0000", "#FF8000", "#FFFF00", @@ -165,7 +168,7 @@ LINE_COLORS = { "#00FF00", ], "ram": [ - REF_COLORS[0], + # REF_COLORS[0], "#FF0000", "#FF8000", "#FFFF00", @@ -185,7 +188,6 @@ LINE_COLORS = { JS_FILE_TEMPLATE = """var {var_name} = {{ {elem_name}: {{ description: "{description}", - direction: -1, ticks: [ {ticks} ], @@ -199,48 +201,57 @@ JS_FILE_TEMPLATE = """var {var_name} = {{ }}; """ +# static references commented out for now FILE_DATA = { "wmops": { - "var_name": "Graphs_WMOPS", + "var_name": "Graphs_WMOPS{}", "elem_name": "wmops_worstcase", "description": "Worst Case WMOPS", - "filename": "graphs_wmops_flc.js", - "references": { - "3xEVS": REF_WMOPS_3EVS, - "6xEVS": REF_WMOPS_6EVS, - "10xEVS": REF_WMOPS_10EVS, - }, + "filename": "graphs_wmops_flc{}.js", + # "references": { + # "3xEVS": REF_WMOPS_3EVS, + # "6xEVS": REF_WMOPS_6EVS, + # "10xEVS": REF_WMOPS_10EVS, + # }, + "references": {}, }, "rom": { - "var_name": "Graphs_ROM", + "var_name": "Graphs_ROM{}", "elem_name": "rom_worstcase", "description": "ROM", - "filename": "graphs_rom_flc.js", - "references": { - "requirementRom": 0, - }, + "filename": "graphs_rom_flc{}.js", + # "references": { + # "requirementRom": 0, + # }, + "references": {}, }, "ram": { - "var_name": "Graphs_RAM", + "var_name": "Graphs_RAM{}", "elem_name": "ram_worstcase", "description": "Worst Case RAM", - "filename": "graphs_ram_flc.js", - "references": { - "requirementRam": 0, - }, + "filename": "graphs_ram_flc{}.js", + # "references": { + # "requirementRam": 0, + # }, + "references": {}, }, "wmops_per_op": { - "var_name": "Graphs_WMOPS_perOP", + "var_name": "Graphs_WMOPS_perOP{}", "elem_name": "wmops_worstcase_per_op", "description": "Worst Case WMOPS per OP", - "filename": "graphs_wmops_flc_perOP.js", + "filename": "graphs_wmops_flc_perOP{}.js", "references": {}, }, } def main( - wmops_log, wmops_per_op_log, rom_log, ram_log, wmops_per_op_log_for_comparison + wmops_log, + wmops_per_op_log, + rom_log, + ram_log, + wmops_per_op_log_for_comparison, + suffix, ): FILE_DATA["wmops"]["log_file"] = wmops_log FILE_DATA["wmops_per_op"]["log_file"] = wmops_per_op_log @@ -248,6 +259,10 @@ def main( FILE_DATA["ram"]["log_file"] = ram_log for x, data in FILE_DATA.items(): + if data["log_file"] is None: + print(f"Skipping '{x}' because no logfile given") + continue + with open(data["log_file"]) as f: log_lines = f.readlines() @@ -279,14 +294,14 @@ def main( ticks = ",\n".join(ticks) js_string = JS_FILE_TEMPLATE.format( - var_name=data["var_name"], + var_name=data["var_name"].format(suffix), elem_name=data["elem_name"], description=data["description"], runs=runs, displays=displays, ticks=ticks, ) - with open(data["filename"], "w") as f: + with open(data["filename"].format(suffix), "w") as f: print(js_string, file=f) @@ -444,11 +459,12 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="Generate Javascript code for the complexity webpage" ) - parser.add_argument("wmops_log") - parser.add_argument("wmops_per_op_log") - parser.add_argument("rom_log") - parser.add_argument("ram_log") + parser.add_argument("--wmops_log", default=None) + parser.add_argument("--wmops_per_op_log", default=None) + parser.add_argument("--rom_log", default=None) + parser.add_argument("--ram_log", default=None) parser.add_argument("--wmops_per_op_log_for_comparison", default=None) + parser.add_argument("--suffix", default="") args = parser.parse_args() @@ -458,4 +474,5 @@ if __name__ == "__main__": args.rom_log, args.ram_log, args.wmops_per_op_log_for_comparison, + args.suffix, ) diff --git a/ci/complexity_measurements/getWmops.sh b/ci/complexity_measurements/getWmops.sh index e38ba3cc234128fbfc62e341ed4c28881b58fd7f..3f49630530fe007fa4f0adefc2a72a2dc3f9a4f1 100755 --- a/ci/complexity_measurements/getWmops.sh +++ b/ci/complexity_measurements/getWmops.sh @@ -28,6 +28,8 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. +set -euxo pipefail + function usage { echo "Usage: $0 \"ivas-format(s)\" \"output-format(s)\" \"mode{full(default)|mem_only}\" \"repo{float(default)|basop}\" [wmops_ref_logfile]" exit 1 @@ -94,31 +96,59 @@ fi ./scripts/IvasBuildAndRunChecks.py $mode_arg_script $repo_arg_script -p $config_file --checks COMPLEXITY --create_complexity_tables ${wmopsFilenameFlc} -C $ivas_format $mode_arg -f ${ep} --oc $output_format ret_val=$? -# get the info on worst-case operating point: WMOPS number, enc-operating mode, dec-operating mode -### WMOPS -${scriptDir}/parseNewsletterWmops.py ${wmopsFilenameFlc}_WMOPS.csv ${wmopsFilenameFlcLast}_WMOPS.csv ${commit_sha} ${shortDate} ${fullDate} >>${destDir}/wmops/log_wmops_all.txt +# TODO: only split wmops and ram files +./ci/complexity_measurements/split_by_levels.py $(ls ${wmopsFilenameFlc}*.csv) + +# for wmops and RAM graphs, split by levels +split_suffixes=("_rate_sw" "_level_1" "_level_2" "_level_3") +for suffix in "${split_suffixes[@]}"; do + echo "$suffix" + + wmopsFilenameFlc_split="${wmopsFilenameFlc}_WMOPS${suffix}.csv" + wmopsFilenameFlcLast_split="${wmopsFilenameFlcLast}_WMOPS${suffix}.csv" + log_wmops_all_split="${destDir}/wmops/log_wmops_all${suffix}.txt" + latest_wmops_split="${wmopsFilenameFlcDir}/latest_WMOPS${suffix}.csv" + + heapFilenameFlc_split="${wmopsFilenameFlc}_HEAP${suffix}.csv" + stackFilenameFlc_split="${wmopsFilenameFlc}_STACK${suffix}.csv" + ramFilenameFlc_split="${wmopsFilenameFlc}_RAM${suffix}.csv" + ramFilenameFlcLast_split="${wmopsFilenameFlcLast}_RAM${suffix}.csv" + log_ram_all_split="${destDir}/wmops/log_ram_all${suffix}.txt" + + # get the info on worst-case operating point: WMOPS number, enc-operating mode, dec-operating mode + ### WMOPS + ${scriptDir}/parseNewsletterWmops.py ${wmopsFilenameFlc_split} ${wmopsFilenameFlcLast_split} ${commit_sha} ${shortDate} ${fullDate} >>${log_wmops_all_split} + + # create copy of WMOPS csv file with special name to easily get it from artifacts when comparing BASOP to float numbers + cp ${wmopsFilenameFlc_split} ${latest_wmops_split} + + # get memory info for webpage + ### RAM + ${scriptDir}/mergeNewsletterRam.py ${heapFilenameFlc_split} ${stackFilenameFlc_split} >${ramFilenameFlc_split} + ${scriptDir}/parseNewsletterRam.py ${heapFilenameFlc_split} ${stackFilenameFlc_split} ${ramFilenameFlcLast_split} ${commit_sha} ${shortDate} ${fullDate} >>${log_ram_all_split} -# create copy of WMOPS csv file with special name to easily get it from artifacts when comparing BASOP to float numbers -cp ${wmopsFilenameFlc}_WMOPS.csv ${wmopsFilenameFlcDir}/latest_WMOPS.csv + # generate javascript code from split log files + python3 ci/complexity_measurements/genWebpageData.py --wmops_log ${log_wmops_all_split} --ram_log ${log_ram_all_split} --suffix ${suffix} +done -# get memory info for webpage -### RAM -${scriptDir}/mergeNewsletterRam.py ${wmopsFilenameFlc}_HEAP.csv ${wmopsFilenameFlc}_STACK.csv >${wmopsFilenameFlc}_RAM.csv -${scriptDir}/parseNewsletterRam.py ${wmopsFilenameFlc}_HEAP.csv ${wmopsFilenameFlc}_STACK.csv ${wmopsFilenameFlcLast}_RAM.csv ${commit_sha} ${shortDate} ${fullDate} >>${destDir}/wmops/log_ram_all.txt +# for ROM and wmops_per_op graphs, no split is done +promFilenameFlc="${wmopsFilenameFlc}_PROM.csv" +tromFilenameFlc="${wmopsFilenameFlc}_TROM.csv" +romFilenameFlc="${wmopsFilenameFlc}_ROM.csv" +romFilenameFlcLast="${wmopsFilenameFlcLast}_ROM.csv" +log_rom_all="${destDir}/wmops/log_rom_all.txt" ### ROM -${scriptDir}/mergeNewsletterRom.py ${wmopsFilenameFlc}_PROM.csv ${wmopsFilenameFlc}_TROM.csv >${wmopsFilenameFlc}_ROM.csv -${scriptDir}/parseNewsletterRom.py ${wmopsFilenameFlc}_PROM.csv ${wmopsFilenameFlc}_TROM.csv ${wmopsFilenameFlcLast}_ROM.csv ${commit_sha} ${shortDate} ${fullDate} >>${destDir}/wmops/log_rom_all.txt +${scriptDir}/mergeNewsletterRom.py ${promFilenameFlc} ${tromFilenameFlc} >${romFilenameFlc} +${scriptDir}/parseNewsletterRom.py ${promFilenameFlc} ${tromFilenameFlc} ${romFilenameFlcLast} ${commit_sha} ${shortDate} ${fullDate} >>${log_rom_all} -# generate javascript code from log files -# pass the ref log for comparison only if arg is given if [ "$wmops_ref_logfile" != "" ]; then - python3 ci/complexity_measurements/genWebpageData.py ${destDir}/wmops/log_wmops_all.txt ${wmopsFilenameFlc}_WMOPS.csv ${destDir}/wmops/log_rom_all.txt ${destDir}/wmops/log_ram_all.txt --wmops_per_op_log_for_comparison $wmops_ref_logfile + python3 ci/complexity_measurements/genWebpageData.py --wmops_per_op_log ${wmopsFilenameFlc}_WMOPS.csv --rom_log ${log_rom_all} --wmops_per_op_log_for_comparison $wmops_ref_logfile else - python3 ci/complexity_measurements/genWebpageData.py ${destDir}/wmops/log_wmops_all.txt ${wmopsFilenameFlc}_WMOPS.csv ${destDir}/wmops/log_rom_all.txt ${destDir}/wmops/log_ram_all.txt + python3 ci/complexity_measurements/genWebpageData.py --wmops_per_op_log ${wmopsFilenameFlc}_WMOPS.csv --rom_log ${log_rom_all} fi -python3 ${scriptDir}/check_for_changes.py ${destDir}/wmops/log_wmops_all.txt ${destDir}/wmops/log_ram_all.txt ${destDir}/wmops/log_rom_all.txt +python3 ${scriptDir}/check_for_changes.py --wmops_logfiles $(ls ${destDir}/wmops/log_wmops_all*.txt) --ram_logfiles $(ls ${destDir}/wmops/log_ram_all*.txt) --rom_logfile ${destDir}/wmops/log_rom_all.txt if [ "$?" != "0" ]; then ret_val=1 fi diff --git a/ci/complexity_measurements/index_complexity.html b/ci/complexity_measurements/index_complexity.html index baa3f531c1b7357d1e67a47c5f4cb3fe859a80ee..7f52cf9f6683a9c634ef7c7aa8f2ca722d6be790 100755 --- a/ci/complexity_measurements/index_complexity.html +++ b/ci/complexity_measurements/index_complexity.html @@ -44,271 +44,158 @@ - - + + + + + - - - + + + + + + - - -

IVAS FORMAT - Worst Case WMOPS Performance

-
-
-
- -
- -
+
+

Level 1 (13.2 - 80 kbps)

+
+
+ +
+

Level 2 (96 - 192 kbps)

+
+
+ +
+

Level 3 (256 - 512 kbps)

+
+
+ +
+

Rate switching (level-independent)

+
+
+ +
+ +
-

IVAS FORMAT - Worst Case WMOPS Performance per Operating Point

+

IVAS FORMAT - Worst Case WMOPS Performance per Operating Point

Encoder

-
-
-
+
+
+

Decoder

-
-
-
- -
-
-
- -
- - - +
+

Rate switching (level-independent)

+
+
-

IVAS FORMAT - Worst Case RAM Demand

-
-
-
- -
- +
-
+
- +
+ +
+ +

FAQ

Q:
Which input files are used for audio-input? What error pattern is used?
A:
The input files can be found here. The error pattern is here
. -
Q:
The legend lists some References, but I can't see them in the plot?
-
A:
The plot Axes are scaled to fit the measured numbers. The reference values might be too high to fit into the plot. Put differently: if you can't see the references, you are probably below them (all is well).
. -
Q:
What is the meaning of these funny symbols in the navigation box, in the left upper corner of this page?
-
A:
- 1) Traffic light ●●●, ●●● or ●●●: !!!CURRENTLY NOT WORKING CORRECTLY AS NO REQUIREMENTS DEFINED YET!!! The traffic light symbols show, whether the last datapoint matches the requirement (green) or not (red). A yellow traffic light means that the requirement is matched, but the score is very close (within a 3% margin) to the requirement.
- 2) Arrow ↑, ↓, →: The arrow indicates the trend of the last datapoint, compared to the last but one. An upwards arrow means that the score got higher (i.e. worse), downwards arrow arrow means that the score got lower (i.e. better), and a rightwards arrow means that the score was kept constant (within a 1% margin). -
@@ -433,8 +320,11 @@ return max; } + function wmopsFormatter(v, axis) { + return v + " WMOPS"; + } + function WMOPS() { - var previousPoint = null; function drawGraph(elt, graph, max_val) { @@ -442,12 +332,7 @@ yaxis: { min: 0, max: max_val, - tickFormatter: function (v, axis) { - if (graph.direction == -1) - return v + " WMOPS"; - return v; - }, - invert: graph.direction == 1 + tickFormatter: wmopsFormatter, }, xaxis: { tickFormatter: function (v, axis) { @@ -488,11 +373,7 @@ var y = item.datapoint[1]; var text = "" + item.series.id + "
" - text += 'Score: ' + y; - - if (graph.direction == -1) - text += " WMOPS"; - text += "
"; + text += 'Score: ' + y + " WMOPS" + "
"; // add more info only if not a reference line if ( !item.series.id.includes("EVS") ) @@ -503,8 +384,7 @@ var diff = Math.round((thisValue - prevValue) * 100) / 100; var pdiff = calcPercentDiff(thisValue, prevValue); var better; - if ((pdiff < 0 && graph.direction == -1) || - (pdiff > 0 && graph.direction == 1)) { + if (pdiff < 0) { better = "worse"; } else { better = "better"; @@ -512,44 +392,28 @@ pdiff = Math.abs(pdiff); if (diff === diff) { text += String.fromCharCode(916) + ": " + diff; - if (graph.direction == -1) - text += " WMOPS"; + text += " WMOPS"; text += " (" + pdiff + "% " + better + ")
"; } } if( item.series.id == "worst case enc/dec" ){ - text += "Worst case enc: " + graph.runs[x].worstCaseEnc + "
"; - text += "Worst case dec: " + graph.runs[x].worstCaseDec + "
"; + text += "Encoder OP: " + graph.runs[x].worstCaseEnc + "
"; + text += "Decoder OP: " + graph.runs[x].worstCaseDec + "
"; } if( item.series.id == "worst case codec" ){ - text += "Worst case codec: " + graph.runs[x].worstCaseCodec + "
"; + text += "OP: " + graph.runs[x].worstCaseCodec + "
"; } if( item.series.id == "worst case enc" ){ - text += "Worst case enc: " + graph.runs[x].worstCaseEnc + "
"; + text += "OP: " + graph.runs[x].worstCaseEnc + "
"; } if( item.series.id == "worst case dec" ){ - text += "Worst case dec: " + graph.runs[x].worstCaseDec + "
"; - } - if( item.series.id == "worst case enc/dec rs" ){ - text += "Worst case enc rateswitching: " + graph.runs[x].worstCaseEncRs + "
"; - text += "Worst case dec rateswitching: " + graph.runs[x].worstCaseDecRs + "
"; - } - if( item.series.id == "worst case codec rs" ){ - text += "Worst case codec rateswitching: " + graph.runs[x].worstCaseCodecRs + "
"; - } - if( item.series.id == "worst case enc rs" ){ - text += "Worst case enc rateswitching: " + graph.runs[x].worstCaseEncRs + "
"; + text += "OP: " + graph.runs[x].worstCaseDec + "
"; } - if( item.series.id == "worst case dec rs" ){ - text += "Worst case dec rateswitching: " + graph.runs[x].worstCaseDecRs + "
"; - } - text += "
" text += "Revision: " + graph.runs[x].revision + "
"; text += "Date: " + graph.runs[x].fullDate + "
"; - text += "Fixpoint scal. fac. to reach 138 WMOPS: " + graph.runs[x].fixpointScalingFac + "

"; text += "Logfile
"; } @@ -559,44 +423,25 @@ } $(document).ready(function () { - var max = get_max_y_val_for_plotting(Graphs_WMOPS.wmops_worstcase.displays, 50); - drawGraph($("#wmops-graph"), Graphs_WMOPS.wmops_worstcase, max); + var max = get_max_y_val_for_plotting(Graphs_WMOPS_level_1.wmops_worstcase.displays, 50); + drawGraph($("#wmops-graph-level_1"), Graphs_WMOPS_level_1.wmops_worstcase, max); + }); + $(document).ready(function () { + var max = get_max_y_val_for_plotting(Graphs_WMOPS_level_2.wmops_worstcase.displays, 50); + drawGraph($("#wmops-graph-level_2"), Graphs_WMOPS_level_2.wmops_worstcase, max); + }); + $(document).ready(function () { + var max = get_max_y_val_for_plotting(Graphs_WMOPS_level_3.wmops_worstcase.displays, 50); + drawGraph($("#wmops-graph-level_3"), Graphs_WMOPS_level_3.wmops_worstcase, max); + }); + $(document).ready(function () { + var max = get_max_y_val_for_plotting(Graphs_WMOPS_rate_sw.wmops_worstcase.displays, 50); + drawGraph($("#wmops-graph-rate_sw"), Graphs_WMOPS_rate_sw.wmops_worstcase, max); }); - - var refData = Graphs_WMOPS.wmops_worstcase.displays[0]; - var testData = Graphs_WMOPS.wmops_worstcase.displays[2]; - var testDataRs = Graphs_WMOPS.wmops_worstcase.displays[6]; - var nEntries = testData.data.length; - - if( testDataRs.data[nEntries-1][1] > testData.data[nEntries-1][1] ) { - testData = testDataRs; - } - - if(testData.data[nEntries-1][1] > refData.data[nEntries-1][1] ) { - document.getElementById("wmops_tl_l").style.color="#FF0000"; - } else if(testData.data[nEntries-1][1] > 0.97 * refData.data[nEntries-1][1] ) { - document.getElementById("wmops_tl_c").style.color="#FFFF00"; - } else { - document.getElementById("wmops_tl_r").style.color="#00FF00"; - } - - if(nEntries > 1) { - if( testData.data[nEntries-1][1] > 1.01 * testData.data[nEntries-2][1] ) { - document.getElementById("wmops_trend").innerHTML="↑"; - document.getElementById("wmops_trend").style.color="#FF0000"; - } else if(testData.data[nEntries-1][1] < 0.99 * testData.data[nEntries-2][1] ) { - document.getElementById("wmops_trend").innerHTML="↓"; - document.getElementById("wmops_trend").style.color="#00FF00"; - } else { - document.getElementById("wmops_trend").innerHTML="→"; - document.getElementById("wmops_trend").style.color="#FFFFFF"; - } } -} function WMOPS_perOP() { - var previousPoint = null; function drawGraph(elt, graph, max_val) { @@ -604,12 +449,7 @@ function WMOPS_perOP() { yaxis: { min: 0, max: max_val, - tickFormatter: function (v, axis) { - if (graph.direction == -1) - return v + " WMOPS"; - return v; - }, - invert: graph.direction == 1 + tickFormatter: wmopsFormatter, }, xaxis: { ticks: graph.ticks @@ -710,20 +550,35 @@ function WMOPS_perOP() { drawGraph($("#wmops_per_op-graph-dec"), graph_dec, max_dec); }); - var nEntriesWmopsGraph = Graphs_WMOPS.wmops_worstcase.runs.length - 1; + var nEntriesWmopsGraph = Graphs_WMOPS_level_1.wmops_worstcase.runs.length - 1; var legend = "

Numbers derived from revision " + - Graphs_WMOPS.wmops_worstcase.runs[nEntriesWmopsGraph].revision + + Graphs_WMOPS_level_1.wmops_worstcase.runs[nEntriesWmopsGraph].revision + ", " + - Graphs_WMOPS.wmops_worstcase.runs[nEntriesWmopsGraph].fullDate + + Graphs_WMOPS_level_1.wmops_worstcase.runs[nEntriesWmopsGraph].fullDate + "

"; document.getElementById("wmops_per_op-legend-enc").innerHTML = legend; } + function memTickFormatter(v, axis) { + var decimals = axis.tickDecimals; + var str_val = ""; + if (v > 1000000) { + str_val = (v / 1000000).toFixed(decimals) + " MB"; + } + else if (v > 1000) { + str_val = (v / 1000).toFixed(decimals) + " kB"; + } + else { + str_val = v.toFixed(decimals) + " B"; + } + return str_val; + } + function RAM() { - + var previousPoint = null; function drawGraph(elt, graph, max_val) { @@ -731,12 +586,7 @@ function RAM() { yaxis: { min: 0, max: max_val, - tickFormatter: function (v, axis) { - if (graph.direction == -1) - return v + " bytes"; - return v; - }, - invert: graph.direction == 1 + tickFormatter: memTickFormatter, }, xaxis: { tickFormatter: function (v, axis) { @@ -781,8 +631,7 @@ function RAM() { var text = "" + item.series.id + "
" text += 'Score: ' + y; - if (graph.direction == -1) - text += " bytes"; + text += " bytes"; text += "
"; if (x > 0) { @@ -791,8 +640,7 @@ function RAM() { var diff = Math.round((thisValue - prevValue) * 100) / 100; var pdiff = calcPercentDiff(thisValue, prevValue); var better; - if ((pdiff < 0 && graph.direction == -1) || - (pdiff > 0 && graph.direction == 1)) { + if (pdiff < 0) { better = "worse"; } else { better = "better"; @@ -800,8 +648,7 @@ function RAM() { pdiff = Math.abs(pdiff); if (diff === diff) { text += String.fromCharCode(916) + ": " + diff; - if (graph.direction == -1) - text += " bytes"; + text += " bytes"; text += " (" + pdiff + "% " + better + ")
"; } } @@ -852,40 +699,26 @@ function RAM() { } $(document).ready(function () { - var max = get_max_y_val_for_plotting(Graphs_RAM.ram_worstcase.displays, 50000); - drawGraph($("#ram-graph"), Graphs_RAM.ram_worstcase, max); + var max = get_max_y_val_for_plotting(Graphs_RAM_level_1.ram_worstcase.displays, 50000); + drawGraph($("#ram-graph-level_1"), Graphs_RAM_level_1.ram_worstcase, max); + }); + $(document).ready(function () { + var max = get_max_y_val_for_plotting(Graphs_RAM_level_2.ram_worstcase.displays, 50000); + drawGraph($("#ram-graph-level_2"), Graphs_RAM_level_2.ram_worstcase, max); + }); + $(document).ready(function () { + var max = get_max_y_val_for_plotting(Graphs_RAM_level_3.ram_worstcase.displays, 50000); + drawGraph($("#ram-graph-level_3"), Graphs_RAM_level_3.ram_worstcase, max); + }); + $(document).ready(function () { + var max = get_max_y_val_for_plotting(Graphs_RAM_rate_sw.ram_worstcase.displays, 50000); + drawGraph($("#ram-graph-rate_sw"), Graphs_RAM_rate_sw.ram_worstcase, max); }); - - var testData = Graphs_RAM.ram_worstcase.displays[1]; - var refData = Graphs_RAM.ram_worstcase.displays[0]; - var nEntries = testData.data.length; - - if( testData.data[nEntries-1][1] > refData.data[nEntries-1][1] ) { - document.getElementById("ram_tl_l").style.color="#FF0000"; - } else if( testData.data[nEntries-1][1] > 0.97 * refData.data[nEntries-1][1] ) { - document.getElementById("ram_tl_c").style.color="#FFFF00"; - } else { - document.getElementById("ram_tl_r").style.color="#00FF00"; - } - - if(nEntries > 1) { - if( testData.data[nEntries-1][1] > 1.01 * testData.data[nEntries-2][1] ) { - document.getElementById("ram_trend").innerHTML="↑"; - document.getElementById("ram_trend").style.color="#FF0000"; - } else if( testData.data[nEntries-1][1] < 0.99 * testData.data[nEntries-2][1] ) { - document.getElementById("ram_trend").innerHTML="↓"; - document.getElementById("ram_trend").style.color="#00FF00"; - } else { - document.getElementById("ram_trend").innerHTML="→"; - document.getElementById("ram_trend").style.color="#FFFFFF"; - } - } } function ROM() { - var previousPoint = null; function drawGraph(elt, graph, max_val) { @@ -893,12 +726,7 @@ function ROM() { yaxis: { min: 0, max: max_val, - tickFormatter: function (v, axis) { - if (graph.direction == -1) - return v + " bytes"; - return v; - }, - invert: graph.direction == 1 + tickFormatter: memTickFormatter, }, xaxis: { tickFormatter: function (v, axis) { @@ -943,8 +771,7 @@ function ROM() { var text = "" + item.series.id + "
" text += 'Score: ' + y; - if (graph.direction == -1) - text += " bytes"; + text += " bytes"; text += "
"; if (x > 0) { @@ -953,8 +780,7 @@ function ROM() { var diff = Math.round((thisValue - prevValue) * 100) / 100; var pdiff = calcPercentDiff(thisValue, prevValue); var better; - if ((pdiff < 0 && graph.direction == -1) || - (pdiff > 0 && graph.direction == 1)) { + if (pdiff < 0) { better = "worse"; } else { better = "better"; @@ -962,62 +788,16 @@ function ROM() { pdiff = Math.abs(pdiff); if (diff === diff) { text += String.fromCharCode(916) + ": " + diff; - if (graph.direction == -1) - text += " bytes"; + text += " bytes"; text += " (" + pdiff + "% " + better + ")
"; } } -/* - if( item.series.id == "TotalRomCodecScore" ){ - text += "Worst case enc: " + graph.runs[x].TotalRomEnc + "
"; - text += "Worst case dec: " + graph.runs[x].TotalRomDec + "
"; - } - if( item.series.id == "TotalRomEncScore" ){ - text += "Worst case enc: " + graph.runs[x].TotalRomEnc + "
"; - } - if( item.series.id == "TotalRomDecScore" ){ - text += "Worst case dec: " + graph.runs[x].TotalRomDec + "
"; - } - if( item.series.id == "PROMCodecScore" ){ - text += "Worst case enc: " + graph.runs[x].PromEnc + "
"; - text += "Worst case dec: " + graph.runs[x].PromDec + "
"; - } - if( item.series.id == "PROMEncScore" ){ - text += "Worst case enc: " + graph.runs[x].PromEnc + "
"; - } - if( item.series.id == "PROMDecScore" ){ - text += "Worst case dec: " + graph.runs[x].PromDec + "
"; - } - if( item.series.id == "TROMCodecScore" ){ - text += "Worst case enc: " + graph.runs[x].TromEnc + "
"; - text += "Worst case dec: " + graph.runs[x].TromDec + "
"; - } - if( item.series.id == "TROMEncScore" ){ - text += "Worst case enc: " + graph.runs[x].TromEnc + "
"; - } - if( item.series.id == "TROMDecScore" ){ - text += "Worst case dec: " + graph.runs[x].TromDec + "
"; - } -*/ - text += "
" text += "Revision: " + graph.runs[x].revision + "
"; text += "Date: " + graph.runs[x].fullDate + "

"; text += "Logfile
"; - /* - if( item.series.id == "maxRomFlc" ){ - text += "Logfile
"; - } - */ - - /* - if( item.series.id == "maxRomBasop" ){ - text += "Logfile
"; - } - */ - } else { text = "ROM requirement: 800000 bytes"; } @@ -1032,66 +812,10 @@ function ROM() { drawGraph($("#rom-graph"), Graphs_ROM.rom_worstcase, max); }); - - /* FLC */ - var testData = Graphs_ROM.rom_worstcase.displays[1]; - var refData = Graphs_ROM.rom_worstcase.displays[0]; - var nEntries = testData.data.length; - - if( testData.data[nEntries-1][1] > refData.data[nEntries-1][1] ) { - document.getElementById("rom_tl_l").style.color="#FF0000"; - } else if( testData.data[nEntries-1][1] > 0.97 * refData.data[nEntries-1][1] ) { - document.getElementById("rom_tl_c").style.color="#FFFF00"; - } else { - document.getElementById("rom_tl_r").style.color="#00FF00"; - } - - if(nEntries > 1) { - if( testData.data[nEntries-1][1] > 1.01 * testData.data[nEntries-2][1] ) { - document.getElementById("rom_trend").innerHTML="↑"; - document.getElementById("rom_trend").style.color="#FF0000"; - } else if( testData.data[nEntries-1][1] < 0.99 * testData.data[nEntries-2][1] ) { - document.getElementById("rom_trend").innerHTML="↓"; - document.getElementById("rom_trend").style.color="#00FF00"; - } else { - document.getElementById("rom_trend").innerHTML="→"; - document.getElementById("rom_trend").style.color="#FFFFFF"; - } - } - - - /* BASOP */ - /* - var testData = Graphs_ROM_BASOP.rom_worstcase.displays[2]; - var refData = Graphs_ROM.rom_worstcase.displays[0]; - var nEntries = testData.data.length; - - if( testData.data[nEntries-1][1] > refData.data[nEntries-1][1] ) { - document.getElementById("rom_basop_tl_l").style.color="#FF0000"; - } else if( testData.data[nEntries-1][1] > 0.97 * refData.data[nEntries-1][1] ) { - document.getElementById("rom_basop_tl_c").style.color="#FFFF00"; - } else { - document.getElementById("rom_basop_tl_r").style.color="#00FF00"; - } - - if(nEntries > 1) { - if( testData.data[nEntries-1][1] > 1.01 * testData.data[nEntries-2][1] ) { - document.getElementById("rom_basop_trend").innerHTML="↑"; - document.getElementById("rom_basop_trend").style.color="#FF0000"; - } else if( testData.data[nEntries-1][1] < 0.99 * testData.data[nEntries-2][1] ) { - document.getElementById("rom_basop_trend").innerHTML="↓"; - document.getElementById("rom_basop_trend").style.color="#00FF00"; - } else { - document.getElementById("rom_basop_trend").innerHTML="→"; - document.getElementById("rom_basop_trend").style.color="#FFFFFF"; - } - } - */ } WMOPS(); WMOPS_perOP(); - RAM(); ROM(); diff --git a/ci/complexity_measurements/split_by_levels.py b/ci/complexity_measurements/split_by_levels.py new file mode 100755 index 0000000000000000000000000000000000000000..3b4320abefa46ee309f080193a088553b3a990bf --- /dev/null +++ b/ci/complexity_measurements/split_by_levels.py @@ -0,0 +1,83 @@ +#! /usr/bin/env python3 + +# (C) 2022-2025 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository. All Rights Reserved. + +# This software is protected by copyright law and by international treaties. +# The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, +# Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., +# Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, +# Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other +# contributors to this repository retain full ownership rights in their respective contributions in +# the software. This notice grants no license of any kind, including but not limited to patent +# license, nor is any license granted by implication, estoppel or otherwise. + +# Contributors are required to enter into the IVAS codec Public Collaboration agreement before making +# contributions. + +# This software is provided "AS IS", without any express or implied warranties. The software is in the +# development stage. It is intended exclusively for experts who have experience with such software and +# solely for the purpose of inspection. All implied warranties of non-infringement, merchantability +# and fitness for a particular purpose are hereby disclaimed and excluded. + +# Any dispute, controversy or claim arising under or in relation to providing this software shall be +# submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in +# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and +# the United Nations Convention on Contracts on the International Sales of Goods. + +import pandas as pd +import numpy as np +import argparse +import pathlib + +LEVELS_2_MAX_BR = { + "_level_1": (0, 80), + "_level_2": (96, 192), + "_level_3": (256, 512), +} + + +def main(csv_files: list): + for csv_file in csv_files: + df = pd.read_csv(csv_file, delimiter=";") + + mask_vbr = df["conf"].str.contains("RS") + df_vbr = df[mask_vbr] + filename_vbr = str(csv_file.with_suffix("")) + "_rate_sw" + csv_file.suffix + df_vbr.to_csv( + filename_vbr, + index=False, + sep=";", + ) + + df_cbr = df[np.logical_not(mask_vbr)] + columns = df_cbr.columns + df_cbr["bitrate"] = df_cbr["conf"].str.extract(r"@(.*) kbps").astype("float") + + for lvl_suffix, (min_br, max_br) in LEVELS_2_MAX_BR.items(): + mask_lvl = np.logical_and( + df_cbr["bitrate"] >= min_br, df_cbr["bitrate"] <= max_br + ) + df_lvl = df_cbr[mask_lvl] + filename_lvl = str(csv_file.with_suffix("")) + lvl_suffix + csv_file.suffix + df_lvl[columns].to_csv( + filename_lvl, + index=False, + sep=";", + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "csv_files", + nargs="+", + help="CSV files to split by complexity levels.", + type=pathlib.Path, + ) + + args = parser.parse_args() + main(args.csv_files) diff --git a/ci/complexity_measurements/style.css b/ci/complexity_measurements/style.css index 5450bf1e8656b174862459d3306c09e9080d1f08..4e4d0fa9da5a92371f558735c45f50f9748b3e4d 100755 --- a/ci/complexity_measurements/style.css +++ b/ci/complexity_measurements/style.css @@ -49,7 +49,22 @@ hr { background-color: #FF8000; border-radius: 20px; } -#wmops-graph { +#wmops-graph-level_1 { + height:500px; + width:1600px; + float:left; +} +#wmops-graph-level_2 { + height:500px; + width:1600px; + float:left; +} +#wmops-graph-level_3 { + height:500px; + width:1600px; + float:left; +} +#wmops-graph-rate_sw { height:500px; width:1600px; float:left; @@ -89,7 +104,22 @@ hr { width:1600px; float:left; } -#ram-graph { +#ram-graph-level_1 { + height:500px; + width:1600px; + float:left; +} +#ram-graph-level_2 { + height:500px; + width:1600px; + float:left; +} +#ram-graph-level_3 { + height:500px; + width:1600px; + float:left; +} +#ram-graph-rate_sw { height:500px; width:1600px; float:left;