From 22b7a4dc793e59c461dd8d7498719679e0cfcd53 Mon Sep 17 00:00:00 2001 From: Tapani Pihlajakuja Date: Mon, 24 Jun 2024 16:23:46 +0300 Subject: [PATCH] Remove separate MASA test system and add the coverage to param file tests. --- scripts/config/self_test.prm | 56 +++ scripts/config/self_test_ltv.prm | 56 +++ .../test_masa_enc_dec.py | 338 ------------------ 3 files changed, 112 insertions(+), 338 deletions(-) delete mode 100644 tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py diff --git a/scripts/config/self_test.prm b/scripts/config/self_test.prm index 3a071276cb..ea60b47439 100644 --- a/scripts/config/self_test.prm +++ b/scripts/config/self_test.prm @@ -1067,6 +1067,62 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 512000 48 testv/stv2MASA2TC48c.wav bit ../IVAS_dec 5_1 48 bit testv/stv2MASA2TC48c.wav_512000_48-48_5_1.tst +// MASA 1dir 1TC at 13.2 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/stv1MASA1TC48c.met 13200 48 testv/stv1MASA1TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv1MASA1TC48c.wav_13200_48-48_EXT.tst + +// MASA 1dir 2TC at 16.4 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv1MASA2TC48c.met 16400 48 testv/stv1MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv1MASA2TC48c.wav_16400_48-48_EXT.tst + +// MASA 2dir 1TC at 24.4 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/stv2MASA1TC48c.met 24400 48 testv/stv2MASA1TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA1TC48c.wav_24400_48-48_EXT.tst + +// MASA 2dir 2TC at 32 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 32000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_32000_48-48_EXT.tst + +// MASA 1dir 1TC at 48 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/stv1MASA1TC48c.met 48000 48 testv/stv1MASA1TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv1MASA1TC48c.wav_48000_48-48_EXT.tst + +// MASA 1dir 2TC at 64 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv1MASA2TC48c.met 64000 48 testv/stv1MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv1MASA2TC48c.wav_64000_48-48_EXT.tst + +// MASA 2dir 1TC at 80 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/stv2MASA1TC48c.met 80000 48 testv/stv2MASA1TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA1TC48c.wav_80000_48-48_EXT.tst + +// MASA 2dir 2TC at 96 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 96000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_96000_48-48_EXT.tst + +// MASA 2dir 2TC at 128 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 128000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_128000_48-48_EXT.tst + +// MASA 2dir 2TC at 160 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 160000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_160000_48-48_EXT.tst + +// MASA 2dir 2TC at 192 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 192000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_192000_48-48_EXT.tst + +// MASA 2dir 2TC at 256 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 256000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_256000_48-48_EXT.tst + +// MASA 2dir 2TC at 384 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 384000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_384000_48-48_EXT.tst + +// MASA 2dir 2TC at 512 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/stv2MASA2TC48c.met 512000 48 testv/stv2MASA2TC48c.wav bit +../IVAS_dec EXT 48 bit testv/stv2MASA2TC48c.wav_512000_48-48_EXT.tst + // MASA 1dir 1TC at 13.2 kbps, 48kHz in, 48kHz out, DTX on, 7_1_4 out ../IVAS_cod -dtx -masa 1 testv/stv1MASA1TC48n.met 13200 48 testv/stv1MASA1TC48n.wav bit ../IVAS_dec 7_1_4 48 bit testv/stv1MASA1TC48n.wav_13200_48-48_DTX_7_1_4.tst diff --git a/scripts/config/self_test_ltv.prm b/scripts/config/self_test_ltv.prm index 2a66699563..aa5f8b1eeb 100644 --- a/scripts/config/self_test_ltv.prm +++ b/scripts/config/self_test_ltv.prm @@ -1066,6 +1066,62 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1 ../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 512000 48 testv/ltv48_MASA2TC.wav bit ../IVAS_dec 5_1 48 bit testv/ltv48_MASA2TC.wav_512000_48-48_5_1.tst +// MASA 1TC at 13.2 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 13200 48 testv/ltv48_MASA1TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_13200_48-48_EXT.tst + +// MASA 2TC at 16.4 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 16400 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_16400_48-48_EXT.tst + +// MASA 1TC at 24.4 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 24400 48 testv/ltv48_MASA1TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_24400_48-48_EXT.tst + +// MASA 2TC at 32 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 32000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_32000_48-48_EXT.tst + +// MASA 1TC at 48 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 48000 48 testv/ltv48_MASA1TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_48000_48-48_EXT.tst + +// MASA 2TC at 64 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 64000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_64000_48-48_EXT.tst + +// MASA 1TC at 80 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 1 testv/ltv48_MASA1TC.met 80000 48 testv/ltv48_MASA1TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA1TC.wav_80000_48-48_EXT.tst + +// MASA 2TC at 96 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 96000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_96000_48-48_EXT.tst + +// MASA 2TC at 128 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 128000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_128000_48-48_EXT.tst + +// MASA 2TC at 160 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 160000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_160000_48-48_EXT.tst + +// MASA 2TC at 192 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 192000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_192000_48-48_EXT.tst + +// MASA 2TC at 256 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 256000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_256000_48-48_EXT.tst + +// MASA 2TC at 384 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 384000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_384000_48-48_EXT.tst + +// MASA 2TC at 512 kbps, 48kHz in, 48kHz out, EXT out +../IVAS_cod -masa 2 testv/ltv48_MASA2TC.met 512000 48 testv/ltv48_MASA2TC.wav bit +../IVAS_dec EXT 48 bit testv/ltv48_MASA2TC.wav_512000_48-48_EXT.tst + // MASA 1dir 1TC at 13.2 kbps, 48kHz in, 48kHz out, DTX on, 7_1_4 out ../IVAS_cod -dtx -masa 1 testv/ltv48_MASA1TC.met 13200 48 testv/ltv48_MASA1TC.wav bit ../IVAS_dec 7_1_4 48 bit testv/ltv48_MASA1TC.wav_13200_48-48_DTX_7_1_4.tst diff --git a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py b/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py deleted file mode 100644 index 34fa40e278..0000000000 --- a/tests/codec_be_on_mr_nonselection/test_masa_enc_dec.py +++ /dev/null @@ -1,338 +0,0 @@ -__copyright__ = """ - (C) 2022-2024 IVAS codec Public Collaboration with portions copyright Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository. All Rights Reserved. - - This software is protected by copyright law and by international treaties. - The IVAS codec Public Collaboration consisting of Dolby International AB, Ericsson AB, - Fraunhofer-Gesellschaft zur Foerderung der angewandten Forschung e.V., Huawei Technologies Co. LTD., - Koninklijke Philips N.V., Nippon Telegraph and Telephone Corporation, Nokia Technologies Oy, Orange, - Panasonic Holdings Corporation, Qualcomm Technologies, Inc., VoiceAge Corporation, and other - contributors to this repository retain full ownership rights in their respective contributions in - the software. This notice grants no license of any kind, including but not limited to patent - license, nor is any license granted by implication, estoppel or otherwise. - - Contributors are required to enter into the IVAS codec Public Collaboration agreement before making - contributions. - - This software is provided "AS IS", without any express or implied warranties. The software is in the - development stage. It is intended exclusively for experts who have experience with such software and - solely for the purpose of inspection. All implied warranties of non-infringement, merchantability - and fitness for a particular purpose are hereby disclaimed and excluded. - - Any dispute, controversy or claim arising under or in relation to providing this software shall be - submitted to and settled by the final, binding jurisdiction of the courts of Munich, Germany in - accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and - the United Nations Convention on Contracts on the International Sales of Goods. - """ - -__doc__ = """ - Test file to run C encoder and decoder code. - The outputs are compared with C generated references. - """ - -import errno -import os -import re -from filecmp import cmp -from typing import Optional - -import pytest - -from tests.cmp_pcm import cmp_pcm -from tests.conftest import DecoderFrontend, EncoderFrontend - -from ..constants import MLD_PATTERN, MAX_DIFF_PATTERN - -# params -# output_mode_list = ['MONO', 'STEREO', '5_1', '7_1', '5_1_2', '5_1_4', '7_1_4', 'FOA', 'HOA2', 'HOA3', 'BINAURAL', 'BINAURAL_ROOM', 'EXT'] -output_mode_list = ["BINAURAL", "EXT"] -ivas_br_masa = [ - 13200, - 16400, - 24400, - 32000, - 48000, - 64000, - 80000, - 96000, - 128000, - 160000, - 192000, - 256000, - 384000, - 512000, -] - -# Write file-based parameter sets here (metafile, pcm/wave, numDir, numTC, DTX_toggle) -masa_metadata_audio_ndir_ntransportch_dtx_list = [ # ('stv1MASA1TC48c.met', 'stv1MASA1TC48c.wav', 1, 1, False), - # ('stv1MASA2TC48c.met', 'stv1MASA2TC48c.wav', 1, 2, False), - ("stv2MASA1TC48c.met", "stv2MASA1TC48c.wav", 2, 1, False), - # ('stv2MASA2TC48c.met', 'stv2MASA2TC48c.wav', 2, 2, False), - # ('stv1MASA1TC48n.met', 'stv1MASA1TC48n.wav', 1, 1, True), - ("stv1MASA2TC48n.met", "stv1MASA2TC48n.wav", 1, 2, True), -] - -# Used to not test every combination -test_split_br = [13200, 24400, 48000, 80000, 128000, 256000, 512000] -AbsTol = "0" - - -def check_and_makedir(dir_path): - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise # raises the error again - - -@pytest.mark.create_ref -@pytest.mark.parametrize("output_mode", output_mode_list) -@pytest.mark.parametrize("ivas_br", ivas_br_masa) -@pytest.mark.parametrize( - "masa_metadata_audio_ndir_ntransportch_dtx", - masa_metadata_audio_ndir_ntransportch_dtx_list, -) -def test_masa_enc_dec( - record_property, - dut_encoder_frontend: EncoderFrontend, - dut_decoder_frontend: DecoderFrontend, - ref_encoder_frontend: EncoderFrontend, - ref_decoder_frontend: DecoderFrontend, - reference_path, - dut_base_path, - update_ref, - keep_files, - ivas_br, - masa_metadata_audio_ndir_ntransportch_dtx, - test_vector_path, - output_mode, - get_mld, - get_mld_lim, - decoder_only, - abs_tol, -): - # Input parameters - in_fs = 48 - out_fs = 48 - metadata = masa_metadata_audio_ndir_ntransportch_dtx[0] - input_audio = masa_metadata_audio_ndir_ntransportch_dtx[1] - n_directions = masa_metadata_audio_ndir_ntransportch_dtx[2] - masa_channel_count = masa_metadata_audio_ndir_ntransportch_dtx[3] - dtx = masa_metadata_audio_ndir_ntransportch_dtx[4] - masa_path = f"{test_vector_path}/{metadata}" - input_audio_path = f"{test_vector_path}/{input_audio}" - - # Apply test skipping here - if dtx: - if output_mode != "EXT": - if ivas_br not in test_split_br: - pytest.skip( - "Skipping some DTX bitrates for other than EXT output to save time" - ) - - # Set reference encoder and decoder - - # Set output paths - out_dir_bs_ref = f"{reference_path}/masa_test/bitstreams" - out_dir_bs_dut = f"{dut_base_path}/masa_test/bitstreams" - out_dir_dec_output_ref = f"{reference_path}/masa_test/dec_output" - out_dir_dec_output_dut = f"{dut_base_path}/masa_test/dec_output" - check_and_makedir(out_dir_bs_ref) - check_and_makedir(out_dir_bs_dut) - check_and_makedir(out_dir_dec_output_ref) - check_and_makedir(out_dir_dec_output_dut) - - # To avoid conflicting names in case of parallel test execution, differentiate all cases - output_bitstream_ref = f"{out_dir_bs_ref}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.192" - output_bitstream_dut = f"{out_dir_bs_dut}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.192" - dec_output_ref = f"{out_dir_dec_output_ref}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.wav" - dec_output_dut = f"{out_dir_dec_output_dut}/masa{masa_channel_count}_ndirs{n_directions}_outputmode{output_mode}_ivasbr{ivas_br}k_DTX{dtx}.wav" - if output_mode == "EXT": - dec_met_output_ref = f"{dec_output_ref}.met" - dec_met_output_dut = f"{dec_output_dut}.met" - else: - dec_met_output_ref = None - dec_met_output_dut = None - - if update_ref == 2 or update_ref == 1: - # Encode REF - ivas_enc( - ref_encoder_frontend, - masa_channel_count, - masa_path, - ivas_br, - in_fs, - input_audio_path, - output_bitstream_ref, - dtx, - ) - - # Decode REF - ivas_dec( - ref_decoder_frontend, - output_mode, - out_fs, - output_bitstream_ref, - dec_output_ref, - ) - - if update_ref == 2 or update_ref == 0: - if decoder_only: - output_bitstream_dut = output_bitstream_ref - else: - # Encode DUT - ivas_enc( - dut_encoder_frontend, - masa_channel_count, - masa_path, - ivas_br, - in_fs, - input_audio_path, - output_bitstream_dut, - dtx, - ) - - # Decode DUT - ivas_dec( - dut_decoder_frontend, - output_mode, - out_fs, - output_bitstream_dut, - dec_output_dut, - ) - - # Compare outputs. For EXT output, also compare metadata. - if output_mode == "EXT": - # Compare metadata as binary blob - metacmp_res = cmp(dec_met_output_ref, dec_met_output_dut) - - # Compare audio outputs - pcmcmp_res, reason = cmp_pcm( - dec_output_dut, - dec_output_ref, - output_mode, - int(out_fs * 1000), - get_mld=get_mld, - mld_lim=get_mld_lim, - abs_tol=abs_tol, - ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - max_diff = 0 - if pcmcmp_res: - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) - - if get_mld and get_mld_lim > 0: - if pcmcmp_res != 0: - pytest.fail(reason) - else: - # Fail if compare fails compare result - if metacmp_res == False and pcmcmp_res != 0: - pytest.fail("Metadata and transport output difference detected") - elif metacmp_res == False: - pytest.fail("Metadata output difference detected") - elif pcmcmp_res != 0: - pytest.fail("Transport output difference detected") - else: - print("Comparison bit exact") - - else: - # Compare audio outputs - filecmp_res = cmp(dec_output_ref, dec_output_dut) - if filecmp_res == False: - cmp_result, reason = cmp_pcm( - dec_output_dut, - dec_output_ref, - output_mode, - int(out_fs * 1000), - get_mld=get_mld, - mld_lim=get_mld_lim, - abs_tol=abs_tol, - ) - if get_mld: - mld = re.search(MLD_PATTERN, reason).groups(1)[0] - record_property("MLD", mld) - - search_result = re.search(MAX_DIFF_PATTERN, reason) - max_diff = search_result.groups(1)[0] - record_property("MAXIMUM ABS DIFF", max_diff) - - # Report compare result - if cmp_result != 0: - pytest.fail(reason) - else: - if get_mld: - record_property("MLD", "0") - print("Comparison bit exact") - - -######################################################### -# -------------------- test function -------------------- -def ivas_enc( - encoder_frontend, - masa_channel_count, - masa_path, - ivas_br, - sampling_rate, - input_audio_path, - output_bitstream, - dtx: Optional[bool] = False, -): - # ------------ run cmd ------------ - options = ["-masa", f"{masa_channel_count}", f"{masa_path}"] - - # call encoder - encoder_frontend.run( - ivas_br, - sampling_rate, - input_audio_path, - output_bitstream, - dtx_mode=dtx, - add_option_list=options, - ) - - -def ivas_dec( - decoder_frontend, - output_mode, - sampling_rate, - input_bitstream, - output_path, -): - # -------- run cmd ------------ - - # call decoder - decoder_frontend.run( - output_mode, - sampling_rate, - input_bitstream, - output_path, - ) - - -def remove_output( - keep_files, - output_bitstream_ref, - output_bitstream_dut, - dec_output_ref, - dec_output_dut, - dec_met_output_ref: Optional[str] = None, - dec_met_output_dut: Optional[str] = None, -): - if not keep_files: - os.remove(output_bitstream_ref) - os.remove(output_bitstream_dut) - os.remove(dec_output_ref) - os.remove(dec_output_dut) - if dec_met_output_ref is not None: - os.remove(dec_met_output_ref) - if dec_met_output_dut is not None: - os.remove(dec_met_output_dut) -- GitLab