From 71650173f5e0e5b539687d0cda8c55f4dab99716 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 12:44:19 +0000 Subject: [PATCH 1/8] Merging commits from other test branch --- testing/asn_compile_targets.json | 7 ++ testing/asn_ignore.txt | 1 + testing/asn_ignore_lint.txt | 1 + testing/asn_lint_exceptions.json | 22 ++++ testing/asn_process.py | 183 +++++++++++++++++++++++++++++++ testing/lint_asn1.py | 21 +++- 6 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 testing/asn_compile_targets.json create mode 100644 testing/asn_ignore.txt create mode 100644 testing/asn_ignore_lint.txt create mode 100644 testing/asn_lint_exceptions.json create mode 100644 testing/asn_process.py diff --git a/testing/asn_compile_targets.json b/testing/asn_compile_targets.json new file mode 100644 index 00000000..1beceb2f --- /dev/null +++ b/testing/asn_compile_targets.json @@ -0,0 +1,7 @@ +[ + ["./33128/r15/TS33128Payloads.asn"], + ["./33128/r16/TS33128Payloads.asn"], + ["./33128/r16/TS33128IdentityAssociation.asn"], + ["./33128/r17/TS33128Payloads.asn"], + ["./33128/r17/TS33128IdentityAssociation.asn"] +] \ No newline at end of file diff --git a/testing/asn_ignore.txt b/testing/asn_ignore.txt new file mode 100644 index 00000000..70aa0c5b --- /dev/null +++ b/testing/asn_ignore.txt @@ -0,0 +1 @@ +33108 \ No newline at end of file diff --git a/testing/asn_ignore_lint.txt b/testing/asn_ignore_lint.txt new file mode 100644 index 00000000..03e7f54e --- /dev/null +++ b/testing/asn_ignore_lint.txt @@ -0,0 +1 @@ +dependencies \ No newline at end of file diff --git a/testing/asn_lint_exceptions.json b/testing/asn_lint_exceptions.json new file mode 100644 index 00000000..bbdf3615 --- /dev/null +++ b/testing/asn_lint_exceptions.json @@ -0,0 +1,22 @@ +{ + "33128/r15/TS33128Payloads.asn" : [ + "Enumerations for UDMServingSystemMethod start at 0, not 1", + "Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE" + ], + "33128/r16/TS33128Payloads.asn" : [ + "Enumerations for EstablishmentStatus start at 0, not 1", + "Enumerations for RequestIndication start at 0, not 1", + "Enumerations for UDMServingSystemMethod start at 0, not 1", + "Enumerations for MMSDirection start at 0, not 1", + "Enumerations for MMSReplyCharging start at 0, not 1", + "Enumerations for MMStatusExtension start at 0, not 1" + ], + "33128/r17/TS33128Payloads.asn" : [ + "Enumerations for EstablishmentStatus start at 0, not 1", + "Enumerations for RequestIndication start at 0, not 1", + "Enumerations for UDMServingSystemMethod start at 0, not 1", + "Enumerations for MMSDirection start at 0, not 1", + "Enumerations for MMSReplyCharging start at 0, not 1", + "Enumerations for MMStatusExtension start at 0, not 1" + ] +} \ No newline at end of file diff --git a/testing/asn_process.py b/testing/asn_process.py new file mode 100644 index 00000000..49deb161 --- /dev/null +++ b/testing/asn_process.py @@ -0,0 +1,183 @@ +import logging +import json +from pathlib import Path +from subprocess import run + +from pycrate_asn1c.asnproc import * + +import lint_asn1 + + +def syntaxCheckASN (fileList): + """ + Performs ASN syntax checking on a list of filenames (or pathlib Paths) + + :param fileList: List of filenames (str or Pathlib Path) + :returns: Dict with result, return code and message for each filename + + Calls the open-source asn1c compiler with the "syntax only" option. + As a result, asn1c must be available to run. + """ + results = {} + for file in fileList: + try: + p = run(['asn1c', '-E', str(file)], capture_output=True) + if (p.returncode != 0): + results[str(file)] = { + 'ok' : False, + 'code' : p.returncode, + 'message' : p.stderr.decode().splitlines()[0] + } + else: + results[str(file)] = { + 'ok' : True + } + except Exception as ex: + results[str(file)] = { + 'ok' : False, + 'code' : -1, + 'message' : f"{ex!r}" + } + return results + + + +def compileAllTargets (compileTargets): + """ + Attempts to compile a set of compile targets using the pycrate ASN1 tools + + :param compileTargets: list of compile targets, each of which is a list of filenames + :returns: A dict of outcome against the first filename of each compile target. Return code and message are included for failures. + + For each compile target (list of filenames) the first filename is assumed + to be the "primary" file. This doesn't have any relavance to the compilation, + but will be used as the identifier when reporting any compile errors. + The compilation is performed by the pycrate ASN compile functions; errors + are caught as exceptions and rendered into a list. + + Unfortunately, the pycrate compiler doesn't report line numbers. + The asn1c compiler does, but doesn't properly handle identifiers with the + same name in different modules; as this occurs multiple times in TS 33.108, + we can't use it. + """ + results = {} + for target in compileTargets: + firstTarget = target[0] + logging.debug(f"Compiling {firstTarget}") + try: + fileTexts = [] + fileNames = [] + GLOBAL.clear() + for filename in target: + with open(filename) as f: + fileTexts.append(f.read()) + fileNames.append(str(filename)) + logging.debug (f" Loading {filename}") + compile_text(fileTexts, filenames = fileNames) + results[str(firstTarget)] = { + 'ok' : True, + } + except Exception as ex: + results[str(firstTarget)] = { + 'ok' : False, + 'code' : -1, + 'message' : f"{ex!r}" + } + continue + return results + + + +def processResults (results, stageName): + """ + Counts the number of errors and writes out the output per filename + + :param results: List of filenames (str or Pathlib Path) + :param stageName: Name to decorate the output with + :returns: The number of files which had errors + """ + print("") + errorCount = sum([1 for r in results.values() if not r['ok']]) + logging.info(f"{errorCount} {stageName} errors encountered") + + print(f"{'-':-<60}") + print(f"{stageName} results:") + print(f"{'-':-<60}") + for filename, result in results.items(): + print(f" {filename:.<55}{'..OK' if result['ok'] else 'FAIL'}") + if not result['ok']: + if isinstance(result['message'], list): + for thing in result['message']: + print(f" {thing['message']}") + else: + print(f" {result['message']}") + + print(f"{'-':-<60}") + print(f"{stageName} errors: {errorCount}") + print(f"{'-':-<60}") + + return errorCount + + +if __name__ == '__main__': + logging.info('Searching for ASN.1 files') + fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) + logging.info(f'{len(fileList)} ASN.1 files found') + for file in fileList: + logging.debug(f' {file}') + + ignoreList = Path('testing/asn_ignore.txt').read_text().splitlines() + ignoredFiles = [] + for ignore in ignoreList: + logging.debug(f'Ignoring pattern {ignore}') + for file in fileList: + if ignore in str(file): + ignoredFiles.append(file) + logging.debug(f" Ignoring {str(file)} as contains {ignore}") + ignoredFiles = list(set(ignoredFiles)) + logging.info(f'{len(ignoredFiles)} files ignored') + for file in ignoredFiles: + logging.debug(f' {file}') + + fileList = [file for file in fileList if file not in ignoredFiles] + logging.info(f'{len(fileList)} files to process') + for file in fileList: + logging.debug(f' {file}') + + if len(fileList) == 0: + logging.warning ("No files specified") + exit(0) + + logging.info("Parsing ASN1 files") + parseResults = syntaxCheckASN(fileList) + if processResults(parseResults, "Parsing") > 0: + exit(-1) + + logging.info ("Getting compile targets") + compileTargets = json.loads(Path('testing/asn_compile_targets.json').read_text()) + logging.info (f"{len(compileTargets)} compile targets found") + + compileResults = compileAllTargets(compileTargets) + if processResults(compileResults, "Compiling") > 0: + exit(-1) + + logging.info ("Linting files") + ignoreLintingList = Path('testing/asn_ignore_lint.txt').read_text().splitlines() + ignoredFiles = [] + for ignore in ignoreLintingList: + logging.debug(f'Ignoring pattern {ignore} for linting') + for file in fileList: + if ignore in str(file): + ignoredFiles.append(file) + logging.debug(f" Ignoring {str(file)} for linting as contains {ignore}") + ignoredFiles = list(set(ignoredFiles)) + logging.info(f'{len(ignoredFiles)} files ignored for linting') + for file in ignoredFiles: + logging.debug(f' {file}') + fileList = [file for file in fileList if file not in ignoredFiles] + lintExceptions = json.loads(Path('testing/asn_lint_exceptions.json').read_text()) + lintResults = lint_asn1.lintASN1Files(fileList, lintExceptions) + if processResults(lintResults, "Linting") > 0: + exit(-1) + + exit(0) diff --git a/testing/lint_asn1.py b/testing/lint_asn1.py index 704d37c6..66d36095 100644 --- a/testing/lint_asn1.py +++ b/testing/lint_asn1.py @@ -98,7 +98,7 @@ def D41 (module, context): testDescription = "AUTOMATIC TAGS not used") def D42(module, context): errors = [] - if (module['tags'] == 'AUTOMATIC'): + if ('tags' in module) and (module['tags'] == 'AUTOMATIC'): appendFailure(errors, context, {"message" : "AUTOMATIC TAGS directive used"}) return errors @@ -167,9 +167,9 @@ def checkD45 (t, context): return errors -def lintASN1File (asnFile): - print (f"File: {asnFile}") +def lintASN1File (asnFile, exceptions): errors = [] + suppressed = [] context = {'file' : asnFile} try: logging.info ("Checking file {0}...".format(asnFile)) @@ -191,10 +191,17 @@ def lintASN1File (asnFile): except ParseError as ex: appendFailure(errors, context, { "message" : "ParseError: {0}".format(ex)}) logging.error("ParseError: {0}".format(ex)) - return errors + if len(exceptions) > 0: + suppressed = [error for error in errors if error['message'] in exceptions] + errors = [error for error in errors if error['message'] not in exceptions] + return { + 'ok' : len(errors) == 0, + 'message' : errors, + 'suppressed' : suppressed + } -def lintASN1Files (fileList): +def lintASN1Files (fileList, exceptions): if len(fileList) == 0: logging.warning ("No files specified") return {} @@ -202,7 +209,9 @@ def lintASN1Files (fileList): errorMap = {} logging.info("Checking files...") for f in fileList: - errorMap[str(f)] = lintASN1File(str(f)) + unixf = str(f).replace('\\', '/') + errorMap[str(f)] = lintASN1File(str(f), exceptions[unixf] if unixf in exceptions else []) + return errorMap -- GitLab From 07c2c82a98f199c17167da93d81c0fa52c9304ea Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 12:47:19 +0000 Subject: [PATCH 2/8] Updating jobs in yml --- .gitlab-ci.yml | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c98eee4..968e1256 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,37 +1,17 @@ -image: "mcanterb/forge-cicd:latest" - -before_script: - - python3 --version - stages: - - Syntax - - CompileAndLint - - Merge + - check -checkXSD: - stage: Syntax +process_asn: + image: "mcanterb/asn1test:latest" + stage: check + interruptible: true script: - - python3 testing/check_xsd.py + - python3 testing/asn_process.py -parseASN1: - stage: Syntax +process_xsd: + image: python:3.9-slim-bullseye + stage: check + interruptible: true script: - - python3 testing/parse_asn1.py - -compileASN1: - stage: CompileAndLint - script: - - python3 testing/compile_asn1.py - -lintASN1: - stage: CompileAndLint - script: - - python3 testing/lint_asn1.py - allow_failure: true - -MergeTest: - stage: Merge - script: - - python3 testing/merge_test.py - + - python3 --version -- GitLab From 3d6f5c788e07fbbd5155fbcd526d4c374a8dfe2f Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 12:49:18 +0000 Subject: [PATCH 3/8] Adding editor config --- .editorconfig | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..a3b74067 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +[**] +insert_final_newline = true + +[**.{asn1,asn,xsd,xml}] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true \ No newline at end of file -- GitLab From b075f62f898928967896861c121dc849a82b66e6 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 12:53:04 +0000 Subject: [PATCH 4/8] Removing uneeded py files --- testing/compile_asn1.py | 47 ------------------------------------ testing/lint_asn1.py | 2 -- testing/lintingexceptions.py | 7 ------ testing/parse_asn1.py | 40 ------------------------------ 4 files changed, 96 deletions(-) delete mode 100644 testing/compile_asn1.py delete mode 100644 testing/lintingexceptions.py delete mode 100644 testing/parse_asn1.py diff --git a/testing/compile_asn1.py b/testing/compile_asn1.py deleted file mode 100644 index 35fd5954..00000000 --- a/testing/compile_asn1.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging - -import asn1tools -from pathlib import Path - -from pprint import pprint - -ignoreReleases = {'33108' : [f'r{i}' for i in range(5, 17)], - '33128' : [] } - -def prepareFile(f): - with open(f) as fh: - s = fh.read() - s = s.replace("RELATIVE-OID", "OBJECT IDENTIFIER") # sigh - return s - -if __name__ == '__main__': - fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) - - ignoredFiles = [file for file in fileList if file.parts[1] in ignoreReleases[file.parts[0]]] - logging.info(f"Ignoring {len(ignoredFiles)} files") - logging.debug(ignoredFiles) - - fileList = [file for file in fileList if file not in ignoredFiles] - - if len(fileList) == 0: - logging.warning ("No files specified") - exit(0) - - print ("ASN.1 Compilation checks:") - print ("-----------------------------") - logging.info("Parsing files...") - errorCount = 0 - for f in fileList: - try: - s = prepareFile(str(f)) - asn1tools.compile_string(s) # this won't work for modules with IMPORTs - except asn1tools.ParseError as ex: - logging.info (f" {f}: Failed - {ex!r}") - print (f" {f}: Failed - {ex!r}") - errorCount += 1 - continue - print (f" {f}: OK") - print ("-----------------------------") - print (f"Compile errors: {errorCount}") - print ("-----------------------------") - exit(errorCount) diff --git a/testing/lint_asn1.py b/testing/lint_asn1.py index 66d36095..822c3e4b 100644 --- a/testing/lint_asn1.py +++ b/testing/lint_asn1.py @@ -8,8 +8,6 @@ import string from pprint import pprint import functools -import lintingexceptions - moduleLevelTests = [] typeLevelTests = [] diff --git a/testing/lintingexceptions.py b/testing/lintingexceptions.py deleted file mode 100644 index 423b5d39..00000000 --- a/testing/lintingexceptions.py +++ /dev/null @@ -1,7 +0,0 @@ -exceptedStrings = ["D.4.4: Enumerations for UDMServingSystemMethod start at 0, not 1", -"D.4.5: Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE", -"D.4.4: Enumerations for EstablishmentStatus start at 0, not 1", -"D.4.4: Enumerations for MMSDirection start at 0, not 1", -"D.4.4: Enumerations for MMSReplyCharging start at 0, not 1", -"D.4.4: Enumerations for MMStatusExtension start at 0, not 1", -"D.4.4: Enumerations for RequestIndication start at 0, not 1"] diff --git a/testing/parse_asn1.py b/testing/parse_asn1.py deleted file mode 100644 index 1602bfb9..00000000 --- a/testing/parse_asn1.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging - -from asn1tools import parse_files, ParseError -from pathlib import Path - -from pprint import pprint - -ignoreReleases = {'33108' : [f'r{i}' for i in range(5, 16)], - '33128' : [] } - -if __name__ == '__main__': - fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) - - ignoredFiles = [file for file in fileList if file.parts[1] in ignoreReleases[file.parts[0]]] - logging.info(f"Ignoring {len(ignoredFiles)} files") - logging.debug(ignoredFiles) - - fileList = [file for file in fileList if file not in ignoredFiles] - - if len(fileList) == 0: - logging.warning ("No files specified") - exit(0) - - print ("ASN.1 Parser checks:") - print ("-----------------------------") - logging.info("Parsing files...") - errorCount = 0 - for f in fileList: - try: - parse_files(str(f)) - except ParseError as ex: - logging.info (f" {f}: Failed - {ex!r}") - print (f" {f}: Failed - {ex!r}") - errorCount += 1 - continue - print (f" {f}: OK") - print ("-----------------------------") - print (f"Parse errors: {errorCount}") - print ("-----------------------------") - exit(errorCount) -- GitLab From 5151dad0c9e9ba745c7035c607829c32a8e404d9 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 13:25:53 +0000 Subject: [PATCH 5/8] Updating XSD processing --- 33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd | 4 ++-- testing/{check_xsd.py => xsd_process.py} | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) rename testing/{check_xsd.py => xsd_process.py} (90%) diff --git a/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd b/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd index 96f3da73..78f87ee9 100644 --- a/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd +++ b/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd @@ -13,7 +13,7 @@ - + @@ -23,7 +23,7 @@ - + diff --git a/testing/check_xsd.py b/testing/xsd_process.py similarity index 90% rename from testing/check_xsd.py rename to testing/xsd_process.py index bbe8a72a..e5588423 100644 --- a/testing/check_xsd.py +++ b/testing/xsd_process.py @@ -1,12 +1,8 @@ import logging logging.basicConfig(level=logging.INFO) -import glob -import sys from pathlib import Path -from pprint import pprint -from lxml import etree -from xml.etree.ElementTree import ParseError +from xmlschema.etree import etree_tostring from xmlschema import XMLSchema, XMLSchemaParseError @@ -22,7 +18,7 @@ def BuildSchemaDictonary (fileList): xs = XMLSchema(schemaFile, validation='skip') schemaLocations.append((xs.default_namespace, str(Path(schemaFile).resolve()))) logging.info(" [ {0} -> {1} ]".format(xs.default_namespace, schemaFile)) - except ParseError as ex: + except XMLSchemaParseError as ex: logging.warning (" [ {0} failed to parse: {1} ]".format(schemaFile, ex)) return schemaLocations @@ -47,13 +43,15 @@ def ValidateXSDFiles (fileList): logging.info("Schema validation:") for schemaFile in fileList: try: - schema = XMLSchema(schemaFile, locations = schemaLocations) + schema = XMLSchema(schemaFile, locations = schemaLocations, validation="lax") + print (schema.all_errors) logging.info(schemaFile + ": OK") - errors[schemaFile] = [] + errors[schemaFile] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in schema.all_errors] except XMLSchemaParseError as ex: logging.warning(schemaFile + ": Failed validation ({0})".format(ex.message)) if (ex.schema_url) and (ex.schema_url != ex.origin_url): logging.warning(" Error comes from {0}, suppressing".format(ex.schema_url)) + errors[schemaFile] = [] else: errors[schemaFile] = [ex] return errors -- GitLab From 96699188151671338fdd8aa593ed4ec30bd168bf Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 13:44:04 +0000 Subject: [PATCH 6/8] Updating XSD tests --- .gitignore | 3 +++ .gitlab-ci.yml | 4 ++-- testing/dockerfile | 7 ------- 3 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 testing/dockerfile diff --git a/.gitignore b/.gitignore index 3e376037..58d77c67 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Dockerfiles +dockerfile_* + # Editors .vscode/ .idea/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 968e1256..b4e3df33 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,9 +9,9 @@ process_asn: - python3 testing/asn_process.py process_xsd: - image: python:3.9-slim-bullseye + image: "mcanterb/xsdtest:latest" stage: check interruptible: true script: - - python3 --version + - python3 testing/xsd_process.py diff --git a/testing/dockerfile b/testing/dockerfile deleted file mode 100644 index d71a5bfb..00000000 --- a/testing/dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -# docker build -t mcanterb/forge-cicd -# docker push mcanterb/forge-cicd - -FROM python:3.8 -RUN apt update && apt-get install -y git -RUN git config --global user.name "forgeRobot" && git config --global user.email "forgeRobot@example.com" -RUN pip3 install -q asn1tools lxml xmlschema requests gitpython \ No newline at end of file -- GitLab From 9c892cb1af15662111539aa5b72884269dfe60e5 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 13:45:51 +0000 Subject: [PATCH 7/8] Removing deliberate XSD errors and reducing log output --- 33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd | 4 ++-- testing/xsd_process.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd b/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd index 78f87ee9..96f3da73 100644 --- a/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd +++ b/33128/r17/urn_3GPP_ns_li_3GPPStateTransfer.xsd @@ -13,7 +13,7 @@ - + @@ -23,7 +23,7 @@ - + diff --git a/testing/xsd_process.py b/testing/xsd_process.py index e5588423..6f109363 100644 --- a/testing/xsd_process.py +++ b/testing/xsd_process.py @@ -1,5 +1,4 @@ import logging -logging.basicConfig(level=logging.INFO) from pathlib import Path from xmlschema.etree import etree_tostring -- GitLab From d3ffabd0c31b743e0b08dc21c610ce717d37ecc3 Mon Sep 17 00:00:00 2001 From: mark Date: Tue, 22 Mar 2022 13:47:03 +0000 Subject: [PATCH 8/8] Removing errant print --- testing/xsd_process.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/xsd_process.py b/testing/xsd_process.py index 6f109363..a7dcd617 100644 --- a/testing/xsd_process.py +++ b/testing/xsd_process.py @@ -43,7 +43,6 @@ def ValidateXSDFiles (fileList): for schemaFile in fileList: try: schema = XMLSchema(schemaFile, locations = schemaLocations, validation="lax") - print (schema.all_errors) logging.info(schemaFile + ": OK") errors[schemaFile] = [f"{etree_tostring(e.elem, e.namespaces, ' ', 20)} - {e.message}" for e in schema.all_errors] except XMLSchemaParseError as ex: -- GitLab