diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c24254352b71a0b76474ad9981fac41d875aebcc..add3fe7e410e7e30b247844527046e46f19dc036 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,29 +4,34 @@ before_script: - python3 --version stages: + - Merge - Syntax - - Lint - - Compile + - CompileAndLint -parseASN1: - stage: Syntax +MergeTest: + stage: Merge script: - - python3 testing/parse_asn1.py + - python3 testing/merge_test.py checkXSD: stage: Syntax script: - python3 testing/check_xsd.py -lintASN1: - stage: Lint +parseASN1: + stage: Syntax script: - - python3 testing/lint_asn1.py - allow_failure: true + - python3 testing/parse_asn1.py compileASN1: - stage: Compile + stage: CompileAndLint script: - python3 testing/compile_asn1.py + +lintASN1: + stage: CompileAndLint + script: + - python3 testing/lint_asn1.py allow_failure: true + diff --git a/testing/check_asn1.py b/testing/check_asn1.py deleted file mode 100644 index ab868ff335d3118e15125cc3f4a2b460c82b4b36..0000000000000000000000000000000000000000 --- a/testing/check_asn1.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging - -from asn1tools import parse_files, compile_dict, ParseError, CompileError -from glob import glob -from pathlib import Path - -from pprint import pprint - - -def parseASN1File (asnFile): - try: - parse_files(asnFile) - except ParseError as ex: - return [ex] - return [] - - -def parseASN1Files (fileList): - if len(fileList) == 0: - logging.warning ("No files specified") - return {} - errors = {} - logging.info("Parsing files...") - for f in fileList: - ex = parseASN1File(f) - if ex: - logging.info (f" {f}: Failed - {ex!r}") - else: - logging.info (f" {f}: OK") - errors[f] = ex - return errors - - -def compileASN1Files (fileList): - logging.info("Compiling files...") - errors = [] - try: - d = parse_files(fileList) - for modulename, module in d.items(): - # Weird fix because the compiler doesn't like RELATIVE-OID as a type - # Not sure if the on-the-wire encoding would be affected or not - # but for most checking purposes this doesn't matter - module['types']["RELATIVE-OID"] = {'type' : 'OBJECT IDENTIFIER'} - c = compile_dict(d) - except CompileError as ex: - logging.info (f"Compiler error: {ex}") - errors.append(ex) - except ParseError as ex: - logging.info (f"Parse error: {ex}") - errors.append(ex) - logging.info ("Compiled OK") - return errors - - -def validateASN1Files (fileList): - parseErrors = parseASN1Files(fileList) -# if len(parseErrors > 0): -# logging.info ("Abandonding compile due to parse errors") - compileErrors = compileASN1Files(fileList) - return parseErrors, compileErrors - - -def validateAllASN1FilesInPath (path): - globPattern = str(Path(path)) + '/*.asn1' - logging.info("Searching: " + globPattern) - schemaGlob = glob(globPattern, recursive=True) - return validateASN1Files(schemaGlob) - - -if __name__ == '__main__': - parseErrors, compileErrors = validateAllASN1FilesInPath("./") - parseErrorCount = 0 - print ("ASN.1 Parser checks:") - print ("-----------------------------") - for filename, errors in parseErrors.items(): - if len(errors) > 0: - parseErrorCount += len(errors) - print (f"{filename}: {len(errors)} errors") - for error in errors: - print (" " + str(error)) - else: - print (f"{filename}: OK") - print ("-----------------------------") - print ("ASN.1 Compilation:") - print ("-----------------------------") - if len(compileErrors) > 0: - for error in compileErrors: - print (" " + str(error)) - else: - print ("Compilation OK") - print ("-----------------------------") - print (f"{parseErrorCount} parse errors, {len(compileErrors)} compile errors") - exit (parseErrorCount + len(compileErrors)) - diff --git a/testing/compile_asn1.py b/testing/compile_asn1.py index 6bd311bbbf83ce25280d985318b1a54d8d19df9d..35fd5954bd92e0b923980a8debab01595acf5da4 100644 --- a/testing/compile_asn1.py +++ b/testing/compile_asn1.py @@ -1 +1,47 @@ -print ("Not implemented yet") \ No newline at end of file +import logging + +import asn1tools +from pathlib import Path + +from pprint import pprint + +ignoreReleases = {'33108' : [f'r{i}' for i in range(5, 17)], + '33128' : [] } + +def prepareFile(f): + with open(f) as fh: + s = fh.read() + s = s.replace("RELATIVE-OID", "OBJECT IDENTIFIER") # sigh + return s + +if __name__ == '__main__': + fileList = list(Path(".").rglob("*.asn1")) + list(Path(".").rglob("*.asn")) + + ignoredFiles = [file for file in fileList if file.parts[1] in ignoreReleases[file.parts[0]]] + logging.info(f"Ignoring {len(ignoredFiles)} files") + logging.debug(ignoredFiles) + + fileList = [file for file in fileList if file not in ignoredFiles] + + if len(fileList) == 0: + logging.warning ("No files specified") + exit(0) + + print ("ASN.1 Compilation checks:") + print ("-----------------------------") + logging.info("Parsing files...") + errorCount = 0 + for f in fileList: + try: + s = prepareFile(str(f)) + asn1tools.compile_string(s) # this won't work for modules with IMPORTs + except asn1tools.ParseError as ex: + logging.info (f" {f}: Failed - {ex!r}") + print (f" {f}: Failed - {ex!r}") + errorCount += 1 + continue + print (f" {f}: OK") + print ("-----------------------------") + print (f"Compile errors: {errorCount}") + print ("-----------------------------") + exit(errorCount) diff --git a/testing/dockerfile b/testing/dockerfile index d907429836a7429329ca4931d4fc4d296d4bcb22..d71a5bfbb31a252d405d0c71cba867f4fbb23587 100644 --- a/testing/dockerfile +++ b/testing/dockerfile @@ -1,2 +1,7 @@ -FROM python:3.7 -RUN pip3 install -q asn1tools lxml xmlschema \ No newline at end of file +# docker build -t mcanterb/forge-cicd +# docker push mcanterb/forge-cicd + +FROM python:3.8 +RUN apt update && apt-get install -y git +RUN git config --global user.name "forgeRobot" && git config --global user.email "forgeRobot@example.com" +RUN pip3 install -q asn1tools lxml xmlschema requests gitpython \ No newline at end of file diff --git a/testing/lintingexceptions.py b/testing/lintingexceptions.py index 16bdd84676a52cce028871b27ba4246b02a41873..e3fdad86d949579e228f9732f02d31725b433550 100644 --- a/testing/lintingexceptions.py +++ b/testing/lintingexceptions.py @@ -1,2 +1,7 @@ -exceptedStrings = ["D.4.4: Enumerations for UDMServingSystemMethod start at 0, not 1", -"D.4.5: Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE"] \ No newline at end of file +exceptedStrings = ["D.4.4: Enumerations for UDMServingSystemMethod start at 0, not 1", +"D.4.5: Field 'aNNodeID' in GlobalRANNodeID is an anonymous CHOICE", +"D.4.4: Enumerations for EstablishmentStatus start at 0, not 1", +"D.4.4: Enumerations for MMSDirection start at 0, not 1", +"D.4.4: Enumerations for MMSReplyCharging start at 0, not 1", +"D.4.4: Enumerations for MMStatusExtension start at 0, not 1"] + diff --git a/testing/merge_test.py b/testing/merge_test.py new file mode 100644 index 0000000000000000000000000000000000000000..fc3e5802a674db7583e4e62444b7245bb0f33568 --- /dev/null +++ b/testing/merge_test.py @@ -0,0 +1,66 @@ +import os +import pprint +import requests +import json +import subprocess + +crCommitBranch = os.environ.get("CI_COMMIT_REF_NAME", "NOTFOUND") +apiUrl = os.environ.get("CI_API_V4_URL", "https://forge.3gpp.org/rep/api/v4") +projectId = os.environ.get("CI_PROJECT_ID", "13") + +def gapi (query): + url = f"{apiUrl}/projects/{projectId}/{query}" + r = requests.get(url) + return json.loads(r.text) + +def do (commandline): + #print (" Attempting: " + commandline) + completedProc = subprocess.run(commandline, capture_output=True, shell=True) + #print (" STDOUT > " + ("empty" if completedProc.stdout is None else completedProc.stdout.decode('utf-8'))) + #print (" STDERR > " + ("empty" if completedProc.stderr is None else completedProc.stderr.decode('utf-8'))) + #print (f" Completed with code {completedProc.returncode}") + return (completedProc.returncode == 0, completedProc.stdout.decode('utf-8')) + +print ("Searching for corresponding MR...") + +mrs = gapi(f"merge_requests?source_branch={crCommitBranch}") +if len(mrs) == 0: + print ("No MR found... aborting") + exit() + +if len(mrs) > 1: + print (f"{len(mrs)} MRs found, 1 expected - aborting") + exit(-1) + +mr = mrs[0] + +print (f"Found MR {mr['reference']} ({mr['title']})") +print (f"Target branch is {mr['target_branch']}") +print ("Searching for open MRs targeting same branch...") + +mrs = gapi(f"merge_requests?target_branch={mr['target_branch']}&state=opened") +mrs = [m for m in mrs if m['reference'] != mr['reference']] +print (f"{len(mrs)} MRs found") + +mergeConflicts = {} + +for mr in mrs: + source_branch = mr['source_branch'] + print (source_branch) + + try: + do(f"git fetch origin {source_branch}:{source_branch}") + success, errStr = do(f"git merge --no-commit {source_branch}") + if not success: + print ("Merge NOT OK") + mergeConflicts[source_branch] = errStr + else: + print ("Merge OK") + except Exception as ex: + mergeConflicts[source_branch] = str(ex) + raise + finally: + do("git merge --abort") + +print (f"Merge conflicts with following branches: {mergeConflicts}") +exit(len(mergeConflicts.keys())) \ No newline at end of file