#!/usr/bin/env python3 import os import re import json import yaml import collections.abc import argparse root = "." ci_root = ".tekton" def save_json(filename, data): """Save a Json file.""" print("saving to", filename, json.dumps(data)) with open(filename, "w") as file: file.write(json.dumps(data)) def load_json(filename): """Load a json file.""" data = {} with open(filename, "r") as file: data = json.loads(file.read()) return data def load_yaml(filename): """Load a file.""" docs = [] with open(filename, "r") as file: try: data = yaml.safe_load_all(file) for doc in data: docs.append(doc) except yaml.constructor.ConstructorError: pass else: pass return docs def load_config(root_dir, ci_root_dir): """Load the configuration from the configuration directory.""" ret = { "files": [], "languages": ["markdown", "docker", "rust", "shell", "python", "yaml", "js", "make"], "markdown": {"extentions": ["md"]}, "docker": {"extentions": ["Dockerfile"]}, "rust": {"extentions": ["rs"]}, "shell": {"extentions": ["sh", "ksh"], "shellcheck-args": []}, "python": { "extentions": ["py"], "black-args": ["--check", "--diff"], "pylint-args": [], }, "yaml": { "extentions": ["yaml", "yml"], "detect": True, "ansible": {"enable": False}, "kube": {"enable": False}, }, "js": { "extentions": ["ts", "js"], "files": ["package.json", "yarn.lock", "schema.prisma"], }, "make": { "files": ["Makefile"], "checkmake-args": [] }, } if not os.path.isdir(ci_root_dir): return ret files = [ f for f in os.listdir(ci_root_dir) if os.path.isfile(os.path.join(ci_root_dir, f)) and re.match(".yaml$", f) ] if "auto-ci.yaml" in files: for doc in load_yaml(os.path.join(ci_root_dir, "auto-ci.yaml")): ret = {**ret, **doc} ret["files"] = files return ret def detect_files(config, root_dir): """Detect files based on their extention.""" ret = {} supported_extentions = [] supported_filename = [] for lang in config["languages"]: if "extentions" in config[lang]: supported_extentions.extend(config[lang]["extentions"]) if "files" in lang: supported_filename.extend(config[lang]["files"]) for directory, subdir_list, file_list in os.walk(root_dir): for filename in file_list: if filename in supported_filename: if not filename in ret: ret[filename] = [] ret[filename].append(os.path.join(directory, filename)) ext = filename.split(".")[len(filename.split(".")) - 1] if ext in supported_extentions: if not ext in ret: ret[ext] = [] ret[ext].append(os.path.join(directory, filename)) return ret def get_images_name(dockerfiles, root_dir): """Generate the images names for the detected Dockerfile.""" ret = [] for f in dockerfiles: dir = os.path.dirname(f) if dir == root_dir: ret.append( "$(params.artifactory-url)/$(params.project-path):$(params.image-version)" ) else: ret.append( "$(params.artifactory-url)/$(params.project-path)-{comp}:$(params.image-version)".format( comp=os.path.basename(dir) ) ) return ret def append_key(to, key, val): """Append a value in {to}[{key}], create the array if not existing.""" if not key in to: to[key] = [] to[key].append(val) def append_stage(to, key, val, files): """Append a value in {to}[{key}], create the array if not existing. if the key-file is found in the files add a custom suffix""" if not key in to: to[key] = [] # Not possible right now #if "{basename}.yaml".format(basename=val) in files: # to[key].append("{stage}-custom".format(stage=val)) #else: to[key].append(val) def set_js_stages(stages, config, files, root_dir): """Add the stages for javascript code.""" if ( "package.json" in files and os.path.join(root_dir, "package.json") in files["package.json"] ): if ( "yarn.lock" in files and os.path.join(root_dir, "yarn.lock") in files["yarn.lock"] ): append_stage(stages, "prepare", "prepare-yarn", config["files"]) else: append_stage(stages, "prepare", "prepare-npm", config["files"]) if ( "schema.prisma" in files and os.path.join(root_dir, "prisma", "schema.prisma") in files["schema.prisma"] ): append_stage(stages, "prepare", "prepare-prisma", config["files"]) defs = load_json(os.path.join(root_dir, "package.json")) if "scripts" in defs and "lint" in defs["scripts"]: append_stage(stages, "lint", "lint-javascript", config["files"]) if "scripts" in defs and "test" in defs["scripts"]: append_stage(stages, "test", "test-javascript", config["files"]) def set_yaml_stages(stages, config, files, root_dir): """Add the stages for yaml files.""" yamls = [] if "yaml" in files: yamls += files["yaml"] if "yml" in files: yamls += files["yml"] have_k8s = ( "kube" in config["yaml"] and "enable" in config["yaml"]["kube"] and config["yaml"]["kube"]["enable"] ) have_ansible = ( "ansible" in config["yaml"] and "enable" in config["yaml"]["ansible"] and config["yaml"]["ansible"]["enable"] ) should_detect = ( "detect" not in config["yaml"] or config["yaml"]["detect"] ) and not (have_k8s and have_ansible) if should_detect: for file in yamls: objs = load_yaml(file) for obj in objs: if obj == None: continue if isinstance(obj, collections.abc.Sequence): for item in obj: if "name" in item and ( "register" in item or "changed_when" in item or "loop_control" in item or "ansible.builtin.template" in item ): have_ansible = True elif "apiVersion" in obj: have_k8s = True append_stage(stages, "lint", "lint-yaml", config["files"]) if have_k8s: append_stage(stages, "lint", "lint-kube", config["files"]) if have_ansible: append_stage(stages, "lint", "lint-ansible", config["files"]) def get_results(config, files, root_dir): """Generate the stages based on the configuration and detected files.""" stages = { "global": [], "prepare": [], "lint": [], "build": [], "test": [], "publish": [], } args = { "shellcheck-args": ( config["shell"]["shellcheck-args"] if "shellcheck-args" in config["shell"] else [] ), "checkmake-args": ( config["make"]["checkmake-args"] if "checkmake-args" in config["make"] else [] ), "black-args": ( config["python"]["black-args"] if "black-args" in config["python"] else [] ), "pylint-args": ( config["python"]["pylint-args"] if "pylint-args" in config["python"] else [] ), } if "on-$(params.pipeline-type).yaml" in config["files"]: append_key(stages, "global", "$(params.pipeline-type)") return stages, args if "Dockerfile" in files: append_stage(stages, "lint", "lint-docker", config["files"]) append_stage(stages, "publish", "publish-docker", config["files"]) if "yaml" in files or "yml" in files: set_yaml_stages(stages, config, files, root_dir) if "sh" in files: append_stage(stages, "lint", "lint-shell", config["files"]) args["shellcheck-args"].extend(files["sh"]) if "sh" in files: append_stage(stages, "lint", "lint-shell", config["files"]) args["shellcheck-args"].extend(files["sh"]) if "Makefile" in files: append_stage(stages, "lint", "lint-make", config["files"]) args["checkmake-args"].extend(files["Makefile"]) if "rs" in files: append_stage(stages, "lint", "lint-clippy", config["files"]) if "py" in files: append_stage(stages, "lint", "lint-python", config["files"]) args["pylint-args"].extend(files["py"]) append_stage(stages, "lint", "lint-black", config["files"]) args["black-args"].extend(files["py"]) if len([t for t in files["py"] if re.match("/test_", t) != None]) > 0: append_stage(stages, "test", "test-python", config["files"]) if "ts" in files or "js" in files: set_js_stages(stages, config, files, root_dir) for stage in ["prepare", "lint", "build", "test", "publish"]: if "{stage}-custom.yaml" in config["files"]: stages[stage].append("{stage}-custom") # Unsupported by tekton... yet :P #if len(stages[stage])>0: # append_stage(stages, "global", "on-{stage}".format(stage = stage), config["files"]) return stages, args config = load_config(root, ci_root) files = detect_files(config, root) stages, args = get_results(config, files, root) save_json("$(results.stages-global.path)", stages["global"]) save_json("$(results.stages-prepare.path)", stages["prepare"]) save_json("$(results.stages-lint.path)", stages["lint"]) save_json("$(results.stages-build.path)", stages["build"]) save_json("$(results.stages-test.path)", stages["test"]) save_json("$(results.stages-publish.path)", stages["publish"]) save_json( "$(results.file-docker.path)", files["Dockerfile"] if "Dockerfile" in files else [] ) save_json( "$(results.images-name.path)", get_images_name(files["Dockerfile"] if "Dockerfile" in files else [], root), ) save_json("$(results.shellcheck-args.path)", args["shellcheck-args"]) save_json("$(results.checkmake-args.path)", args["checkmake-args"]) save_json("$(results.black-args.path)", args["black-args"]) save_json("$(results.pylint-args.path)", args["pylint-args"])