From 8896c6db934637dacbb7b78084f25b09874a34fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Huss?= Date: Mon, 22 Apr 2024 16:25:59 +0200 Subject: [PATCH] fix --- share/gitea-tekton-org/auto-cd.tf | 3 - share/gitea-tekton-org/auto-ci-detector.py | 129 ++++++++++ share/gitea-tekton-org/auto-ci.tf | 99 +++++++- share/gitea-tekton-org/index.rhai | 56 +++++ .../tekton.dev_v1_Pipeline_auto-ci-tag.yaml | 220 ++++++++++++++++++ .../tekton.dev_v1_Task_auto-ci-detector.yaml | 184 --------------- 6 files changed, 500 insertions(+), 191 deletions(-) create mode 100644 share/gitea-tekton-org/auto-ci-detector.py create mode 100644 share/gitea-tekton-org/index.rhai create mode 100644 share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-tag.yaml delete mode 100644 share/gitea-tekton-org/tekton.dev_v1_Task_auto-ci-detector.yaml diff --git a/share/gitea-tekton-org/auto-cd.tf b/share/gitea-tekton-org/auto-cd.tf index 5ca6a0a..9b017ab 100644 --- a/share/gitea-tekton-org/auto-cd.tf +++ b/share/gitea-tekton-org/auto-cd.tf @@ -15,7 +15,6 @@ resource "kubectl_manifest" "cd-trigger-create" { yaml_body = <<-EOF apiVersion: triggers.tekton.dev/v1beta1 kind: Trigger - metadata: metadata: name: "${var.instance}-${var.component}-auto-create" namespace: "${var.namespace}" @@ -78,7 +77,6 @@ resource "kubectl_manifest" "cd-trigger-activate" { yaml_body = <<-EOF apiVersion: triggers.tekton.dev/v1beta1 kind: Trigger - metadata: metadata: name: "${var.instance}-${var.component}-auto-activate" namespace: "${var.namespace}" @@ -141,7 +139,6 @@ resource "kubectl_manifest" "ci-trigger-delete" { yaml_body = <<-EOF apiVersion: triggers.tekton.dev/v1beta1 kind: Trigger - metadata: metadata: name: "${var.instance}-${var.component}-auto-delete" namespace: "${var.namespace}" diff --git a/share/gitea-tekton-org/auto-ci-detector.py b/share/gitea-tekton-org/auto-ci-detector.py new file mode 100644 index 0000000..25da3ff --- /dev/null +++ b/share/gitea-tekton-org/auto-ci-detector.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +import os +import re +import json +import yaml +import collections.abc +import argparse + +root = '.' +def save_json(filename,data): + print('saving to', filename, json.dumps(data)) + with open(filename, 'w') as file: + file.write(json.dumps(data)) +# detect files based on their extention +def detect_files(root_dir): + ret = {} + supported_extention = ['ts','js', 'py', 'yaml','yml', 'sh', 'rs', 'Dockerfile'] + supported_filename = ['package.json', 'yarn.lock', 'schema.prisma'] + for directory, subdir_list, file_list in os.walk(root_dir): + for filename in file_list: + if filename in supported_filename: + if not filename in ret: + ret[filename] = [] + ret[filename].append(os.path.join(directory,filename)) + ext = filename.split(".")[len(filename.split("."))-1] + if ext in supported_extention: + if not ext in ret: + ret[ext] = [] + ret[ext].append(os.path.join(directory,filename)) + return ret +def load_yaml(filename): + docs=[] + with open(filename, 'r') as file: + try: + data = yaml.safe_load_all(file) + for doc in data: + docs.append(doc) + except yaml.constructor.ConstructorError: + pass + else: + pass + return docs +def load_json(filename): + data={} + with open(filename, 'r') as file: + data = json.loads(file.read()) + return data + +def append_key(to,key,val): + if not key in to: + to[key] = [] + to[key].append(val) +def set_js_stages(stages,files,root_dir): + if 'package.json' in files and os.path.join(root_dir,'package.json') in files['package.json']: + if 'yarn.lock' in files and os.path.join(root_dir,'yarn.lock') in files['yarn.lock']: + append_key(stages,'prepare','prepare-yarn') + else: + append_key(stages,'prepare','prepare-npm') + if 'schema.prisma' in files and os.path.join(root_dir,'prisma','schema.prisma') in files['schema.prisma']: + append_key(stages,'prepare','prepare-prisma') + defs = load_json(os.path.join(root_dir,'package.json')) + if 'scripts' in defs and 'lint' in defs['scripts']: + append_key(stages,'lint','lint-javascript') + if 'scripts' in defs and 'test' in defs['scripts']: + append_key(stages,'test','test-javascript') +def set_yaml_stages(stages,files,root_dir): + have_k8s = False + have_ansible = False + yamls = [] + if 'yaml' in files: + yamls += files['yaml'] + if 'yml' in files: + yamls += files['yml'] + for file in yamls: + objs = load_yaml(file) + for obj in objs: + if obj == None: + continue + if isinstance(obj, collections.abc.Sequence): + for item in obj: + if 'name' in item and ('register' in item or 'changed_when' in item or 'loop_control' in item or 'ansible.builtin.template' in item): + have_ansible = True + elif 'apiVersion' in obj: + have_k8s = True + append_key(stages,'lint','lint-yaml') + if have_k8s: + append_key(stages,'lint','lint-kube') + if have_ansible: + append_key(stages,'lint','lint-ansible') +def get_images_name(dockerfiles,root_dir): + ret = [] + for f in dockerfiles: + dir = os.path.dirname(f) + ret.append("$(params.artifactory-url)/$(params.project-path):$(params.image-version)") + print('get_image_name', dir, root_dir) + return ret +def get_stages(files,root_dir): + ret = {} + ret['prepare'] = [] + ret['lint'] = [] + ret['test'] = [] + ret['publish'] = [] + if 'Dockerfile' in files: + append_key(ret,'lint','lint-docker') + append_key(ret,'publish','publish-docker') + if 'yaml' in files or 'yml' in files: + set_yaml_stages(ret,files,root_dir) + if 'sh' in files: + append_key(ret,'lint', 'lint-shell') + if 'rs' in files: + append_key(ret,'lint', 'lint-clippy') + if 'py' in files: + append_key(ret,'lint','lint-python') + append_key(ret,'lint','lint-black') + if len([t for t in files['py'] if re.match('/test_',t) != None]) > 0: + append_key(ret,'test','test-python') + if 'ts' in files or 'js' in files: + set_js_stages(ret,files,root_dir) + return ret +files = detect_files(root) +stages = get_stages(files,root) +save_json("$(results.stages-prepare.path)", stages['prepare']) +save_json("$(results.stages-lint.path)", stages['lint']) +save_json("$(results.stages-test.path)", stages['test']) +save_json("$(results.stages-publish.path)", stages['publish']) +save_json("$(results.file-shell.path)", files['sh'] if 'sh' in files else []) +save_json("$(results.file-python.path)", files['py'] if 'py' in files else []) +save_json("$(results.file-docker.path)", files['Dockerfile'] if 'Dockerfile' in files else []) +save_json("$(results.images-name.path)", get_images_name(files['Dockerfile'] if 'Dockerfile' in files else [],root)) \ No newline at end of file diff --git a/share/gitea-tekton-org/auto-ci.tf b/share/gitea-tekton-org/auto-ci.tf index f378e47..a26c5ae 100644 --- a/share/gitea-tekton-org/auto-ci.tf +++ b/share/gitea-tekton-org/auto-ci.tf @@ -3,10 +3,77 @@ locals { "type" = "branch-push" }) tag-labels = merge(local.common-labels, { - "type" = "tag-new" + "type" = "tag-push" }) } +resource "kubectl_manifest" "auto-ci-detector" { + yaml_body = <<-EOF + apiVersion: tekton.dev/v1 + kind: Task + metadata: + name: auto-ci-detector + namespace: "${var.namespace}" + labels: ${jsonencode(local.push-labels)} + spec: + results: + - name: stages-prepare + description: list of prepare actions + type: array + - name: stages-lint + description: list of lint actions + type: array + - name: stages-test + description: list of test actions + type: array + - name: stages-publish + description: list of publish actions + type: array + - name: file-shell + description: list of shell files if any + type: array + - name: file-python + description: list of python files if any + type: array + - name: file-docker + description: list of Dockerfiles if any + type: array + - name: images-name + description: list of Dockerfiles image-name + type: array + params: + - name: toolbox-image + default: sebt3/basic-toolbox-image:1.30.0 + description: The name of the toolbox image + type: string + - name: artifactory-url + default: docker.io + description: The url of the current artifactory + type: string + - name: project-name + description: The name of the current project + type: string + - name: project-path + description: The path of the current project + type: string + - name: image-version + type: string + steps: + - name: detect-stages + image: $(params.toolbox-image) + workingDir: $(workspaces.source.path) + script: ${file("${path.module}/auto-ci-detector.py")} + + workspaces: + - name: source + mountPath: /data + EOF +} + + + + + resource "kubectl_manifest" "ci-trigger-push" { count = var.autoCI?1:0 yaml_body = <<-EOF @@ -160,10 +227,20 @@ resource "kubectl_manifest" "ci-trigger-tag" { pipelineRef: name: "auto-ci-tag" params: - - name: git-revision - value: $(tt.params.gitrevision) + - name: artifactory-url + value: $(tt.params.artifactory-url) + - name: project-name + value: $(tt.params.project-name) + - name: project-path + value: $(tt.params.project-path) - name: git-url - value: $(tt.params.gitrepositoryurl) + value: $(tt.params.git-repository-url) + - name: git-revision + value: $(tt.params.git-revision) + - name: git-default-branch + value: $(tt.params.git-default-branch) + - name: branch-name + value: $(tt.params.branch-name) workspaces: - name: source volumeClaimTemplate: @@ -176,6 +253,18 @@ resource "kubectl_manifest" "ci-trigger-tag" { resources: requests: storage: 1Gi + - name: dockerconfig + secret: + secretName: gitea-docker + items: + - key: ".dockerconfigjson" + path: "config.json" + - name: sslcertdir + secret: + secretName: gitea + items: + - key: "ca.crt" + path: "ca.crt" - name: ssh secret: secretName: ssh-credentials @@ -188,3 +277,5 @@ resource "kubectl_manifest" "ci-trigger-tag" { path: "id_rsa.pub" EOF } + + diff --git a/share/gitea-tekton-org/index.rhai b/share/gitea-tekton-org/index.rhai new file mode 100644 index 0000000..504e022 --- /dev/null +++ b/share/gitea-tekton-org/index.rhai @@ -0,0 +1,56 @@ +const SRC=src; +const DEST=dest; +const duplicates=["auto-ci-detector.py"]; +const sources=[ + #{url: "https://github.com/tektoncd/catalog/raw/main/task/python-coverage/0.1/python-coverage.yaml", name: "python-coverage.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/black/0.2/black.yaml", name: "black.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/buildkit-daemonless/0.1/buildkit-daemonless.yaml", name: "buildkit-daemonless.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/buildpacks-phases/0.2/buildpacks-phases.yaml", name: "buildpacks-phases.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/buildpacks/0.6/buildpacks.yaml", name: "buildpacks.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/check-make/0.1/check-make.yaml", name: "check-make.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/docker-build/0.1/docker-build.yaml", name: "docker-build.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/generate-build-id/0.1/generate-build-id.yaml", name: "generate-build-id.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/git-clone/0.9/git-clone.yaml", name: "git-clone.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/gitea-set-status/0.1/gitea-set-status.yaml", name: "gitea-set-status.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/hadolint/0.1/hadolint.yaml", name: "hadolint.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/hugo/0.1/hugo.yaml", name: "hugo.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/kaniko/0.6/kaniko.yaml", name: "kaniko.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/kube-linter/0.1/kube-linter.yaml", name: "kube-linter.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/kubeval/0.1/kubeval.yaml", name: "kubeval.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/markdown-lint/0.1/markdown-lint.yaml", name: "markdown-lint.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/npm/0.1/npm.yaml", name: "npm.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/pylint/0.3/pylint.yaml", name: "pylint.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/curl/0.1/curl.yaml", name: "curl.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/buildah/0.7/buildah.yaml", name: "buildah.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/pytest/0.2/pytest.yaml", name: "pytest.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/python-coverage/0.1/python-coverage.yaml", name: "python-coverage.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/shellcheck/0.1/shellcheck.yaml", name: "shellcheck.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/sonarqube-scanner/0.4/sonarqube-scanner.yaml", name: "sonarqube-scanner.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/ts-lint/0.1/ts-lint.yaml", name: "ts-lint.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/write-file/0.1/write-file.yaml", name: "write-file.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/yaml-lint/0.1/yaml-lint.yaml", name: "yaml-lint.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/skopeo-copy/0.3/skopeo-copy.yaml", name: "skopeo-copy.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/prometheus-gate/0.1/prometheus-gate.yaml", name: "prometheus-gate.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/ansible-builder/0.1/ansible-builder.yaml", name: "ansible-builder.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/ansible-runner/0.2/ansible-runner.yaml", name: "ansible-runner.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/task/yq/0.4/yq.yaml", name: "yq.yaml"}, + #{url: "https://github.com/tektoncd/catalog/raw/main/pipeline/buildpacks/0.2/buildpacks.yaml", name: "buildpacks-pipeline.yaml"} +]; + +// https://github.com/tektoncd/catalog/raw/main/task/trivy-scanner/0.2/trivy-scanner.yaml + +fn pre_pack() { + for file in global::sources { + //shell(`curl -sL ${file.url} > ${global::SRC}/${file.name}`); + } +} +fn post_pack() { + for file in global::duplicates { + shell(`cp ${global::SRC}/${file} ${global::DEST}/${file}`); + } +} +fn post_template() { + for file in global::duplicates { + shell(`cp ${global::SRC}/${file} ${global::DEST}/${file}`); + } +} diff --git a/share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-tag.yaml b/share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-tag.yaml new file mode 100644 index 0000000..293fc7c --- /dev/null +++ b/share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-tag.yaml @@ -0,0 +1,220 @@ +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: auto-ci-tag +spec: + workspaces: + - name: source + - name: dockerconfig + - name: ssh + - name: sslcertdir + params: + - name: artifactory-url + default: docker.io + description: The url of the current artifactory + type: string + - name: project-name + description: The name of the current project + type: string + - name: project-path + description: The path of the current project + type: string + - name: git-url + type: string + - name: git-revision + type: string + - name: tag-name + type: string + finally: + - name: inform-gitea-success + when: + - cel: "'$(tasks.status)' in ['Succeeded','Completed']" + params: + - name: REPO_FULL_NAME + value: $(params.project-path) + - name: SHA + value: $(params.git-revision) + - name: TARGET_URL + value: "https://okd.media.rennes.home/k8s/ns/$(context.pipelineRun.namespace)/tekton.dev~v1~PipelineRun/$(context.pipelineRun.name)" + - name: DESCRIPTION + value: "auto-ci-push" + - name: STATE + value: "success" + taskRef: + name: gitea-set-status + - name: inform-gitea-warning + when: + - cel: "'$(tasks.status)' == 'None'" + params: + - name: REPO_FULL_NAME + value: $(params.project-path) + - name: SHA + value: $(params.git-revision) + - name: TARGET_URL + value: "https://okd.media.rennes.home/k8s/ns/$(context.pipelineRun.namespace)/tekton.dev~v1~PipelineRun/$(context.pipelineRun.name)" + - name: DESCRIPTION + value: "auto-ci-push" + - name: STATE + value: "warning" + taskRef: + name: gitea-set-status + - name: inform-gitea-error + when: + - cel: "'$(tasks.status)' == 'Failed'" + params: + - name: REPO_FULL_NAME + value: $(params.project-path) + - name: SHA + value: $(params.git-revision) + - name: TARGET_URL + value: "https://okd.media.rennes.home/k8s/ns/$(context.pipelineRun.namespace)/tekton.dev~v1~PipelineRun/$(context.pipelineRun.name)" + - name: DESCRIPTION + value: "auto-ci-push" + - name: STATE + value: "error" + taskRef: + name: gitea-set-status + tasks: + - name: inform-gitea-start + onError: continue + params: + - name: REPO_FULL_NAME + value: $(params.project-path) + - name: SHA + value: $(params.git-revision) + - name: TARGET_URL + value: "https://okd.media.rennes.home/k8s/ns/$(context.pipelineRun.namespace)/tekton.dev~v1~PipelineRun/$(context.pipelineRun.name)" + - name: DESCRIPTION + value: "auto-ci-push" + - name: STATE + value: "pending" + taskRef: + name: gitea-set-status + - name: git-clone + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.git-revision) + - name: depth + value: 0 + taskRef: + name: git-clone + workspaces: + - name: output + workspace: source + - name: ssh-directory + workspace: ssh + - name: git-version + runAfter: [git-clone] + params: + - name: branch + value: $(params.branch-name) + taskRef: + name: git-version + workspaces: + - name: source + - name: detect-stages + runAfter: [git-version] + params: + - name: artifactory-url + value: $(params.artifactory-url) + - name: project-name + value: $(params.project-name) + - name: project-path + value: $(params.project-path) + - name: image-version + value: "$(params.branch-name)-$(tasks.git-version.results.packageVersion)" + taskRef: + name: auto-ci-detector + workspaces: + - name: source + - name: lint-shell + runAfter: [detect-stages] + when: + - input: "lint-shell" + operator: in + values: ["$(tasks.detect-stages.results.stages-lint[*])"] + params: + - name: args + value: $(tasks.detect-stages.results.file-shell) + taskRef: + name: shellcheck + workspaces: + - name: shared-workspace + workspace: source + - name: lint-docker + runAfter: [detect-stages] + when: + - input: "lint-docker" + operator: in + values: ["$(tasks.detect-stages.results.stages-lint[*])"] + taskRef: + name: hadolint + matrix: + params: + - name: dockerfile-path + value: $(tasks.detect-stages.results.file-docker) + workspaces: + - name: source + - name: lint-yaml + runAfter: [detect-stages] + when: + - input: "lint-yaml" + operator: in + values: ["$(tasks.detect-stages.results.stages-lint[*])"] + params: + - name: args + value: ["."] + taskRef: + name: yaml-lint + workspaces: + - name: shared-workspace + workspace: source + - name: lint-black + runAfter: [detect-stages] + when: + - input: "lint-black" + operator: in + values: ["$(tasks.detect-stages.results.stages-lint[*])"] + params: + - name: args + value: ["--check", "--diff", "$(tasks.detect-stages.results.file-python[*])"] + taskRef: + name: black + workspaces: + - name: shared-workspace + workspace: source + - name: lint-python + runAfter: [detect-stages] + when: + - input: "lint-python" + operator: in + values: ["$(tasks.detect-stages.results.stages-lint[*])"] + params: + - name: args + value: ["$(tasks.detect-stages.results.file-python[*])"] + taskRef: + name: pylint + workspaces: + - name: shared-workspace + workspace: source + - name: publish-docker + runAfter: ["lint-docker"] + when: + - input: "publish-docker" + operator: in + values: ["$(tasks.detect-stages.results.stages-publish[*])"] + - cel: "'$(params.branch-name)' == '$(params.git-default-branch)'" + taskRef: + name: buildah + matrix: + params: + - name: DOCKERFILE + value: $(tasks.detect-stages.results.file-docker) + - name: IMAGE + value: $(tasks.detect-stages.results.images-name) + workspaces: + - name: source + - name: sslcertdir + - name: dockerconfig \ No newline at end of file diff --git a/share/gitea-tekton-org/tekton.dev_v1_Task_auto-ci-detector.yaml b/share/gitea-tekton-org/tekton.dev_v1_Task_auto-ci-detector.yaml deleted file mode 100644 index 5b84783..0000000 --- a/share/gitea-tekton-org/tekton.dev_v1_Task_auto-ci-detector.yaml +++ /dev/null @@ -1,184 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: Task -metadata: - name: auto-ci-detector -spec: - results: - - name: stages-prepare - description: list of prepare actions - type: array - - name: stages-lint - description: list of lint actions - type: array - - name: stages-test - description: list of test actions - type: array - - name: stages-publish - description: list of publish actions - type: array - - name: file-shell - description: list of shell files if any - type: array - - name: file-python - description: list of python files if any - type: array - - name: file-docker - description: list of Dockerfiles if any - type: array - - name: images-name - description: list of Dockerfiles image-name - type: array - params: - - name: toolbox-image - default: sebt3/basic-toolbox-image:1.30.0 - description: The name of the toolbox image - type: string - - name: artifactory-url - default: docker.io - description: The url of the current artifactory - type: string - - name: project-name - description: The name of the current project - type: string - - name: project-path - description: The path of the current project - type: string - - name: image-version - type: string - steps: - - name: detect-stages - image: $(params.toolbox-image) - workingDir: $(workspaces.source.path) - script: | - #!/usr/bin/env python3 - import os - import re - import json - import yaml - import collections.abc - import argparse - - root = '.' - def save_json(filename,data): - print('saving to', filename, json.dumps(data)) - with open(filename, 'w') as file: - file.write(json.dumps(data)) - # detect files based on their extention - def detect_files(root_dir): - ret = {} - supported_extention = ['ts','js', 'py', 'yaml','yml', 'sh', 'rs', 'Dockerfile'] - supported_filename = ['package.json', 'yarn.lock', 'schema.prisma'] - for directory, subdir_list, file_list in os.walk(root_dir): - for filename in file_list: - if filename in supported_filename: - if not filename in ret: - ret[filename] = [] - ret[filename].append(os.path.join(directory,filename)) - ext = filename.split(".")[len(filename.split("."))-1] - if ext in supported_extention: - if not ext in ret: - ret[ext] = [] - ret[ext].append(os.path.join(directory,filename)) - return ret - def load_yaml(filename): - docs=[] - with open(filename, 'r') as file: - try: - data = yaml.safe_load_all(file) - for doc in data: - docs.append(doc) - except yaml.constructor.ConstructorError: - pass - else: - pass - return docs - def load_json(filename): - data={} - with open(filename, 'r') as file: - data = json.loads(file.read()) - return data - - def append_key(to,key,val): - if not key in to: - to[key] = [] - to[key].append(val) - def set_js_stages(stages,files,root_dir): - if 'package.json' in files and os.path.join(root_dir,'package.json') in files['package.json']: - if 'yarn.lock' in files and os.path.join(root_dir,'yarn.lock') in files['yarn.lock']: - append_key(stages,'prepare','prepare-yarn') - else: - append_key(stages,'prepare','prepare-npm') - if 'schema.prisma' in files and os.path.join(root_dir,'prisma','schema.prisma') in files['schema.prisma']: - append_key(stages,'prepare','prepare-prisma') - defs = load_json(os.path.join(root_dir,'package.json')) - if 'scripts' in defs and 'lint' in defs['scripts']: - append_key(stages,'lint','lint-javascript') - if 'scripts' in defs and 'test' in defs['scripts']: - append_key(stages,'test','test-javascript') - def set_yaml_stages(stages,files,root_dir): - have_k8s = False - have_ansible = False - yamls = [] - if 'yaml' in files: - yamls += files['yaml'] - if 'yml' in files: - yamls += files['yml'] - for file in yamls: - objs = load_yaml(file) - for obj in objs: - if obj == None: - continue - if isinstance(obj, collections.abc.Sequence): - for item in obj: - if 'name' in item and ('register' in item or 'changed_when' in item or 'loop_control' in item or 'ansible.builtin.template' in item): - have_ansible = True - elif 'apiVersion' in obj: - have_k8s = True - append_key(stages,'lint','lint-yaml') - if have_k8s: - append_key(stages,'lint','lint-kube') - if have_ansible: - append_key(stages,'lint','lint-ansible') - def get_images_name(dockerfiles,root_dir): - ret = [] - for f in dockerfiles: - dir = os.path.dirname(f) - ret.append("$(params.artifactory-url)/$(params.project-path):$(params.image-version)") - print('get_image_name', dir, root_dir) - return ret - def get_stages(files,root_dir): - ret = {} - ret['prepare'] = [] - ret['lint'] = [] - ret['test'] = [] - ret['publish'] = [] - if 'Dockerfile' in files: - append_key(ret,'lint','lint-docker') - append_key(ret,'publish','publish-docker') - if 'yaml' in files or 'yml' in files: - set_yaml_stages(ret,files,root_dir) - if 'sh' in files: - append_key(ret,'lint', 'lint-shell') - if 'rs' in files: - append_key(ret,'lint', 'lint-clippy') - if 'py' in files: - append_key(ret,'lint','lint-python') - append_key(ret,'lint','lint-black') - if len([t for t in files['py'] if re.match('/test_',t) != None]) > 0: - append_key(ret,'test','test-python') - if 'ts' in files or 'js' in files: - set_js_stages(ret,files,root_dir) - return ret - files = detect_files(root) - stages = get_stages(files,root) - save_json("$(results.stages-prepare.path)", stages['prepare']) - save_json("$(results.stages-lint.path)", stages['lint']) - save_json("$(results.stages-test.path)", stages['test']) - save_json("$(results.stages-publish.path)", stages['publish']) - save_json("$(results.file-shell.path)", files['sh'] if 'sh' in files else []) - save_json("$(results.file-python.path)", files['py'] if 'py' in files else []) - save_json("$(results.file-docker.path)", files['Dockerfile'] if 'Dockerfile' in files else []) - save_json("$(results.images-name.path)", get_images_name(files['Dockerfile'] if 'Dockerfile' in files else [],root)) - workspaces: - - name: source - mountPath: /data \ No newline at end of file