From c21c7c5b732a1b9a3b61ac649e315e9e34f4747e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Huss?= Date: Wed, 24 Apr 2024 17:53:21 +0200 Subject: [PATCH] fix --- share/gitea-tekton-org/auto-ci-detector.py | 18 +- share/gitea-tekton-org/auto-ci.tf | 3 + share/gitea-tekton-org/index.rhai | 1 - .../tekton.dev_v1_Pipeline_auto-ci-push.yaml | 14 + .../tekton.dev_v1beta1_Task_ts-lint.yaml | 31 -- .../v1_ConfigMap_auto-cd-templates.yaml | 460 +++++++++++++++--- 6 files changed, 435 insertions(+), 92 deletions(-) delete mode 100644 share/gitea-tekton-org/tekton.dev_v1beta1_Task_ts-lint.yaml diff --git a/share/gitea-tekton-org/auto-ci-detector.py b/share/gitea-tekton-org/auto-ci-detector.py index 72bc0c8..73aa251 100644 --- a/share/gitea-tekton-org/auto-ci-detector.py +++ b/share/gitea-tekton-org/auto-ci-detector.py @@ -44,7 +44,7 @@ def load_config(root_dir, ci_root_dir): """Load the configuration from the configuration directory.""" ret = { "files": [], - "languages": ["markdown", "docker", "rust", "shell", "python", "yaml", "js"], + "languages": ["markdown", "docker", "rust", "shell", "python", "yaml", "js", "make"], "markdown": {"extentions": ["md"]}, "docker": {"extentions": ["Dockerfile"]}, "rust": {"extentions": ["rs"]}, @@ -64,6 +64,10 @@ def load_config(root_dir, ci_root_dir): "extentions": ["ts", "js"], "files": ["package.json", "yarn.lock", "schema.prisma"], }, + "make": { + "files": ["Makefile"], + "checkmake-args": [] + }, } if not os.path.isdir(ci_root_dir): return ret @@ -226,6 +230,11 @@ def get_results(config, files, root_dir): if "shellcheck-args" in config["shell"] else [] ), + "checkmake-args": ( + config["make"]["checkmake-args"] + if "checkmake-args" in config["make"] + else [] + ), "black-args": ( config["python"]["black-args"] if "black-args" in config["python"] else [] ), @@ -245,6 +254,12 @@ def get_results(config, files, root_dir): if "sh" in files: append_stage(stages, "lint", "lint-shell", config["files"]) args["shellcheck-args"].extend(files["sh"]) + if "sh" in files: + append_stage(stages, "lint", "lint-shell", config["files"]) + args["shellcheck-args"].extend(files["sh"]) + if "Makefile" in files: + append_stage(stages, "lint", "lint-make", config["files"]) + args["checkmake-args"].extend(files["Makefile"]) if "rs" in files: append_stage(stages, "lint", "lint-clippy", config["files"]) if "py" in files: @@ -282,5 +297,6 @@ save_json( get_images_name(files["Dockerfile"] if "Dockerfile" in files else [], root), ) save_json("$(results.shellcheck-args.path)", args["shellcheck-args"]) +save_json("$(results.checkmake-args.path)", args["checkmake-args"]) save_json("$(results.black-args.path)", args["black-args"]) save_json("$(results.pylint-args.path)", args["pylint-args"]) diff --git a/share/gitea-tekton-org/auto-ci.tf b/share/gitea-tekton-org/auto-ci.tf index c338ee8..83070e7 100644 --- a/share/gitea-tekton-org/auto-ci.tf +++ b/share/gitea-tekton-org/auto-ci.tf @@ -44,6 +44,9 @@ resource "kubectl_manifest" "auto-ci-detector" { - name: shellcheck-args description: Arguments for shellcheck type: array + - name: checkmake-args + description: Arguments for checkmake + type: array - name: black-args description: Arguments for black type: array diff --git a/share/gitea-tekton-org/index.rhai b/share/gitea-tekton-org/index.rhai index f5ff19b..3ee9687 100644 --- a/share/gitea-tekton-org/index.rhai +++ b/share/gitea-tekton-org/index.rhai @@ -21,7 +21,6 @@ const sources=[ #{url: "https://github.com/tektoncd/catalog/raw/main/task/python-coverage/0.1/python-coverage.yaml", name: "python-coverage.yaml"}, #{url: "https://github.com/tektoncd/catalog/raw/main/task/shellcheck/0.1/shellcheck.yaml", name: "shellcheck.yaml"}, #{url: "https://github.com/tektoncd/catalog/raw/main/task/sonarqube-scanner/0.4/sonarqube-scanner.yaml", name: "sonarqube-scanner.yaml"}, - #{url: "https://github.com/tektoncd/catalog/raw/main/task/ts-lint/0.1/ts-lint.yaml", name: "ts-lint.yaml"}, #{url: "https://github.com/tektoncd/catalog/raw/main/task/yaml-lint/0.1/yaml-lint.yaml", name: "yaml-lint.yaml"}, ]; fn pre_pack() { diff --git a/share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-push.yaml b/share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-push.yaml index 38ff5cb..37b19ed 100644 --- a/share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-push.yaml +++ b/share/gitea-tekton-org/tekton.dev_v1_Pipeline_auto-ci-push.yaml @@ -214,6 +214,20 @@ spec: workspaces: - name: shared-workspace workspace: source + - name: lint-python + onError: continue + when: + - input: "lint-python" + operator: in + values: ["$(tasks.detect-stages.results.stages-lint[*])"] + params: + - name: args + value: $(tasks.detect-stages.results.pylint-args) + taskRef: + name: pylint + workspaces: + - name: shared-workspace + workspace: source - name: publish-docker runAfter: ["lint-shell","lint-docker","lint-yaml","lint-black","lint-python"] when: diff --git a/share/gitea-tekton-org/tekton.dev_v1beta1_Task_ts-lint.yaml b/share/gitea-tekton-org/tekton.dev_v1beta1_Task_ts-lint.yaml deleted file mode 100644 index cddd991..0000000 --- a/share/gitea-tekton-org/tekton.dev_v1beta1_Task_ts-lint.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: ts-lint - labels: - app.kubernetes.io/version: "0.1" - annotations: - tekton.dev/pipelines.minVersion: "0.12.1" - tekton.dev/categories: Code Quality - tekton.dev/tags: linter - tekton.dev/displayName: "TypeScript linter" - tekton.dev/platforms: "linux/amd64" -spec: - description: >- - This task can be used to perform lint check on TypeScript files - workspaces: - - name: shared-workspace - description: A workspace that contains the fetched git repository. - params: - - name: args - type: array - description: extra args needs to append - default: ["--help"] - steps: - - name: lint-typescript - image: docker.io/pipelinecomponents/tslint:0.14.1@sha256:85c7dd3f2b6d411fd01b8ad489afd20fb6ac4b2560f29ca506782c1f1a8d9b45 #tag: 0.14.1 - workingDir: $(workspaces.shared-workspace.path) - command: - - tslint - args: - - $(params.args) \ No newline at end of file diff --git a/share/gitea-tekton-org/v1_ConfigMap_auto-cd-templates.yaml b/share/gitea-tekton-org/v1_ConfigMap_auto-cd-templates.yaml index 36dc52e..5b1acb7 100644 --- a/share/gitea-tekton-org/v1_ConfigMap_auto-cd-templates.yaml +++ b/share/gitea-tekton-org/v1_ConfigMap_auto-cd-templates.yaml @@ -31,8 +31,17 @@ data: cleanup() { rm -rf .* * || true } + del_resources() { + local ress="$1" file="$2" + yq -i 'del(.resources[]|select(.=="'"$ress"'"))' "$file" + } + add_resources() { + local ress="$1" file="$2" + del_resources "$@" + yq -i '.resources += "'"$ress"'"' "$file" + } git_prepare() { - local url=$1 username=$2 email=$3 + local url="$1" username="$2" email="$3" mkdir -p "${HOME}/.ssh" cp -v "${WORKSPACE_SSH_DIRECTORY_PATH}"/* "${HOME}/.ssh" chmod 700 "${HOME}/.ssh" @@ -42,8 +51,9 @@ data: git clone "$url" --branch main --depth 1 . } install_base() { - mkdir -p bases/project bases/install bases/deploy bases/images + mkdir -p bases/project bases/install bases/deploy bases/images bases/trigger-push bases/trigger-tag ci template base-update.yaml.tmpl bases/images/update.yaml + copy empty-kusto.yaml "ci/kustomization.yaml" copy yamllint.yaml .yamllint.yaml copy base-repo.yaml bases/images/repo.yaml copy base-cert.yaml bases/project/cert.yaml @@ -57,18 +67,32 @@ data: copy install-install.yaml bases/install/install.yaml copy install-kusto.yaml bases/install/kustomization.yaml copy deploy-kusto.yaml bases/deploy/kustomization.yaml + copy trigger-kusto.yaml bases/trigger-tag/kustomization.yaml + copy trigger-tag.yaml bases/trigger-tag/trigger.yaml + copy trigger-kusto.yaml bases/trigger-push/kustomization.yaml + copy trigger-push.yaml bases/trigger-push/trigger.yaml template deploy-repo.yaml.tmpl bases/deploy/repo.yaml + if [ -f README.md ] && [ $(wc -l < README.md) -eq 2 ];then + rm README.md + fi + copy README.md README.md } create_prj() { export ORG_NAME=$(echo ${PROJECT_PATH}|sed 's#/.*##') - mkdir -p "projects/${PROJECT_NAME}" + mkdir -p "projects/${PROJECT_NAME}" "ci/${PROJECT_NAME}" + template ci-kusto.yaml.tmpl "ci/${PROJECT_NAME}/kustomization.yaml" + add_resources "${PROJECT_NAME}" "ci/kustomization.yaml" template project-kusto.yaml.tmpl "projects/${PROJECT_NAME}/kustomization.yaml" while [ $# -gt 0 ];do export STAGE=$1 mkdir -p "stages/${STAGE}/${PROJECT_NAME}" "stages/${STAGE}/deploy/${PROJECT_NAME}" - template stagep-kusto.yaml.tmpl "stages/deploy/${PROJECT_NAME}/kustomization.yaml" - template stages-kusto.yaml.tmpl "stages/deploy/kustomization.yaml" - template stages-kusto.yaml.tmpl "stages/${STAGE}/kustomization.yaml" + template deploy-project-kusto.yaml.tmpl "stages/${STAGE}/deploy/${PROJECT_NAME}/kustomization.yaml" + if [ "${STAGE}" == "prod" || "${STAGE}" == "production" ];then + copy deploy-policy-tag.yaml "stages/${STAGE}/policy.yaml" + else + copy deploy-policy-default.yaml "stages/${STAGE}/policy.yaml" + fi + copy empty-kusto.yaml "stages/${STAGE}/deploy/kustomization.yaml" template stage-kusto.yaml.tmpl "stages/${STAGE}/${PROJECT_NAME}/kustomization.yaml" template stage-ingress.yaml.tmpl "stages/${STAGE}/${PROJECT_NAME}/ingress.yaml" template stage-cert.yaml.tmpl "stages/${STAGE}/${PROJECT_NAME}/cert.yaml" @@ -77,19 +101,359 @@ data: done } activate_prj() { - touch /tmp/toto + export ORG_NAME=$(echo ${PROJECT_PATH}|sed 's#/.*##') + while [ $# -gt 0 ];do + export STAGE=$1 + add_resources "${PROJECT_NAME}" "stages/${STAGE}/deploy/kustomization.yaml" + shift + done } delete_prj() { rm -rf "projects/${PROJECT_NAME}" + while [ $# -gt 0 ];do + export STAGE=$1 + rm -rf "stages/${STAGE}/${PROJECT_NAME}" "stages/${STAGE}/deploy/${PROJECT_NAME}" + del_resources "${PROJECT_NAME}" "stages/${STAGE}/deploy/kustomization.yaml" + del_resources "${PROJECT_NAME}" "ci/kustomization.yaml" + shift + done } - staged-kusto.yaml: |- + ci-kusto.yaml.tmpl: |- + --- + apiVersion: kustomize.config.k8s.io/v1beta1 + kind: Kustomization + namePrefix: ${PROJECT_NAME}- + resources: + - ../../bases/trigger-push + - ../../bases/trigger-tag + patches: + - target: + kind: Trigger + name: push + patch: |- + - op: modify + path: /spec/template/spec/resourcetemplates/0/spec/pipelineRef/name + value: auto-ci-push + - target: + kind: Trigger + name: push + patch: |- + apiVersion: triggers.tekton.dev/v1beta1 + kind: Trigger + metadata: + name: "push" + spec: + interceptors: + - name: project-name + ref: + kind: ClusterInterceptor + name: cel + params: + - name: filter + value: body.repository.name == '${PROJECT_NAME}' + - target: + kind: Trigger + name: push + patch: |- + - op: modify + path: /spec/template/spec/resourcetemplates/0/spec/pipelineRef/name + value: auto-ci-tag + - target: + kind: Trigger + name: tag + patch: |- + apiVersion: triggers.tekton.dev/v1beta1 + kind: Trigger + metadata: + name: "tag" + spec: + interceptors: + - name: project-name + ref: + kind: ClusterInterceptor + name: cel + params: + - name: filter + value: body.repository.name == '${PROJECT_NAME}' + deploy-project-kusto.yaml.tmpl: |- + --- + apiVersion: kustomize.config.k8s.io/v1beta1 + kind: Kustomization + namePrefix: ${PROJECT_NAME}- + resources: + - ../../../../bases/install + - ../../../../bases/images + patches: + - target: + kind: ImagePolicy + name: policy + patch: |- + apiVersion: image.toolkit.fluxcd.io/v1beta2 + kind: ImagePolicy + metadata: + name: policy + spec: + imageRepositoryRef: + name: ${PROJECT_NAME}-repo + - target: + kind: ImagePolicy + name: policy + patch: ../policy.yaml + - target: + kind: ImageRepository + name: repo + patch: |- + apiVersion: image.toolkit.fluxcd.io/v1beta2 + kind: ImageRepository + metadata: + name: repo + spec: + image: ${ARTIFACTORY_URL}/${PROJECT_PATH} + - target: + kind: Kustomization + name: install + patch: |- + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: install + spec: + path: "./stages/${STAGE}/${PROJECT_NAME}" + - target: + kind: ImageUpdateAutomation + name: update + patch: |- + apiVersion: image.toolkit.fluxcd.io/v1beta1 + kind: ImageUpdateAutomation + metadata: + name: update + spec: + update: + path: ./stages/${STAGE}/${PROJECT_NAME} + deploy-policy-tag.yaml: |- + - op: add + path: /spec/filterTags/pattern + value: '^v(?P.*)$' + - op: add + path: /spec/filterTags/extract + value: '$semver' + - op: add + path: /spec/policy/semver/range + value: '>=0.1.0' + deploy-policy-default.yaml: |- + - op: add + path: /spec/filterTags/pattern + value: '^main-(?P.*)$' + - op: add + path: /spec/filterTags/extract + value: '$semver' + - op: add + path: /spec/policy/semver/range + value: '>=0.0.1-1' + README.md: |- + # Deployment configuration + Only valid if FluxCD is activated in the cluster + ## File structure + - `ci/`: Configuration for Tekton pipeline for . Should contain triggers and pipelines. + - `projects/`: Global configuration for deployment, affect every stages + - `stages//`: Configuration for deployment in the plateform + - `stages//deploy`: FluxCD configuration for , should be deployed on that stage namespace as it deploy everything needed to deploy the projects + trigger-kusto.yaml: |- + --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - policy.yaml - - update.yaml - - ../install - stages-kusto.yaml: |- + - trigger.yaml + trigger-push.yaml: |- + apiVersion: triggers.tekton.dev/v1beta1 + kind: Trigger + metadata: + metadata: + name: "push" + labels: + type: branch-push + spec: + bindings: + - name: artifactory-url + value: "$(extensions.artifactory-url)" + - name: project-name + value: "$(extensions.project-name)" + - name: project-path + value: "$(extensions.project-path)" + - name: git-repository-url + value: "$(extensions.git-repository-url)" + - name: git-revision + value: "$(extensions.git-revision)" + - name: branch-name + value: "$(extensions.branch-name)" + - name: git-default-branch + value: "$(extensions.git-default-branch)" + - name: generate-name + value: "$(extensions.generate-name)" + template: + spec: + params: + - name: artifactory-url + description: The url of the current artifactory + - name: project-name + description: The git repository name + - name: project-path + description: The path of the current project + - name: git-repository-url + description: The git repository url + - name: git-revision + description: The git revision + default: main + - name: git-default-branch + description: The git revision + default: main + - name: branch-name + description: The git branch + default: main + - name: generate-name + resourcetemplates: + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + generateName: $(tt.params.generate-name)- + annotations: + "mayfly.cloud.namecheap.com/expire": "336h" # 2 weeks + spec: + pipelineRef: + name: "auto-ci-push" + params: + - name: artifactory-url + value: $(tt.params.artifactory-url) + - name: project-name + value: $(tt.params.project-name) + - name: project-path + value: $(tt.params.project-path) + - name: git-url + value: $(tt.params.git-repository-url) + - name: git-revision + value: $(tt.params.git-revision) + - name: git-default-branch + value: $(tt.params.git-default-branch) + - name: branch-name + value: $(tt.params.branch-name) + workspaces: + - name: source + persistentVolumeClaim: + claimName: source + subPath: $(tt.params.git-revision) + - name: dockerconfig + secret: + secretName: gitea-docker + items: + - key: ".dockerconfigjson" + path: "config.json" + - name: sslcertdir + secret: + secretName: gitea + items: + - key: "ca.crt" + path: "ca.crt" + - name: ssh + secret: + secretName: ssh-credentials + items: + - key: "known_hosts" + path: "known_hosts" + - key: "ssh-privatekey" + path: "id_rsa" + - key: "ssh-publickey" + path: "id_rsa.pub" + trigger-tag.yaml: |- + apiVersion: triggers.tekton.dev/v1beta1 + kind: Trigger + metadata: + name: "tag" + labels: + type: tag-push + spec: + bindings: + - name: artifactory-url + value: "$(extensions.artifactory-url)" + - name: project-name + value: "$(extensions.project-name)" + - name: project-path + value: "$(extensions.project-path)" + - name: git-repository-url + value: "$(extensions.git-repository-url)" + - name: git-revision + value: "$(extensions.git-revision)" + - name: tag-name + value: $(extensions.tag-name) + - name: generate-name + value: "$(extensions.generate-name)" + template: + spec: + params: + - name: artifactory-url + description: The url of the current artifactory + - name: project-name + description: The git repository name + - name: project-path + description: The path of the current project + - name: git-repository-url + description: The git repository url + - name: git-revision + description: The git revision + default: main + - name: tag-name + description: The git tag + - name: generate-name + resourcetemplates: + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + generateName: $(tt.params.generate-name)- + annotations: + "mayfly.cloud.namecheap.com/expire": "1440h" # 2 months + spec: + pipelineRef: + name: "auto-ci-tag" + params: + - name: artifactory-url + value: $(tt.params.artifactory-url) + - name: project-name + value: $(tt.params.project-name) + - name: project-path + value: $(tt.params.project-path) + - name: git-url + value: $(tt.params.git-repository-url) + - name: git-revision + value: $(tt.params.git-revision) + - name: tag-name + value: $(tt.params.tag-name) + workspaces: + - name: source + persistentVolumeClaim: + claimName: source + subPath: $(tt.params.git-revision) + - name: dockerconfig + secret: + secretName: gitea-docker + items: + - key: ".dockerconfigjson" + path: "config.json" + - name: sslcertdir + secret: + secretName: gitea + items: + - key: "ca.crt" + path: "ca.crt" + - name: ssh + secret: + secretName: ssh-credentials + items: + - key: "known_hosts" + path: "known_hosts" + - key: "ssh-privatekey" + path: "id_rsa" + - key: "ssh-publickey" + path: "id_rsa.pub" + empty-kusto.yaml: |- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: @@ -354,10 +718,9 @@ data: --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization - namePrefix: deploy- resources: - - repo-git.yaml + - repo.yaml - ../install project-kusto.yaml.tmpl: |- --- @@ -371,52 +734,31 @@ data: component: ${PROJECT_NAME} resources: - - ../../bases/project + - ../../bases/project patches: - - target: - kind: ImagePolicy - name: policy - patch: |- - apiVersion: image.toolkit.fluxcd.io/v1beta2 - kind: ImagePolicy - metadata: - name: policy - spec: - imageRepositoryRef: - name: ${PROJECT_NAME}-repo - - target: - kind: ImageRepository - name: repo - patch: |- - apiVersion: image.toolkit.fluxcd.io/v1beta2 - kind: ImageRepository - metadata: - name: repo - spec: - image: ${ARTIFACTORY_URL}/${PROJECT_PATH} - - target: - kind: Deployment + - target: + kind: Deployment + name: app + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: name: app - patch: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: app - annotations: - configmap.reloader.stakater.com/reload: "${PROJECT_NAME}-config" - secret.reloader.stakater.com/reload: "${PROJECT_NAME}-secret" - spec: - selector: - template: - spec: - securityContext: - runAsGroup: 1000 - runAsUser: 1000 - fsGroup: 1000 - containers: + annotations: + configmap.reloader.stakater.com/reload: "${PROJECT_NAME}-config" + secret.reloader.stakater.com/reload: "${PROJECT_NAME}-secret" + spec: + selector: + template: + spec: + securityContext: + runAsGroup: 1000 + runAsUser: 1000 + fsGroup: 1000 + containers: + - name: app + ports: - name: app - ports: - - name: app - containerPort: 8080 - protocol: TCP \ No newline at end of file + containerPort: 8080 + protocol: TCP \ No newline at end of file