This commit is contained in:
2024-04-24 17:53:21 +02:00
parent 23405a43c3
commit c21c7c5b73
6 changed files with 435 additions and 92 deletions

View File

@@ -44,7 +44,7 @@ def load_config(root_dir, ci_root_dir):
"""Load the configuration from the configuration directory."""
ret = {
"files": [],
"languages": ["markdown", "docker", "rust", "shell", "python", "yaml", "js"],
"languages": ["markdown", "docker", "rust", "shell", "python", "yaml", "js", "make"],
"markdown": {"extentions": ["md"]},
"docker": {"extentions": ["Dockerfile"]},
"rust": {"extentions": ["rs"]},
@@ -64,6 +64,10 @@ def load_config(root_dir, ci_root_dir):
"extentions": ["ts", "js"],
"files": ["package.json", "yarn.lock", "schema.prisma"],
},
"make": {
"files": ["Makefile"],
"checkmake-args": []
},
}
if not os.path.isdir(ci_root_dir):
return ret
@@ -226,6 +230,11 @@ def get_results(config, files, root_dir):
if "shellcheck-args" in config["shell"]
else []
),
"checkmake-args": (
config["make"]["checkmake-args"]
if "checkmake-args" in config["make"]
else []
),
"black-args": (
config["python"]["black-args"] if "black-args" in config["python"] else []
),
@@ -245,6 +254,12 @@ def get_results(config, files, root_dir):
if "sh" in files:
append_stage(stages, "lint", "lint-shell", config["files"])
args["shellcheck-args"].extend(files["sh"])
if "sh" in files:
append_stage(stages, "lint", "lint-shell", config["files"])
args["shellcheck-args"].extend(files["sh"])
if "Makefile" in files:
append_stage(stages, "lint", "lint-make", config["files"])
args["checkmake-args"].extend(files["Makefile"])
if "rs" in files:
append_stage(stages, "lint", "lint-clippy", config["files"])
if "py" in files:
@@ -282,5 +297,6 @@ save_json(
get_images_name(files["Dockerfile"] if "Dockerfile" in files else [], root),
)
save_json("$(results.shellcheck-args.path)", args["shellcheck-args"])
save_json("$(results.checkmake-args.path)", args["checkmake-args"])
save_json("$(results.black-args.path)", args["black-args"])
save_json("$(results.pylint-args.path)", args["pylint-args"])

View File

@@ -44,6 +44,9 @@ resource "kubectl_manifest" "auto-ci-detector" {
- name: shellcheck-args
description: Arguments for shellcheck
type: array
- name: checkmake-args
description: Arguments for checkmake
type: array
- name: black-args
description: Arguments for black
type: array

View File

@@ -21,7 +21,6 @@ const sources=[
#{url: "https://github.com/tektoncd/catalog/raw/main/task/python-coverage/0.1/python-coverage.yaml", name: "python-coverage.yaml"},
#{url: "https://github.com/tektoncd/catalog/raw/main/task/shellcheck/0.1/shellcheck.yaml", name: "shellcheck.yaml"},
#{url: "https://github.com/tektoncd/catalog/raw/main/task/sonarqube-scanner/0.4/sonarqube-scanner.yaml", name: "sonarqube-scanner.yaml"},
#{url: "https://github.com/tektoncd/catalog/raw/main/task/ts-lint/0.1/ts-lint.yaml", name: "ts-lint.yaml"},
#{url: "https://github.com/tektoncd/catalog/raw/main/task/yaml-lint/0.1/yaml-lint.yaml", name: "yaml-lint.yaml"},
];
fn pre_pack() {

View File

@@ -214,6 +214,20 @@ spec:
workspaces:
- name: shared-workspace
workspace: source
- name: lint-python
onError: continue
when:
- input: "lint-python"
operator: in
values: ["$(tasks.detect-stages.results.stages-lint[*])"]
params:
- name: args
value: $(tasks.detect-stages.results.pylint-args)
taskRef:
name: pylint
workspaces:
- name: shared-workspace
workspace: source
- name: publish-docker
runAfter: ["lint-shell","lint-docker","lint-yaml","lint-black","lint-python"]
when:

View File

@@ -1,31 +0,0 @@
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: ts-lint
labels:
app.kubernetes.io/version: "0.1"
annotations:
tekton.dev/pipelines.minVersion: "0.12.1"
tekton.dev/categories: Code Quality
tekton.dev/tags: linter
tekton.dev/displayName: "TypeScript linter"
tekton.dev/platforms: "linux/amd64"
spec:
description: >-
This task can be used to perform lint check on TypeScript files
workspaces:
- name: shared-workspace
description: A workspace that contains the fetched git repository.
params:
- name: args
type: array
description: extra args needs to append
default: ["--help"]
steps:
- name: lint-typescript
image: docker.io/pipelinecomponents/tslint:0.14.1@sha256:85c7dd3f2b6d411fd01b8ad489afd20fb6ac4b2560f29ca506782c1f1a8d9b45 #tag: 0.14.1
workingDir: $(workspaces.shared-workspace.path)
command:
- tslint
args:
- $(params.args)

View File

@@ -31,8 +31,17 @@ data:
cleanup() {
rm -rf .* * || true
}
del_resources() {
local ress="$1" file="$2"
yq -i 'del(.resources[]|select(.=="'"$ress"'"))' "$file"
}
add_resources() {
local ress="$1" file="$2"
del_resources "$@"
yq -i '.resources += "'"$ress"'"' "$file"
}
git_prepare() {
local url=$1 username=$2 email=$3
local url="$1" username="$2" email="$3"
mkdir -p "${HOME}/.ssh"
cp -v "${WORKSPACE_SSH_DIRECTORY_PATH}"/* "${HOME}/.ssh"
chmod 700 "${HOME}/.ssh"
@@ -42,8 +51,9 @@ data:
git clone "$url" --branch main --depth 1 .
}
install_base() {
mkdir -p bases/project bases/install bases/deploy bases/images
mkdir -p bases/project bases/install bases/deploy bases/images bases/trigger-push bases/trigger-tag ci
template base-update.yaml.tmpl bases/images/update.yaml
copy empty-kusto.yaml "ci/kustomization.yaml"
copy yamllint.yaml .yamllint.yaml
copy base-repo.yaml bases/images/repo.yaml
copy base-cert.yaml bases/project/cert.yaml
@@ -57,18 +67,32 @@ data:
copy install-install.yaml bases/install/install.yaml
copy install-kusto.yaml bases/install/kustomization.yaml
copy deploy-kusto.yaml bases/deploy/kustomization.yaml
copy trigger-kusto.yaml bases/trigger-tag/kustomization.yaml
copy trigger-tag.yaml bases/trigger-tag/trigger.yaml
copy trigger-kusto.yaml bases/trigger-push/kustomization.yaml
copy trigger-push.yaml bases/trigger-push/trigger.yaml
template deploy-repo.yaml.tmpl bases/deploy/repo.yaml
if [ -f README.md ] && [ $(wc -l < README.md) -eq 2 ];then
rm README.md
fi
copy README.md README.md
}
create_prj() {
export ORG_NAME=$(echo ${PROJECT_PATH}|sed 's#/.*##')
mkdir -p "projects/${PROJECT_NAME}"
mkdir -p "projects/${PROJECT_NAME}" "ci/${PROJECT_NAME}"
template ci-kusto.yaml.tmpl "ci/${PROJECT_NAME}/kustomization.yaml"
add_resources "${PROJECT_NAME}" "ci/kustomization.yaml"
template project-kusto.yaml.tmpl "projects/${PROJECT_NAME}/kustomization.yaml"
while [ $# -gt 0 ];do
export STAGE=$1
mkdir -p "stages/${STAGE}/${PROJECT_NAME}" "stages/${STAGE}/deploy/${PROJECT_NAME}"
template stagep-kusto.yaml.tmpl "stages/deploy/${PROJECT_NAME}/kustomization.yaml"
template stages-kusto.yaml.tmpl "stages/deploy/kustomization.yaml"
template stages-kusto.yaml.tmpl "stages/${STAGE}/kustomization.yaml"
template deploy-project-kusto.yaml.tmpl "stages/${STAGE}/deploy/${PROJECT_NAME}/kustomization.yaml"
if [ "${STAGE}" == "prod" || "${STAGE}" == "production" ];then
copy deploy-policy-tag.yaml "stages/${STAGE}/policy.yaml"
else
copy deploy-policy-default.yaml "stages/${STAGE}/policy.yaml"
fi
copy empty-kusto.yaml "stages/${STAGE}/deploy/kustomization.yaml"
template stage-kusto.yaml.tmpl "stages/${STAGE}/${PROJECT_NAME}/kustomization.yaml"
template stage-ingress.yaml.tmpl "stages/${STAGE}/${PROJECT_NAME}/ingress.yaml"
template stage-cert.yaml.tmpl "stages/${STAGE}/${PROJECT_NAME}/cert.yaml"
@@ -77,19 +101,359 @@ data:
done
}
activate_prj() {
touch /tmp/toto
export ORG_NAME=$(echo ${PROJECT_PATH}|sed 's#/.*##')
while [ $# -gt 0 ];do
export STAGE=$1
add_resources "${PROJECT_NAME}" "stages/${STAGE}/deploy/kustomization.yaml"
shift
done
}
delete_prj() {
rm -rf "projects/${PROJECT_NAME}"
while [ $# -gt 0 ];do
export STAGE=$1
rm -rf "stages/${STAGE}/${PROJECT_NAME}" "stages/${STAGE}/deploy/${PROJECT_NAME}"
del_resources "${PROJECT_NAME}" "stages/${STAGE}/deploy/kustomization.yaml"
del_resources "${PROJECT_NAME}" "ci/kustomization.yaml"
shift
done
}
staged-kusto.yaml: |-
ci-kusto.yaml.tmpl: |-
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namePrefix: ${PROJECT_NAME}-
resources:
- ../../bases/trigger-push
- ../../bases/trigger-tag
patches:
- target:
kind: Trigger
name: push
patch: |-
- op: modify
path: /spec/template/spec/resourcetemplates/0/spec/pipelineRef/name
value: auto-ci-push
- target:
kind: Trigger
name: push
patch: |-
apiVersion: triggers.tekton.dev/v1beta1
kind: Trigger
metadata:
name: "push"
spec:
interceptors:
- name: project-name
ref:
kind: ClusterInterceptor
name: cel
params:
- name: filter
value: body.repository.name == '${PROJECT_NAME}'
- target:
kind: Trigger
name: push
patch: |-
- op: modify
path: /spec/template/spec/resourcetemplates/0/spec/pipelineRef/name
value: auto-ci-tag
- target:
kind: Trigger
name: tag
patch: |-
apiVersion: triggers.tekton.dev/v1beta1
kind: Trigger
metadata:
name: "tag"
spec:
interceptors:
- name: project-name
ref:
kind: ClusterInterceptor
name: cel
params:
- name: filter
value: body.repository.name == '${PROJECT_NAME}'
deploy-project-kusto.yaml.tmpl: |-
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namePrefix: ${PROJECT_NAME}-
resources:
- ../../../../bases/install
- ../../../../bases/images
patches:
- target:
kind: ImagePolicy
name: policy
patch: |-
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImagePolicy
metadata:
name: policy
spec:
imageRepositoryRef:
name: ${PROJECT_NAME}-repo
- target:
kind: ImagePolicy
name: policy
patch: ../policy.yaml
- target:
kind: ImageRepository
name: repo
patch: |-
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImageRepository
metadata:
name: repo
spec:
image: ${ARTIFACTORY_URL}/${PROJECT_PATH}
- target:
kind: Kustomization
name: install
patch: |-
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: install
spec:
path: "./stages/${STAGE}/${PROJECT_NAME}"
- target:
kind: ImageUpdateAutomation
name: update
patch: |-
apiVersion: image.toolkit.fluxcd.io/v1beta1
kind: ImageUpdateAutomation
metadata:
name: update
spec:
update:
path: ./stages/${STAGE}/${PROJECT_NAME}
deploy-policy-tag.yaml: |-
- op: add
path: /spec/filterTags/pattern
value: '^v(?P<semver>.*)$'
- op: add
path: /spec/filterTags/extract
value: '$semver'
- op: add
path: /spec/policy/semver/range
value: '>=0.1.0'
deploy-policy-default.yaml: |-
- op: add
path: /spec/filterTags/pattern
value: '^main-(?P<semver>.*)$'
- op: add
path: /spec/filterTags/extract
value: '$semver'
- op: add
path: /spec/policy/semver/range
value: '>=0.0.1-1'
README.md: |-
# Deployment configuration
Only valid if FluxCD is activated in the cluster
## File structure
- `ci/<project_name>`: Configuration for Tekton pipeline for <project_name>. Should contain triggers and pipelines.
- `projects/<project_name>`: Global configuration for <project_name> deployment, affect every stages
- `stages/<stage>/<project_name>`: Configuration for <project_name> deployment in the <stage> plateform
- `stages/<stage>/deploy`: FluxCD configuration for <stage>, should be deployed on that stage namespace as it deploy everything needed to deploy the projects
trigger-kusto.yaml: |-
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- policy.yaml
- update.yaml
- ../install
stages-kusto.yaml: |-
- trigger.yaml
trigger-push.yaml: |-
apiVersion: triggers.tekton.dev/v1beta1
kind: Trigger
metadata:
metadata:
name: "push"
labels:
type: branch-push
spec:
bindings:
- name: artifactory-url
value: "$(extensions.artifactory-url)"
- name: project-name
value: "$(extensions.project-name)"
- name: project-path
value: "$(extensions.project-path)"
- name: git-repository-url
value: "$(extensions.git-repository-url)"
- name: git-revision
value: "$(extensions.git-revision)"
- name: branch-name
value: "$(extensions.branch-name)"
- name: git-default-branch
value: "$(extensions.git-default-branch)"
- name: generate-name
value: "$(extensions.generate-name)"
template:
spec:
params:
- name: artifactory-url
description: The url of the current artifactory
- name: project-name
description: The git repository name
- name: project-path
description: The path of the current project
- name: git-repository-url
description: The git repository url
- name: git-revision
description: The git revision
default: main
- name: git-default-branch
description: The git revision
default: main
- name: branch-name
description: The git branch
default: main
- name: generate-name
resourcetemplates:
- apiVersion: tekton.dev/v1beta1
kind: PipelineRun
metadata:
generateName: $(tt.params.generate-name)-
annotations:
"mayfly.cloud.namecheap.com/expire": "336h" # 2 weeks
spec:
pipelineRef:
name: "auto-ci-push"
params:
- name: artifactory-url
value: $(tt.params.artifactory-url)
- name: project-name
value: $(tt.params.project-name)
- name: project-path
value: $(tt.params.project-path)
- name: git-url
value: $(tt.params.git-repository-url)
- name: git-revision
value: $(tt.params.git-revision)
- name: git-default-branch
value: $(tt.params.git-default-branch)
- name: branch-name
value: $(tt.params.branch-name)
workspaces:
- name: source
persistentVolumeClaim:
claimName: source
subPath: $(tt.params.git-revision)
- name: dockerconfig
secret:
secretName: gitea-docker
items:
- key: ".dockerconfigjson"
path: "config.json"
- name: sslcertdir
secret:
secretName: gitea
items:
- key: "ca.crt"
path: "ca.crt"
- name: ssh
secret:
secretName: ssh-credentials
items:
- key: "known_hosts"
path: "known_hosts"
- key: "ssh-privatekey"
path: "id_rsa"
- key: "ssh-publickey"
path: "id_rsa.pub"
trigger-tag.yaml: |-
apiVersion: triggers.tekton.dev/v1beta1
kind: Trigger
metadata:
name: "tag"
labels:
type: tag-push
spec:
bindings:
- name: artifactory-url
value: "$(extensions.artifactory-url)"
- name: project-name
value: "$(extensions.project-name)"
- name: project-path
value: "$(extensions.project-path)"
- name: git-repository-url
value: "$(extensions.git-repository-url)"
- name: git-revision
value: "$(extensions.git-revision)"
- name: tag-name
value: $(extensions.tag-name)
- name: generate-name
value: "$(extensions.generate-name)"
template:
spec:
params:
- name: artifactory-url
description: The url of the current artifactory
- name: project-name
description: The git repository name
- name: project-path
description: The path of the current project
- name: git-repository-url
description: The git repository url
- name: git-revision
description: The git revision
default: main
- name: tag-name
description: The git tag
- name: generate-name
resourcetemplates:
- apiVersion: tekton.dev/v1beta1
kind: PipelineRun
metadata:
generateName: $(tt.params.generate-name)-
annotations:
"mayfly.cloud.namecheap.com/expire": "1440h" # 2 months
spec:
pipelineRef:
name: "auto-ci-tag"
params:
- name: artifactory-url
value: $(tt.params.artifactory-url)
- name: project-name
value: $(tt.params.project-name)
- name: project-path
value: $(tt.params.project-path)
- name: git-url
value: $(tt.params.git-repository-url)
- name: git-revision
value: $(tt.params.git-revision)
- name: tag-name
value: $(tt.params.tag-name)
workspaces:
- name: source
persistentVolumeClaim:
claimName: source
subPath: $(tt.params.git-revision)
- name: dockerconfig
secret:
secretName: gitea-docker
items:
- key: ".dockerconfigjson"
path: "config.json"
- name: sslcertdir
secret:
secretName: gitea
items:
- key: "ca.crt"
path: "ca.crt"
- name: ssh
secret:
secretName: ssh-credentials
items:
- key: "known_hosts"
path: "known_hosts"
- key: "ssh-privatekey"
path: "id_rsa"
- key: "ssh-publickey"
path: "id_rsa.pub"
empty-kusto.yaml: |-
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
@@ -354,10 +718,9 @@ data:
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namePrefix: deploy-
resources:
- repo-git.yaml
- repo.yaml
- ../install
project-kusto.yaml.tmpl: |-
---
@@ -371,52 +734,31 @@ data:
component: ${PROJECT_NAME}
resources:
- ../../bases/project
- ../../bases/project
patches:
- target:
kind: ImagePolicy
name: policy
patch: |-
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImagePolicy
metadata:
name: policy
spec:
imageRepositoryRef:
name: ${PROJECT_NAME}-repo
- target:
kind: ImageRepository
name: repo
patch: |-
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImageRepository
metadata:
name: repo
spec:
image: ${ARTIFACTORY_URL}/${PROJECT_PATH}
- target:
kind: Deployment
- target:
kind: Deployment
name: app
patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
patch: |-
apiVersion: apps/v1
kind: Deployment
metadata:
name: app
annotations:
configmap.reloader.stakater.com/reload: "${PROJECT_NAME}-config"
secret.reloader.stakater.com/reload: "${PROJECT_NAME}-secret"
spec:
selector:
template:
spec:
securityContext:
runAsGroup: 1000
runAsUser: 1000
fsGroup: 1000
containers:
annotations:
configmap.reloader.stakater.com/reload: "${PROJECT_NAME}-config"
secret.reloader.stakater.com/reload: "${PROJECT_NAME}-secret"
spec:
selector:
template:
spec:
securityContext:
runAsGroup: 1000
runAsUser: 1000
fsGroup: 1000
containers:
- name: app
ports:
- name: app
ports:
- name: app
containerPort: 8080
protocol: TCP
containerPort: 8080
protocol: TCP