Jenkins + Argo CD (kaniko, harbor, cert-manager)

Jenkins, ArgoCD, Harbor 각각 Azure Kubernetes Service에 배포

 

Ingress Nginx Controller 를 사용하여 각 서비스에 로드밸런싱

모든 Ingress에 인증서 적용하여 TLS 통신 가능

Cert-manager로 인증서 관리 및 Let's encrypt 인증서 자동 갱신

 

Jenkins

kubectl create ns devops-tools

 

account.yaml

#ServiceAccount설치
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: jenkins-admin
rules:
  - apiGroups: [""]
    resources: ["*"]
    verbs: ["*"]

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins-admin
  namespace: devops-tools
  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: jenkins-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: jenkins-admin
subjects:
- kind: ServiceAccount
  name: jenkins-admin
  namespace: devops-tools

 

jenkins-certificate.yaml

# jenkins-certificate.yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: jenkins-secret
  namespace: devops-tools
spec:
  secretName: jenkins-secret
  privateKey:
    rotationPolicy: Always
  commonName: jenkins.k-tech.cloud
  dnsNames:
    - jenkins.k-tech.cloud
  usages:
    - digital signature
    - key encipherment
    - server auth
  issuerRef:
    name: letsencrypt
    kind: ClusterIssuer

 

cluster-issuer.yaml

apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt
spec:
  acme:
    server: <https://acme-v02.api.letsencrypt.org/directory>
    email: kangbock0827@naver.com
    privateKeySecretRef:
      name: letsencrypt
    solvers:
    - http01:
        ingress:
          class: nginx
          podTemplate:
            spec:
              nodeSelector:
                "kubernetes.io/os": linux

 

jenkins.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  namespace: devops-tools
  annotations:
    certmanager.k8s.io/disable-auto-restart: "true"
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jenkins-server
  template:
    metadata:
      labels:
        app: jenkins-server
    spec:
      securityContext:
            fsGroup: 1000
            runAsUser: 1000
      serviceAccountName: jenkins-admin
      containers:
        - name: jenkins
          image: jenkins/jenkins:lts
          resources:
            limits:
              memory: "2Gi"
              cpu: "1000m"
            requests:
              memory: "500Mi"
              cpu: "500m"
          ports:
            - name: httpport
              containerPort: 8080
            - name: jnlpport
              containerPort: 50000
          livenessProbe:
            httpGet:
              path: "/login"
              port: 8080
            initialDelaySeconds: 90
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 5
          readinessProbe:
            httpGet:
              path: "/login"
              port: 8080
            initialDelaySeconds: 60
            periodSeconds: 10
            timeoutSeconds: 5
            failureThreshold: 3
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins-service
  namespace: devops-tools
  annotations:
      prometheus.io/scrape: 'true'
      prometheus.io/path:   /
      prometheus.io/port:   '8080'
spec:
  selector:
    app: jenkins-server
  type: ClusterIP
  ports:
    - name: http
      protocol: TCP
      port: 8080
      targetPort: 8080
    - name: jnlp
      protocol: TCP
      port: 50000
      targetPort: 50000
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: jenkins
  namespace: devops-tools
spec:
  tls:
  - hosts:
      - jenkins.k-tech.cloud
    secretName: jenkins-secret
  ingressClassName: nginx
  rules:
  - host: jenkins.k-tech.cloud
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: jenkins-service
            port:
              number: 8080
      - path: /agents
        pathType: Prefix
        backend:
          service:
            name: jenkins-svc
            port:
              name: agents

 

Jenkins Password

kubectl exec -it svc/jenkins-service -n devops-tools -- cat /var/jenkins_home/secrets/initialAdminPassword

 

플러그인 관리 → kubernetes 설치

시스템 설정 → GitHub Server 연결

Node 관리 → Clouds → New Cloud → WebSocket Check

 

kaniko

Secret 생성

docker login
#harbor.k-tech.cloud 로 도커 로그인 시도
cat ~/.docker/config.json
cat ~/.docker/config.json | base64

 

regcred.yaml

apiVersion: v1
kind: Secret
metadata:
  name: docker-config-secret
  namespace: devops-tools
data:
  .dockerconfigjson: 인코딩한 데이터
type: kubernetes.io/dockerconfigjson

 

kubectl apply -f regcred.yaml
mkdir /home/adminuser/kaniko
cp -r .docker /home/adminuser/kaniko/

 

Pipeline 생성

구성

💡 check : Do not allow the pipeline to resume if the controller restarts

💡 check : GitHub hook trigger for GITScm polling

 

pipeline

podTemplate(yaml: '''
    apiVersion: v1
    kind: Pod
    spec:
      containers:
      - name: alpine
        image: alpine/git:latest
        command:
        - sleep
        args:
        - 99d
      - name: kaniko
        image: gcr.io/kaniko-project/executor:debug
        command:
        - sleep
        args:
        - 99d
        volumeMounts:
        - name: kaniko-secret
          mountPath: /kaniko/.docker
      restartPolicy: Never
      volumes:
      - name: kaniko-secret
        secret:
          secretName: docker-config-secret
          items:
            - key: .dockerconfigjson
              path: config.json
''') {

    node(POD_LABEL) {

        stage('Kaniko Build and Push') {
            git url: 'https://github.com/kangbock/kaniko.git', branch: 'main'
            script(){
                GIT_TAG = sh (
                    script: 'git describe --always',
                    returnStdout: true
                ).trim()
            }
            
            container('kaniko') {
                stage('Build with kaniko') {
                    // sh '/kaniko/executor -f `pwd`/nginx/Dockerfile -c `pwd` --insecure --skip-tls-verify --cache=true --destination=harbor.k-tech.cloud/kaniko/nginx:' + GIT_TAG
                    sh '/kaniko/executor -f `pwd`/nginx/Dockerfile -c `pwd` --insecure --cache=true --destination=harbor.k-tech.cloud/kaniko/nginx:${BUILD_NUMBER}'
                    sh '/kaniko/executor -f `pwd`/nodejs/Dockerfile -c `pwd` --insecure --cache=true --destination=harbor.k-tech.cloud/kaniko/nodejs:${BUILD_NUMBER}'

                }
            }
            
            git url: 'https://github.com/kangbock/ArgoCD.git', branch: 'main'
            container('alpine') {
                stage('GitHub Push') {
                    // 'yq -i \'.imagetag="'+ GIT_TAG + '"\' ./gcmp-api/values_dev.yaml'
                    // sh 'sed -i \'5s/.*/  newTag: '+ GIT_TAG + '/g\' ./overlays/dev/kustomization.yaml'
                    // sh 'sed -i \'17s/nginx:*/nginx: '+ GIT_TAG + '/g\' ./base/n1.yaml'
                    sh 'sed -i s/newTag:.*/newTag:${BUILD_NUMBER}/g ./overlays/dev/kustomization.yaml'
                    sh 'sed -i s/nginx:.*/nginx:${BUILD_NUMBER}/g ./base/n1.yaml'
                    sh 'sed -i s/nodejs:.*/nodejs:${BUILD_NUMBER}/g ./base/j1.yaml'
                    sh 'git config --global --add safe.directory /home/jenkins/agent/workspace/test'
                    sh 'git config --global user.email \'kangbock@naver.com\''
                    sh 'git config --global user.name \'kangbock\''
                    sh 'git add ./'
                    // sh 'git commit -a -m \"updated the image tag to ' + GIT_TAG + '\" || true'
                    sh 'git commit -a -m "updated the image tag to ${BUILD_NUMBER}" || true'
                    sh 'git remote add kb97 https://[GIT_TOKEN]@github.com/kangbock/ArgoCD.git'
                    sh 'git push -u kb97 main'
                }
            }
        }


    }
}

 

 

ArgoCD

argocd-certificate.yaml

# jenkins-certificate.yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: argocd-server-tls
  namespace: argocd
spec:
  secretName: argocd-server-tls
  privateKey:
    rotationPolicy: Always
  commonName: argocd.k-tech.cloud
  dnsNames:
    - argocd.k-tech.cloud
  usages:
    - digital signature
    - key encipherment
    - server auth
  issuerRef:
    name: letsencrypt
    kind: ClusterIssuer

 

kubectl create namespace argocd
kubectl apply -n argocd -f <https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml>

 

argocd-ingress.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: argocd
  namespace: argocd
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
    # If you encounter a redirect loop or are getting a 307 response code
    # then you need to force the nginx ingress to connect to the backend using HTTPS.
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
spec:
  tls:
  - hosts:
      - argocd.k-tech.cloud
    secretName: argocd-server-tls
  ingressClassName: nginx
  rules:
  - host: argocd.k-tech.cloud
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: argocd-server
            port:
              name: https

 

# 암호 찾기
kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d; echo

 

 

nginx-certificate.yaml

# nginx-certificate.yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: nginx-secret
spec:
  secretName: nginx-secret
  privateKey:
    rotationPolicy: Always
  commonName: www.k-tech.cloud
  dnsNames:
    - www.k-tech.cloud
  usages:
    - digital signature
    - key encipherment
    - server auth
  issuerRef:
    name: letsencrypt
    kind: ClusterIssuer

mysql

apt install -y mysql-client=8.*
sh kaniko/nginx/mysql.sh