Feature/cicd

3 years ago · a45cd6ebb2
parent 63075aad00
commit a45cd6ebb2
9 changed files with 365 additions and 81 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,7 +1,10 @@
 .gitignore
-.gitlab/*
 .gitlab-ci.yml
 README.md
-.git/*
 .pre-commit-config.yaml
 .flake8
+.env.like
+.git
+.gitlab
+docker-compose.*
+deploy
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,86 +1,137 @@
+---
+variables:
+  DOCKER_IMAGE_TAG: ${YC_CONTAINER_REGISTRY}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_SLUG}.${CI_COMMIT_SHORT_SHA}
+  RELEASE_DOCKER_IMAGE_TAG: ${YC_CONTAINER_REGISTRY}/${CI_PROJECT_NAME}:${CI_COMMIT_TAG}
+  DEPLOY_KUBER_NAMESPACE: spatial
+
 stages:
  - build
-  - lint
+  # - lint
  - deploy

-variables:
-  CONTAINERS_NAME: sst_postamates
-  IMAGE_NAME: $CI_REGISTRY_IMAGE
-
-default:
+build-docker:
+  stage: build
  tags:
-    - deploy-remote
+    - shell
+  script:
+    - >
+        docker build
+        --build-arg YC_CONTAINER_REGISTRY=${YC_CONTAINER_REGISTRY}
+        -t ${DOCKER_IMAGE_TAG} .
+    - docker push ${DOCKER_IMAGE_TAG}

-.use_cached_requirements: &use_cached_requirements
+deploy_dev_kuber:
+  extends: .deploy_base_kuber
  variables:
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  cache:
-    key: $CI_COMMIT_REF_SLUG
-    paths:
-      - .cache/pip
-    policy: pull-push
-
-build-job:
-  stage: build
-  before_script:
-    - sudo docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY
-    - ls
-  script:
-    - sudo docker pull $IMAGE_NAME:latest || true
-    - sudo docker build --cache-from $IMAGE_NAME:latest --tag $IMAGE_NAME:$CI_COMMIT_SHORT_SHA --tag $IMAGE_NAME:latest .
-    - sudo docker push $IMAGE_NAME:$CI_COMMIT_SHORT_SHA
-    - sudo docker push $IMAGE_NAME:latest
-  rules:
-    - if: $CI_COMMIT_BRANCH == 'sst_main'
+    INGRESS_HOST: "postnet.dev.selftech.ru"
+  tags:
+    - docker
+  environment:
+    name: dev

-.deploy: &deploy-common
+.deploy_base_kuber:
+  image: ${YC_CONTAINER_REGISTRY}/public/helm-kubectl-git:1.0.0
  stage: deploy
  before_script:
-    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - ssh-keyscan $REMOTE_HOST >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-    - ssh $REMOTE_USER docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY
+    - KUBE_CONFIG=`echo ${CI_ENVIRONMENT_NAME}_kubeconfig`
+    - mkdir -p ${HOME}/.kube
+    - 'cat ${!KUBE_CONFIG} > ${HOME}/.kube/config'
+    - chmod -R 700 ${HOME}/.kube
  script:
-    - cd $CI_PROJECT_DIR
-    - ssh $REMOTE_USER docker pull $IMAGE_NAME:$CI_COMMIT_SHORT_SHA
-    - DOCKER_HOST="ssh://$REMOTE_USER" docker-compose -f $COMPOSE up -d
-    - DOCKER_HOST="ssh://$REMOTE_USER" docker restart sst_postamates_martin
+    - sed -i "s|DEPLOY_IMAGE_TAG|${DOCKER_IMAGE_TAG}|g" ./deploy/django.yml
+    - sed -i "s|ADDRESS_INGRESS_HOST|${INGRESS_HOST}|g" ./deploy/django.yml
+    - sed -i "s|DEPLOY_IMAGE_TAG|${DOCKER_IMAGE_TAG}|g" ./deploy/worker.yml
+    - sed -i "s|DEPLOY_IMAGE_TAG|${DOCKER_IMAGE_TAG}|g" ./deploy/beat.yml
+    - kubectl apply -f ./deploy/django.yml
+    - kubectl apply -f ./deploy/worker.yml
+    - kubectl apply -f ./deploy/beat.yml
+  dependencies: []
+  artifacts:
+    paths:
+      - ./deploy/django.yml
+      - ./deploy/worker.yml
+      - ./deploy/beat.yml
+    expire_in: 1 week
+  when: manual

-deploy-test-job:
-  <<: *deploy-common
-  variables:
-    REMOTE_HOST: $REMOTE_HOST_TEST
-    REMOTE_USER: $REMOTE_USER_TEST
-    COMPOSE: docker-compose.test.yml
-  rules:
-    - if: $CI_COMMIT_BRANCH == 'sst_main'
+# variables:
+#   CONTAINERS_NAME: sst_postamates
+#   IMAGE_NAME: $CI_REGISTRY_IMAGE

-pylint:
-  <<: *use_cached_requirements
-  stage: lint
-  before_script:
-    - pip install flake8-codeclimate
-  script:
-    - set -x
-    - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA  | egrep ".py$" | grep -v ".docker/"|| true)
-    - >
-      if [ ! -z "$FLIST" ];
-      then
-        mkdir -p build/reports;
-        flake8 --format junit-xml $FLIST > build/reports/flake8-junit.xml || flake8 $FLIST;
-      else
-        echo "No files to lint"
-        true;
-      fi
-  only:
-    refs:
-      - merge_requests
-  artifacts:
-    reports:
-      junit: ./build/reports/flake8-junit.xml
-    expire_in: 7 days
-  allow_failure: false
+# default:
+#   tags:
+#     - deploy-remote
+
+# .use_cached_requirements: &use_cached_requirements
+#   variables:
+#     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+#   cache:
+#     key: $CI_COMMIT_REF_SLUG
+#     paths:
+#       - .cache/pip
+#     policy: pull-push
+
+# build-job:
+#   stage: build
+#   before_script:
+#     - sudo docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY
+#     - ls
+#   script:
+#     - sudo docker pull $IMAGE_NAME:latest || true
+#     - sudo docker build --cache-from $IMAGE_NAME:latest --tag $IMAGE_NAME:$CI_COMMIT_SHORT_SHA --tag $IMAGE_NAME:latest .
+#     - sudo docker push $IMAGE_NAME:$CI_COMMIT_SHORT_SHA
+#     - sudo docker push $IMAGE_NAME:latest
+#   rules:
+#     - if: $CI_COMMIT_BRANCH == 'sst_main'
+
+# .deploy: &deploy-common
+#   stage: deploy
+#   before_script:
+#     - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+#     - eval $(ssh-agent -s)
+#     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+#     - mkdir -p ~/.ssh
+#     - chmod 700 ~/.ssh
+#     - ssh-keyscan $REMOTE_HOST >> ~/.ssh/known_hosts
+#     - chmod 644 ~/.ssh/known_hosts
+#     - ssh $REMOTE_USER docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY
+#   script:
+#     - cd $CI_PROJECT_DIR
+#     - ssh $REMOTE_USER docker pull $IMAGE_NAME:$CI_COMMIT_SHORT_SHA
+#     - DOCKER_HOST="ssh://$REMOTE_USER" docker-compose -f $COMPOSE up -d
+#     - DOCKER_HOST="ssh://$REMOTE_USER" docker restart sst_postamates_martin
+
+# deploy-test-job:
+#   <<: *deploy-common
+#   variables:
+#     REMOTE_HOST: $REMOTE_HOST_TEST
+#     REMOTE_USER: $REMOTE_USER_TEST
+#     COMPOSE: docker-compose.test.yml
+#   rules:
+#     - if: $CI_COMMIT_BRANCH == 'sst_main'
+
+# pylint:
+#   <<: *use_cached_requirements
+#   stage: lint
+#   before_script:
+#     - pip install flake8-codeclimate
+#   script:
+#     - set -x
+#     - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA  | egrep ".py$" | grep -v ".docker/"|| true)
+#     - >
+#       if [ ! -z "$FLIST" ];
+#       then
+#         mkdir -p build/reports;
+#         flake8 --format junit-xml $FLIST > build/reports/flake8-junit.xml || flake8 $FLIST;
+#       else
+#         echo "No files to lint"
+#         true;
+#       fi
+#   only:
+#     refs:
+#       - merge_requests
+#   artifacts:
+#     reports:
+#       junit: ./build/reports/flake8-junit.xml
+#     expire_in: 7 days
+#   allow_failure: false
--- a/12
+++ b/12
@ -1,16 +1,18 @@
-FROM python:3.8
+ARG YC_CONTAINER_REGISTRY
+FROM ${YC_CONTAINER_REGISTRY}/public/python:3.8

-RUN apt-get update &&\
+RUN apt-get update && \
    apt-get install -y binutils libproj-dev gdal-bin

 ENV PYTHONUNBUFFERED 1

-RUN mkdir /code
-
 WORKDIR /code

 COPY requirements.txt /code/

-RUN pip install -r requirements.txt
+RUN pip install -r /code/requirements.txt
+
+# RUN apt purge binutils libproj-dev gdal-bin -y && \
+#     apt autoremove -y

 COPY . /code/
--- a/deploy/beat.yml
+++ b/deploy/beat.yml
@ -0,0 +1,32 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postamates-beat
+  namespace: spatial
+  labels:
+    app.kubernetes.io/name: postamates-beat
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: postamates-beat
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: postamates-beat
+    spec:
+      containers:
+      - name: beat
+        image: DEPLOY_IMAGE_TAG
+        command: ["sh", "-c", "celery -A postamates beat -l info --scheduler beat_celery_beat.schedulers:DatabaseScheduler"]
+        envFrom:
+          - configMapRef:
+              name: postamates-configmap
+        resources:
+          requests:
+            memory: "256Mi"
+            cpu: "50m"
+          limits:
+            memory: "256Mi"
+            cpu: "50m"
--- a/deploy/django.yml
+++ b/deploy/django.yml
@ -0,0 +1,98 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postamates-django
+  namespace: spatial
+  labels:
+    app.kubernetes.io/name: postamates-django
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: postamates-django
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: postamates-django
+    spec:
+      containers:
+      - name: django
+        image: DEPLOY_IMAGE_TAG
+        command: ["sh", "-c", "python manage.py migrate && python manage.py collectstatic --noinput && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"]
+        ports:
+          - containerPort: 8000
+        envFrom:
+          - configMapRef:
+              name: postamates-configmap
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "300m"
+          limits:
+            memory: "512Mi"
+            cpu: "300m"
+        # readinessProbe:
+        #   httpGet:
+        #     path: /healthz
+        #     port: 3000
+        #   initialDelaySeconds: 3
+        #   periodSeconds: 3
+        # livenessProbe:
+        #   httpGet:
+        #     path: /healthz
+        #     port: 3000
+        #   initialDelaySeconds: 3
+        #   periodSeconds: 3
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postamates-django
+spec:
+  selector:
+    app.kubernetes.io/name: postamates-django
+  ports:
+    - protocol: TCP
+      port: 8000
+      targetPort: 8000
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: postamates-django
+  namespace: spatial
+spec:
+  ingressClassName: nginx-internal
+  rules:
+  - host: ADDRESS_INGRESS_HOST
+    http:
+      paths:
+      - backend:
+          service:
+            name: postamates-django
+            port:
+              number: 8000
+        path: /api/
+        pathType: ImplementationSpecific
+      - backend:
+          service:
+            name: postamates-django
+            port:
+              number: 8000
+        path: /admin/
+        pathType: ImplementationSpecific
+      - backend:
+          service:
+            name: postamates-django
+            port:
+              number: 8000
+        path: /accounts/
+        pathType: ImplementationSpecific
+      - backend:
+          service:
+            name: postamates-django
+            port:
+              number: 8000
+        path: /django_static/
+        pathType: ImplementationSpecific
--- a/deploy/martin.yml
+++ b/deploy/martin.yml
@ -0,0 +1,64 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: martin
+  namespace: spatial
+  labels:
+    app.kubernetes.io/name: martin
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: martin
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: martin
+    spec:
+      containers:
+      - name: martin
+        image: urbica/martin:pr-368
+        ports:
+        - containerPort: 3000
+        # ------------------------
+        env:
+        - name: DANGER_ACCEPT_INVALID_CERTS
+          value: "true"
+        - name: WATCH_MODE
+          value: "true"
+        - name: DATABASE_URL
+          value: "postgres://martin-user:PSQL_PASSWORD@PSQL_HOST:6432/martin_db"
+        # ------------------------
+        resources:
+          requests:
+            memory: "64Mi"
+            cpu: "50m"
+          limits:
+            memory: "64Mi"
+            cpu: "50m"
+        # ------------------------
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 3000
+          initialDelaySeconds: 3
+          periodSeconds: 3
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 3000
+          initialDelaySeconds: 3
+          periodSeconds: 3
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: martin
+spec:
+  selector:
+    app.kubernetes.io/name: martin
+  ports:
+    - protocol: TCP
+      port: 3000
+      targetPort: 3000
--- a/deploy/worker.yml
+++ b/deploy/worker.yml
@ -0,0 +1,32 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postamates-worker
+  namespace: spatial
+  labels:
+    app.kubernetes.io/name: postamates-worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: postamates-worker
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: postamates-worker
+    spec:
+      containers:
+      - name: worker
+        image: DEPLOY_IMAGE_TAG
+        command: ["sh", "-c", "celery -A postamates.celery:app worker"]
+        envFrom:
+          - configMapRef:
+              name: postamates-configmap
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "50m"
+          limits:
+            memory: "512Mi"
+            cpu: "50m"
--- a/postamates/settings.py
+++ b/postamates/settings.py
@ -178,7 +178,8 @@ SWAGGER_SETTINGS = {
 SRID = 4326

 # celery config
-CELERY_BROKER_URL = 'amqp://loyalty-rabbit'
+# CELERY_BROKER_URL = 'amqp://loyalty-rabbit'
+CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL')
 CELERY_NAMESPACE = 'CELERY'
 PROJECT_NAME = 'postamates'
 CACHE_TIMEOUT = 0
--- a/postamates/urls.py
+++ b/postamates/urls.py
@ -12,3 +12,4 @@ urlpatterns = [
 ]

 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)