From a45cd6ebb283965fae25b3ce5c179e51fefbed24 Mon Sep 17 00:00:00 2001 From: Igor Egrorov Date: Fri, 5 May 2023 09:36:56 +0000 Subject: [PATCH] Feature/cicd --- .dockerignore | 7 +- .gitlab-ci.yml | 197 ++++++++++++++++++++++++++--------------- Dockerfile | 12 +-- deploy/beat.yml | 32 +++++++ deploy/django.yml | 98 ++++++++++++++++++++ deploy/martin.yml | 64 +++++++++++++ deploy/worker.yml | 32 +++++++ postamates/settings.py | 3 +- postamates/urls.py | 1 + 9 files changed, 365 insertions(+), 81 deletions(-) create mode 100644 deploy/beat.yml create mode 100644 deploy/django.yml create mode 100644 deploy/martin.yml create mode 100644 deploy/worker.yml diff --git a/.dockerignore b/.dockerignore index c63ded4..e09277c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,10 @@ .gitignore -.gitlab/* .gitlab-ci.yml README.md -.git/* .pre-commit-config.yaml .flake8 +.env.like +.git +.gitlab +docker-compose.* +deploy diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index be090e4..eaddc9e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,86 +1,137 @@ +--- +variables: + DOCKER_IMAGE_TAG: ${YC_CONTAINER_REGISTRY}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_SLUG}.${CI_COMMIT_SHORT_SHA} + RELEASE_DOCKER_IMAGE_TAG: ${YC_CONTAINER_REGISTRY}/${CI_PROJECT_NAME}:${CI_COMMIT_TAG} + DEPLOY_KUBER_NAMESPACE: spatial + stages: - build - - lint + # - lint - deploy -variables: - CONTAINERS_NAME: sst_postamates - IMAGE_NAME: $CI_REGISTRY_IMAGE - -default: +build-docker: + stage: build tags: - - deploy-remote + - shell + script: + - > + docker build + --build-arg YC_CONTAINER_REGISTRY=${YC_CONTAINER_REGISTRY} + -t ${DOCKER_IMAGE_TAG} . + - docker push ${DOCKER_IMAGE_TAG} -.use_cached_requirements: &use_cached_requirements +deploy_dev_kuber: + extends: .deploy_base_kuber variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - cache: - key: $CI_COMMIT_REF_SLUG - paths: - - .cache/pip - policy: pull-push - -build-job: - stage: build - before_script: - - sudo docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY - - ls - script: - - sudo docker pull $IMAGE_NAME:latest || true - - sudo docker build --cache-from $IMAGE_NAME:latest --tag $IMAGE_NAME:$CI_COMMIT_SHORT_SHA --tag $IMAGE_NAME:latest . - - sudo docker push $IMAGE_NAME:$CI_COMMIT_SHORT_SHA - - sudo docker push $IMAGE_NAME:latest - rules: - - if: $CI_COMMIT_BRANCH == 'sst_main' + INGRESS_HOST: "postnet.dev.selftech.ru" + tags: + - docker + environment: + name: dev -.deploy: &deploy-common +.deploy_base_kuber: + image: ${YC_CONTAINER_REGISTRY}/public/helm-kubectl-git:1.0.0 stage: deploy before_script: - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' - - eval $(ssh-agent -s) - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - ssh-keyscan $REMOTE_HOST >> ~/.ssh/known_hosts - - chmod 644 ~/.ssh/known_hosts - - ssh $REMOTE_USER docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY + - KUBE_CONFIG=`echo ${CI_ENVIRONMENT_NAME}_kubeconfig` + - mkdir -p ${HOME}/.kube + - 'cat ${!KUBE_CONFIG} > ${HOME}/.kube/config' + - chmod -R 700 ${HOME}/.kube script: - - cd $CI_PROJECT_DIR - - ssh $REMOTE_USER docker pull $IMAGE_NAME:$CI_COMMIT_SHORT_SHA - - DOCKER_HOST="ssh://$REMOTE_USER" docker-compose -f $COMPOSE up -d - - DOCKER_HOST="ssh://$REMOTE_USER" docker restart sst_postamates_martin + - sed -i "s|DEPLOY_IMAGE_TAG|${DOCKER_IMAGE_TAG}|g" ./deploy/django.yml + - sed -i "s|ADDRESS_INGRESS_HOST|${INGRESS_HOST}|g" ./deploy/django.yml + - sed -i "s|DEPLOY_IMAGE_TAG|${DOCKER_IMAGE_TAG}|g" ./deploy/worker.yml + - sed -i "s|DEPLOY_IMAGE_TAG|${DOCKER_IMAGE_TAG}|g" ./deploy/beat.yml + - kubectl apply -f ./deploy/django.yml + - kubectl apply -f ./deploy/worker.yml + - kubectl apply -f ./deploy/beat.yml + dependencies: [] + artifacts: + paths: + - ./deploy/django.yml + - ./deploy/worker.yml + - ./deploy/beat.yml + expire_in: 1 week + when: manual -deploy-test-job: - <<: *deploy-common - variables: - REMOTE_HOST: $REMOTE_HOST_TEST - REMOTE_USER: $REMOTE_USER_TEST - COMPOSE: docker-compose.test.yml - rules: - - if: $CI_COMMIT_BRANCH == 'sst_main' +# variables: +# CONTAINERS_NAME: sst_postamates +# IMAGE_NAME: $CI_REGISTRY_IMAGE -pylint: - <<: *use_cached_requirements - stage: lint - before_script: - - pip install flake8-codeclimate - script: - - set -x - - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true) - - > - if [ ! -z "$FLIST" ]; - then - mkdir -p build/reports; - flake8 --format junit-xml $FLIST > build/reports/flake8-junit.xml || flake8 $FLIST; - else - echo "No files to lint" - true; - fi - only: - refs: - - merge_requests - artifacts: - reports: - junit: ./build/reports/flake8-junit.xml - expire_in: 7 days - allow_failure: false +# default: +# tags: +# - deploy-remote + +# .use_cached_requirements: &use_cached_requirements +# variables: +# PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" +# cache: +# key: $CI_COMMIT_REF_SLUG +# paths: +# - .cache/pip +# policy: pull-push + +# build-job: +# stage: build +# before_script: +# - sudo docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY +# - ls +# script: +# - sudo docker pull $IMAGE_NAME:latest || true +# - sudo docker build --cache-from $IMAGE_NAME:latest --tag $IMAGE_NAME:$CI_COMMIT_SHORT_SHA --tag $IMAGE_NAME:latest . +# - sudo docker push $IMAGE_NAME:$CI_COMMIT_SHORT_SHA +# - sudo docker push $IMAGE_NAME:latest +# rules: +# - if: $CI_COMMIT_BRANCH == 'sst_main' + +# .deploy: &deploy-common +# stage: deploy +# before_script: +# - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' +# - eval $(ssh-agent -s) +# - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - +# - mkdir -p ~/.ssh +# - chmod 700 ~/.ssh +# - ssh-keyscan $REMOTE_HOST >> ~/.ssh/known_hosts +# - chmod 644 ~/.ssh/known_hosts +# - ssh $REMOTE_USER docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY +# script: +# - cd $CI_PROJECT_DIR +# - ssh $REMOTE_USER docker pull $IMAGE_NAME:$CI_COMMIT_SHORT_SHA +# - DOCKER_HOST="ssh://$REMOTE_USER" docker-compose -f $COMPOSE up -d +# - DOCKER_HOST="ssh://$REMOTE_USER" docker restart sst_postamates_martin + +# deploy-test-job: +# <<: *deploy-common +# variables: +# REMOTE_HOST: $REMOTE_HOST_TEST +# REMOTE_USER: $REMOTE_USER_TEST +# COMPOSE: docker-compose.test.yml +# rules: +# - if: $CI_COMMIT_BRANCH == 'sst_main' + +# pylint: +# <<: *use_cached_requirements +# stage: lint +# before_script: +# - pip install flake8-codeclimate +# script: +# - set -x +# - FLIST=$(git --no-pager diff --diff-filter=d --name-only $(git merge-base --fork-point origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) $CI_COMMIT_SHA | egrep ".py$" | grep -v ".docker/"|| true) +# - > +# if [ ! -z "$FLIST" ]; +# then +# mkdir -p build/reports; +# flake8 --format junit-xml $FLIST > build/reports/flake8-junit.xml || flake8 $FLIST; +# else +# echo "No files to lint" +# true; +# fi +# only: +# refs: +# - merge_requests +# artifacts: +# reports: +# junit: ./build/reports/flake8-junit.xml +# expire_in: 7 days +# allow_failure: false diff --git a/Dockerfile b/Dockerfile index eca9ea2..93c761b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,18 @@ -FROM python:3.8 +ARG YC_CONTAINER_REGISTRY +FROM ${YC_CONTAINER_REGISTRY}/public/python:3.8 -RUN apt-get update &&\ +RUN apt-get update && \ apt-get install -y binutils libproj-dev gdal-bin ENV PYTHONUNBUFFERED 1 -RUN mkdir /code - WORKDIR /code COPY requirements.txt /code/ -RUN pip install -r requirements.txt +RUN pip install -r /code/requirements.txt + +# RUN apt purge binutils libproj-dev gdal-bin -y && \ +# apt autoremove -y COPY . /code/ diff --git a/deploy/beat.yml b/deploy/beat.yml new file mode 100644 index 0000000..16f46d4 --- /dev/null +++ b/deploy/beat.yml @@ -0,0 +1,32 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postamates-beat + namespace: spatial + labels: + app.kubernetes.io/name: postamates-beat +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postamates-beat + template: + metadata: + labels: + app.kubernetes.io/name: postamates-beat + spec: + containers: + - name: beat + image: DEPLOY_IMAGE_TAG + command: ["sh", "-c", "celery -A postamates beat -l info --scheduler beat_celery_beat.schedulers:DatabaseScheduler"] + envFrom: + - configMapRef: + name: postamates-configmap + resources: + requests: + memory: "256Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "50m" diff --git a/deploy/django.yml b/deploy/django.yml new file mode 100644 index 0000000..0a3b605 --- /dev/null +++ b/deploy/django.yml @@ -0,0 +1,98 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postamates-django + namespace: spatial + labels: + app.kubernetes.io/name: postamates-django +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postamates-django + template: + metadata: + labels: + app.kubernetes.io/name: postamates-django + spec: + containers: + - name: django + image: DEPLOY_IMAGE_TAG + command: ["sh", "-c", "python manage.py migrate && python manage.py collectstatic --noinput && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"] + ports: + - containerPort: 8000 + envFrom: + - configMapRef: + name: postamates-configmap + resources: + requests: + memory: "512Mi" + cpu: "300m" + limits: + memory: "512Mi" + cpu: "300m" + # readinessProbe: + # httpGet: + # path: /healthz + # port: 3000 + # initialDelaySeconds: 3 + # periodSeconds: 3 + # livenessProbe: + # httpGet: + # path: /healthz + # port: 3000 + # initialDelaySeconds: 3 + # periodSeconds: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: postamates-django +spec: + selector: + app.kubernetes.io/name: postamates-django + ports: + - protocol: TCP + port: 8000 + targetPort: 8000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: postamates-django + namespace: spatial +spec: + ingressClassName: nginx-internal + rules: + - host: ADDRESS_INGRESS_HOST + http: + paths: + - backend: + service: + name: postamates-django + port: + number: 8000 + path: /api/ + pathType: ImplementationSpecific + - backend: + service: + name: postamates-django + port: + number: 8000 + path: /admin/ + pathType: ImplementationSpecific + - backend: + service: + name: postamates-django + port: + number: 8000 + path: /accounts/ + pathType: ImplementationSpecific + - backend: + service: + name: postamates-django + port: + number: 8000 + path: /django_static/ + pathType: ImplementationSpecific diff --git a/deploy/martin.yml b/deploy/martin.yml new file mode 100644 index 0000000..b504a5c --- /dev/null +++ b/deploy/martin.yml @@ -0,0 +1,64 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: martin + namespace: spatial + labels: + app.kubernetes.io/name: martin +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: martin + template: + metadata: + labels: + app.kubernetes.io/name: martin + spec: + containers: + - name: martin + image: urbica/martin:pr-368 + ports: + - containerPort: 3000 + # ------------------------ + env: + - name: DANGER_ACCEPT_INVALID_CERTS + value: "true" + - name: WATCH_MODE + value: "true" + - name: DATABASE_URL + value: "postgres://martin-user:PSQL_PASSWORD@PSQL_HOST:6432/martin_db" + # ------------------------ + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "64Mi" + cpu: "50m" + # ------------------------ + readinessProbe: + httpGet: + path: /healthz + port: 3000 + initialDelaySeconds: 3 + periodSeconds: 3 + livenessProbe: + httpGet: + path: /healthz + port: 3000 + initialDelaySeconds: 3 + periodSeconds: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: martin +spec: + selector: + app.kubernetes.io/name: martin + ports: + - protocol: TCP + port: 3000 + targetPort: 3000 diff --git a/deploy/worker.yml b/deploy/worker.yml new file mode 100644 index 0000000..c6e1b7d --- /dev/null +++ b/deploy/worker.yml @@ -0,0 +1,32 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postamates-worker + namespace: spatial + labels: + app.kubernetes.io/name: postamates-worker +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: postamates-worker + template: + metadata: + labels: + app.kubernetes.io/name: postamates-worker + spec: + containers: + - name: worker + image: DEPLOY_IMAGE_TAG + command: ["sh", "-c", "celery -A postamates.celery:app worker"] + envFrom: + - configMapRef: + name: postamates-configmap + resources: + requests: + memory: "512Mi" + cpu: "50m" + limits: + memory: "512Mi" + cpu: "50m" diff --git a/postamates/settings.py b/postamates/settings.py index 01b917b..9909a59 100644 --- a/postamates/settings.py +++ b/postamates/settings.py @@ -178,7 +178,8 @@ SWAGGER_SETTINGS = { SRID = 4326 # celery config -CELERY_BROKER_URL = 'amqp://loyalty-rabbit' +# CELERY_BROKER_URL = 'amqp://loyalty-rabbit' +CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL') CELERY_NAMESPACE = 'CELERY' PROJECT_NAME = 'postamates' CACHE_TIMEOUT = 0 diff --git a/postamates/urls.py b/postamates/urls.py index 041f217..e862f63 100644 --- a/postamates/urls.py +++ b/postamates/urls.py @@ -12,3 +12,4 @@ urlpatterns = [ ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)