diff --git a/.env.like b/.env.like index b75a0e8..226d07b 100644 --- a/.env.like +++ b/.env.like @@ -19,3 +19,4 @@ MARTIN_PORT=3000 # Host domain name (maps in docker-compose to REACT_APP_DOMAIN_URL) DOMAIN=postnet-dev.selftech.ru +GEOCODER_API_KEY = api_key_here \ No newline at end of file diff --git a/.gitignore b/.gitignore index f5acaed..474d4cb 100644 --- a/.gitignore +++ b/.gitignore @@ -134,5 +134,4 @@ nginx/nginx.conf.prod docker-compose.dev.yml pg_dumps/ django_static/ -django_media/ dit_frontend/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1bff38c..b1a19cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,14 +35,29 @@ build-static-image: -t ${DOCKER_IMAGE_TAG}-static . - docker push ${DOCKER_IMAGE_TAG}-static +auto-deploy-dev-kuber: + extends: .deploy_base_kuber + variables: + INGRESS_HOST: "postnet.dev.selftech.ru" + tags: + - docker + only: + refs: + - dev + environment: + name: dev + deploy-dev-kuber: extends: .deploy_base_kuber variables: INGRESS_HOST: "postnet.dev.selftech.ru" tags: - docker + except: + - dev environment: name: dev + when: manual deploy-prod-kuber: extends: .deploy_base_kuber @@ -52,6 +67,21 @@ deploy-prod-kuber: - docker-prod environment: name: prod + when: manual + +dev_restart_martin: + extends: .restart_martin_base + tags: + - docker + environment: + name: dev + +prod_restart_martin: + extends: .restart_martin_base + tags: + - docker-prod + environment: + name: prod .deploy_base_kuber: image: ${YC_CONTAINER_REGISTRY}/public/helm-kubectl-git:1.0.0 @@ -80,6 +110,17 @@ deploy-prod-kuber: - ./deploy/beat.yml - ./deploy/django-static.yml expire_in: 1 week + +.restart_martin_base: + image: ${YC_CONTAINER_REGISTRY}/public/helm-kubectl-git:1.0.0 + stage: .pre + before_script: + - KUBE_CONFIG=`echo ${CI_ENVIRONMENT_NAME}_kubeconfig` + - mkdir -p ${HOME}/.kube + - 'cat ${!KUBE_CONFIG} > ${HOME}/.kube/config' + - chmod -R 700 ${HOME}/.kube + script: + - kubectl rollout restart deployment martin -n ${DEPLOY_KUBER_NAMESPACE} when: manual # variables: diff --git a/PrePlacementPoints.xlsx b/PrePlacementPoints.xlsx new file mode 100644 index 0000000..a508bb3 Binary files /dev/null and b/PrePlacementPoints.xlsx differ diff --git a/README.md b/README.md index 2bc10b5..ce74ad2 100644 --- a/README.md +++ b/README.md @@ -104,4 +104,7 @@ server { listen [::]:80 ; return 404; } + + + ``` diff --git a/deploy/django-static.yml b/deploy/django-static.yml index 0a54c6d..c698808 100644 --- a/deploy/django-static.yml +++ b/deploy/django-static.yml @@ -40,6 +40,13 @@ spec: port: 8888 initialDelaySeconds: 3 periodSeconds: 3 + volumeMounts: + - mountPath: "/usr/share/nginx/html/django_media" + name: django-nginx-data + volumes: + - name: django-nginx-data + persistentVolumeClaim: + claimName: django-nginx-pvc --- apiVersion: v1 kind: Service @@ -71,3 +78,10 @@ spec: number: 80 path: /django_static/ pathType: ImplementationSpecific + - backend: + service: + name: django-static + port: + number: 80 + path: /django_media/ + pathType: ImplementationSpecific diff --git a/deploy/django.yml b/deploy/django.yml index 9e8b21e..1b60a89 100644 --- a/deploy/django.yml +++ b/deploy/django.yml @@ -19,7 +19,7 @@ spec: containers: - name: django image: DEPLOY_IMAGE_TAG - command: ["sh", "-c", "python manage.py migrate && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"] + command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"] ports: - containerPort: 8000 name: django-port @@ -28,23 +28,30 @@ spec: name: postamates-configmap resources: requests: - memory: "1500Mi" - cpu: "300m" + memory: "3000Mi" + cpu: "500m" limits: - memory: "1500Mi" - cpu: "300m" - readinessProbe: - httpGet: - path: /api/ao_rayons - port: django-port - initialDelaySeconds: 3 - periodSeconds: 3 - livenessProbe: - httpGet: - path: /api/ao_rayons - port: django-port - initialDelaySeconds: 3 - periodSeconds: 3 + memory: "3000Mi" + cpu: "500m" + # readinessProbe: + # httpGet: + # path: /api/ao_rayons + # port: django-port + # initialDelaySeconds: 3 + # periodSeconds: 3 + # livenessProbe: + # httpGet: + # path: /api/ao_rayons + # port: django-port + # initialDelaySeconds: 3 + # periodSeconds: 3 + volumeMounts: + - mountPath: "/code/django_media" + name: django-nginx-data + volumes: + - name: django-nginx-data + persistentVolumeClaim: + claimName: django-nginx-pvc --- apiVersion: v1 kind: Service @@ -89,4 +96,4 @@ spec: port: number: 8000 path: /accounts/ - pathType: ImplementationSpecific + pathType: ImplementationSpecific \ No newline at end of file diff --git a/deploy/dockerfiles/Dockerfile b/deploy/dockerfiles/Dockerfile index 95d5f00..01aa35a 100644 --- a/deploy/dockerfiles/Dockerfile +++ b/deploy/dockerfiles/Dockerfile @@ -2,7 +2,8 @@ ARG YC_CONTAINER_REGISTRY FROM ${YC_CONTAINER_REGISTRY}/public/python:3.8 RUN apt-get update && \ - apt-get install -y binutils libproj-dev gdal-bin + apt-get install -y binutils libproj-dev gdal-bin && \ + apt-get install -y postgresql-client ENV PYTHONUNBUFFERED 1 diff --git a/deploy/worker.yml b/deploy/worker.yml index 4321b90..8fbf118 100644 --- a/deploy/worker.yml +++ b/deploy/worker.yml @@ -19,14 +19,14 @@ spec: containers: - name: worker image: DEPLOY_IMAGE_TAG - command: ["sh", "-c", "celery -A postamates.celery:app worker"] + command: ["sh", "-c", "celery -A postamates.celery:app worker", "-l", "info", "--concurrency=1"] envFrom: - configMapRef: name: postamates-configmap resources: requests: - memory: "1500Mi" - cpu: "300m" + memory: "3000Mi" + cpu: "500m" limits: - memory: "1500Mi" - cpu: "300m" + memory: "3000Mi" + cpu: "500m" diff --git a/django_media/post_and_pvz_group_images/namekhalva_typepostamat_SizeM.png b/django_media/post_and_pvz_group_images/namekhalva_typepostamat_SizeM.png new file mode 100644 index 0000000..c413a9f Binary files /dev/null and b/django_media/post_and_pvz_group_images/namekhalva_typepostamat_SizeM.png differ diff --git a/django_media/post_and_pvz_group_images/nameozon_typePVZ_SizeM.png b/django_media/post_and_pvz_group_images/nameozon_typePVZ_SizeM.png new file mode 100644 index 0000000..a2e154a Binary files /dev/null and b/django_media/post_and_pvz_group_images/nameozon_typePVZ_SizeM.png differ diff --git a/django_media/post_and_pvz_group_images/nameozon_typepostamat_SizeM.png b/django_media/post_and_pvz_group_images/nameozon_typepostamat_SizeM.png new file mode 100644 index 0000000..95cd3fe Binary files /dev/null and b/django_media/post_and_pvz_group_images/nameozon_typepostamat_SizeM.png differ diff --git a/django_media/post_and_pvz_group_images/namesdek_typePVZ_SizeM.png b/django_media/post_and_pvz_group_images/namesdek_typePVZ_SizeM.png new file mode 100644 index 0000000..cec6d6b Binary files /dev/null and b/django_media/post_and_pvz_group_images/namesdek_typePVZ_SizeM.png differ diff --git a/django_media/post_and_pvz_group_images/namesdek_typepostamat_SizeM.png b/django_media/post_and_pvz_group_images/namesdek_typepostamat_SizeM.png new file mode 100644 index 0000000..e9a4d69 Binary files /dev/null and b/django_media/post_and_pvz_group_images/namesdek_typepostamat_SizeM.png differ diff --git a/django_media/post_and_pvz_group_images/namewb_typePVZ_SizeM.png b/django_media/post_and_pvz_group_images/namewb_typePVZ_SizeM.png new file mode 100644 index 0000000..85e5c53 Binary files /dev/null and b/django_media/post_and_pvz_group_images/namewb_typePVZ_SizeM.png differ diff --git a/django_media/post_and_pvz_group_images/pochta-PVZ-M.png b/django_media/post_and_pvz_group_images/pochta-PVZ-M.png new file mode 100644 index 0000000..2b76e27 Binary files /dev/null and b/django_media/post_and_pvz_group_images/pochta-PVZ-M.png differ diff --git a/django_media/post_and_pvz_group_images/pochta-postamat-M_gBJlHTO.png b/django_media/post_and_pvz_group_images/pochta-postamat-M_gBJlHTO.png new file mode 100644 index 0000000..0d7ffec Binary files /dev/null and b/django_media/post_and_pvz_group_images/pochta-postamat-M_gBJlHTO.png differ diff --git a/django_media/post_and_pvz_group_images/ya.market-PVZ-M_KWksDvo.png b/django_media/post_and_pvz_group_images/ya.market-PVZ-M_KWksDvo.png new file mode 100644 index 0000000..8793c6e Binary files /dev/null and b/django_media/post_and_pvz_group_images/ya.market-PVZ-M_KWksDvo.png differ diff --git a/django_media/post_and_pvz_group_images/ya.market-postamat-M.png b/django_media/post_and_pvz_group_images/ya.market-postamat-M.png new file mode 100644 index 0000000..ab84c1a Binary files /dev/null and b/django_media/post_and_pvz_group_images/ya.market-postamat-M.png differ diff --git a/docker-compose.yml b/docker-compose.yml index b1b42b3..3d08407 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,10 @@ services: sh -c "python manage.py migrate && python manage.py collectstatic --noinput && python manage.py loaddata fixtures/groups.json && + python manage.py loaddata fixtures/post_and_pvz.json && + python manage.py loaddata fixtures/post_and_pvz_groups.json && + python manage.py loaddata fixtures/otherobjectscategorys.json && + python manage.py loaddata fixtures/otherobjectsgroups.json && python manage.py runserver 0.0.0.0:${DJANGO_PORT}" environment: <<: *postgres-variables diff --git a/fixtures/groups.json b/fixtures/groups.json index 48ec9c4..44a2d36 100644 --- a/fixtures/groups.json +++ b/fixtures/groups.json @@ -1 +1 @@ -[{"model": "auth.group", "pk": 1, "fields": {"name": "Администратор пользователей", "permissions": [13, 14, 15, 16]}}, {"model": "auth.group", "pk": 2, "fields": {"name": "Зритель", "permissions": [28]}}, {"model": "auth.group", "pk": 3, "fields": {"name": "Редактор", "permissions": [25, 26, 27, 28]}}] \ No newline at end of file +[{"model": "auth.group", "pk": 1, "fields": {"name": "Зритель", "permissions": [["view_placementpoint", "service", "placementpoint"]]}}, {"model": "auth.group", "pk": 2, "fields": {"name": "Редактор", "permissions": [["add_placementpoint", "service", "placementpoint"], ["change_placementpoint", "service", "placementpoint"], ["delete_placementpoint", "service", "placementpoint"], ["view_placementpoint", "service", "placementpoint"]]}}, {"model": "auth.group", "pk": 3, "fields": {"name": "Администратор пользователей", "permissions": [["add_user", "auth", "user"], ["change_user", "auth", "user"], ["delete_user", "auth", "user"], ["view_user", "auth", "user"]]}}] \ No newline at end of file diff --git a/fixtures/otherobjectscategorys.json b/fixtures/otherobjectscategorys.json new file mode 100644 index 0000000..6e6c9ac --- /dev/null +++ b/fixtures/otherobjectscategorys.json @@ -0,0 +1,186 @@ +[ + { + "model": "service.otherobjectscategory", + "pk": 4, + "fields": { + "name": "business_activity", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 5, + "fields": { + "name": "metro_stations", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 22, + "fields": { + "name": "bargains", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 23, + "fields": { + "name": "BC", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 24, + "fields": { + "name": "flats_cnt", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 25, + "fields": { + "name": "offers_estate", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 26, + "fields": { + "name": "schools", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 27, + "fields": { + "name": "kindergar", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 28, + "fields": { + "name": "stops", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 29, + "fields": { + "name": "pharmacies", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 30, + "fields": { + "name": "sport_centers", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 31, + "fields": { + "name": "supermarkets", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 32, + "fields": { + "name": "supermarkets_premium", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 33, + "fields": { + "name": "banks", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 34, + "fields": { + "name": "recas", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 35, + "fields": { + "name": "labs", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 36, + "fields": { + "name": "clinics", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 37, + "fields": { + "name": "attractions", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 38, + "fields": { + "name": "cultures", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 39, + "fields": { + "name": "public_services", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 40, + "fields": { + "name": "popul_home_job", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 41, + "fields": { + "name": "TC", + "visible": false + } + }, + { + "model": "service.otherobjectscategory", + "pk": 42, + "fields": { + "name": "yndx_food_cnt_amt", + "visible": false + } + } +] \ No newline at end of file diff --git a/fixtures/otherobjectsgroups.json b/fixtures/otherobjectsgroups.json new file mode 100644 index 0000000..af81a58 --- /dev/null +++ b/fixtures/otherobjectsgroups.json @@ -0,0 +1,23 @@ +[{"model": "service.otherobjectsgroup", "pk": 4, "fields": {"name": "business_activity", "category": 4, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 5, "fields": {"name": "metro_stations", "category": 5, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 23, "fields": {"name": "bargains", "category": 22, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 24, "fields": {"name": "BC", "category": 23, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 25, "fields": {"name": "flats_cnt", "category": 24, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 26, "fields": {"name": "offers_estate", "category": 25, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 27, "fields": {"name": "schools", "category": 26, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 28, "fields": {"name": "kindergar", "category": 27, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 29, "fields": {"name": "stops", "category": 28, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 30, "fields": {"name": "pharmacies", "category": 29, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 31, "fields": {"name": "sport_centers", "category": 30, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 32, "fields": {"name": "supermarkets", "category": 31, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 33, "fields": {"name": "supermarkets_premium", "category": 32, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 34, "fields": {"name": "banks", "category": 33, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 35, "fields": {"name": "recas", "category": 34, "image": "", "visible": false}}, { + "model": "service.otherobjectsgroup", "pk": 36, "fields": {"name": "labs", "category": 35, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 37, "fields": {"name": "clinics", "category": 36, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 38, "fields": {"name": "attractions", "category": 37, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 39, "fields": {"name": "cultures", "category": 38, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 40, "fields": {"name": "public_services", "category": 39, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 41, "fields": {"name": "popul_home_job", "category": 40, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 42, "fields": {"name": "TC", "category": 41, "image": "", "visible": false}}, + {"model": "service.otherobjectsgroup", "pk": 43, "fields": {"name": "yndx_food_cnt_amt", "category": 42, "image": "", "visible": false}}] \ No newline at end of file diff --git a/fixtures/post_and_pvz.json b/fixtures/post_and_pvz.json new file mode 100644 index 0000000..2bbd580 --- /dev/null +++ b/fixtures/post_and_pvz.json @@ -0,0 +1,20 @@ +[ + { + "model": "service.post_and_pvzcategory", + "pk": 1, + "fields": { + "name": "ПВЗ", + "visible": true, + "include_in_ml": true + } + }, + { + "model": "service.post_and_pvzcategory", + "pk": 2, + "fields": { + "name": "Постаматы прочих сетей", + "visible": true, + "include_in_ml": true + } + } +] diff --git a/fixtures/post_pvz_groups.json b/fixtures/post_pvz_groups.json new file mode 100644 index 0000000..f57ea6a --- /dev/null +++ b/fixtures/post_pvz_groups.json @@ -0,0 +1,112 @@ +[ + { + "model":"service.post_and_pvzgroup", + "pk":7, + "fields":{ + "name":"Ozon", + "category":1, + "image":"post_and_pvz_group_images/nameozon_typePVZ_SizeM.png", + "visible":true, + "include_in_ml":true + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":8, + "fields":{ + "name":"СДЭК", + "category":1, + "image":"post_and_pvz_group_images/namesdek_typePVZ_SizeM.png", + "visible":true, + "include_in_ml":true + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":9, + "fields":{ + "name":"Халва", + "category":2, + "image":"post_and_pvz_group_images/namekhalva_typepostamat_SizeM.png", + "visible":true, + "include_in_ml":false + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":10, + "fields":{ + "name":"СДЭК", + "category":2, + "image":"post_and_pvz_group_images/namesdek_typepostamat_SizeM.png", + "visible":true, + "include_in_ml":false + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":12, + "fields":{ + "name":"Ozon", + "category":2, + "image":"post_and_pvz_group_images/nameozon_typepostamat_SizeM.png", + "visible":true, + "include_in_ml":false + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":17, + "fields":{ + "name":"WildBerries", + "category":1, + "image":"post_and_pvz_group_images/namewb_typePVZ_SizeM.png", + "visible":true, + "include_in_ml":false + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":19, + "fields":{ + "name":"Яндекс.Маркет", + "category":1, + "image":"post_and_pvz_group_images/ya.market-PVZ-M_KWksDvo.png", + "visible":true, + "include_in_ml":true + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":20, + "fields":{ + "name":"Яндекс.Маркет", + "category":2, + "image":"post_and_pvz_group_images/ya.market-postamat-M.png", + "visible":true, + "include_in_ml":true + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":24, + "fields":{ + "name":"Почта России", + "category":1, + "image":"post_and_pvz_group_images/pochta-PVZ-M.png", + "visible":true, + "include_in_ml":true + } + }, + { + "model":"service.post_and_pvzgroup", + "pk":25, + "fields":{ + "name":"Почта России", + "category":2, + "image":"post_and_pvz_group_images/pochta-postamat-M_gBJlHTO.png", + "visible":true, + "include_in_ml":true + } + } +] \ No newline at end of file diff --git a/postamates/settings.py b/postamates/settings.py index 79695fb..1a878c2 100644 --- a/postamates/settings.py +++ b/postamates/settings.py @@ -22,7 +22,9 @@ BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = 'django-insecure-5czma@e7b(e4v+c*@bkknj(*em%@x52jizednhy6lye)_@ox4@' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = os.getenv('DEBUG', False) == 'True' +# TODO: change DEBUG to +# DEBUG = os.getenv('DEBUG', False) == 'True' +DEBUG = True ALLOWED_HOSTS = ['*'] @@ -133,7 +135,7 @@ MEDIA_ROOT = os.path.join(BASE_DIR, 'django_media') DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -CORS_ORIGIN_ALLOW_ALL = True # If this is used then `CORS_ORIGIN_WHITELIST` will not have any effect +CORS_ALLOW_ALL_ORIGINS = True # If this is used then `CORS_ORIGIN_WHITELIST` will not have any effect CORS_ALLOW_CREDENTIALS = True CORS_ORIGIN_ALLOW = True @@ -174,7 +176,6 @@ SWAGGER_SETTINGS = { 'SWAGGER_PATH': 'django_static/swagger/swagger.yaml', } - SRID = 4326 # celery config @@ -182,9 +183,17 @@ SRID = 4326 CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL') CELERY_NAMESPACE = 'CELERY' PROJECT_NAME = 'postamates' -CACHE_TIMEOUT = 0 +DOMAIN = os.getenv('DOMAIN', 'localhost') + +if DEBUG: + CACHE_TIMEOUT = 60 +else: + CACHE_TIMEOUT = 60 * 60 * 24 * 7 DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS = 500 AGE_DAY_LIMIT = 270 AGE_DAY_BORDER = 30 EXCEL_EXPORT_FILENAME = 'placement_points.xlsx' JSON_EXPORT_FILENAME = 'placement_points.json' +DATA_UPLOAD_MAX_NUMBER_FIELDS = None +GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M') +STATUS_TASK_NAME='status_task' diff --git a/requirements.txt b/requirements.txt index d27fb41..d65d26b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -90,3 +90,5 @@ virtualenv==20.20.0 wcwidth==0.2.6 xlrd==1.2.0 XlsxWriter==3.0.8 +django-filter==23.2 +shap==0.41.0 diff --git a/service/admin.py b/service/admin.py index c3a0cfc..0173cba 100644 --- a/service/admin.py +++ b/service/admin.py @@ -1,14 +1,22 @@ +import os + from django.contrib import admin from django.contrib.admin import AdminSite from django.contrib.admin.sites import NotRegistered from django.contrib.auth.admin import UserAdmin from django.contrib.auth.models import User +from service.layer_service import LayerService from service.models import AO from service.models import PlacementPoint -from service.models import PointDist from service.models import Rayon -from service.models import Rivals +from service.models import PrePlacementPoint, Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, \ + OtherObjectsGroup, \ + OtherObjectsCategory, PrePlacementPointPVZDistance, TempFiles, RaschetGroups, House, RaschetObjects +from service.models import PlacementPointPVZDistance, TaskStatus +from postamates.settings import DEBUG +from django.core.cache import cache +from service.utils import run_psql_command class MyAdminSite(AdminSite): @@ -19,18 +27,140 @@ class MyAdminSite(AdminSite): return super(MyAdminSite, self).index(request, extra_context) -my_admin_site = MyAdminSite(name='myadmin') +my_admin_site = MyAdminSite(name='POSTNET') +if DEBUG: + my_admin_site.register(AO) + my_admin_site.register(Rayon) + my_admin_site.register(TempFiles) + my_admin_site.register(PlacementPointPVZDistance) + + +class Post_and_PVZAdmin(admin.ModelAdmin): + def save_model(self, request, obj, form, change): + obj.save() + LayerService().count_post_pvz_for_placementpoint(obj) + RaschetGroups.objects.create(obj_id=obj.group.id) + + def delete_model(self, request, obj): + RaschetObjects.objects.filter(obj_id=obj.id).delete() + super().delete_model(request, obj) + + +my_admin_site.register(Post_and_pvz, Post_and_PVZAdmin) +my_admin_site.register(OtherObjects) +my_admin_site.register(House) +my_admin_site.register(PrePlacementPoint) +my_admin_site.register(PrePlacementPointPVZDistance) + + +class TaskStatusAdmin(admin.ModelAdmin): + list_display = ('task_name', 'status') + + +class CategoryAdmin(admin.ModelAdmin): + list_display = ('name', 'visible') + + def save_model(self, request, obj, form, change): + obj.save() + cache.clear() + + def delete_model(self, request, obj): + obj.delete() + cache.clear() + + +class PostPvzCategoryAdmin(CategoryAdmin): + def get_readonly_fields(self, request, obj=None): + if obj.id in (1, 2): + return ['id', 'name'] + else: + return super().get_readonly_fields(request, obj) + + def save_model(self, request, obj, form, change): + obj.save() + run_psql_command() + if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: + LayerService.update_categories(obj) + cache.clear() + + def delete_model(self, request, obj): + if obj.id in (1, 2): + pass + else: + super().delete_model(request, obj) + + +class GroupAdmin(admin.ModelAdmin): + list_display = ('name', 'category', 'visible') + + def save_model(self, request, obj, form, change): + obj.save() + cache.clear() + + def delete_model(self, request, obj): + obj.delete() + cache.clear() + + +class PostPvzGroupAdmin(GroupAdmin): + def get_readonly_fields(self, request, obj=None): + if obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25): + return ['id', 'name'] + else: + return super().get_readonly_fields(request, obj) + + def save_model(self, request, obj, form, change): + obj.save() + run_psql_command() + if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: + LayerService.update_groups(obj) + cache.clear() + + def delete_model(self, request, obj): + if obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25): + pass + else: + RaschetGroups.objects.filter(obj_id=obj.id).delete() + super().delete_model(request, obj) + + +class OtherObjectsGroupAdmin(GroupAdmin): + def get_readonly_fields(self, request, obj=None): + if obj.id in (4, 5) or obj.id in list(range(22, 43)): + return ['id', 'name'] + else: + return super().get_readonly_fields(request, obj) + + def delete_model(self, request, obj): + if obj.id in (4, 5) or obj.id in list(range(22, 43)): + pass + else: + super().delete_model(request, obj) + + +class OtherObjectsCategoryAdmin(CategoryAdmin): + def get_readonly_fields(self, request, obj=None): + if obj.id in (4, 5) or obj.id in list(range(22, 43)): + return ['id', 'name'] + else: + return super().get_readonly_fields(request, obj) -my_admin_site.register(AO) -my_admin_site.register(Rayon) -my_admin_site.register(Rivals) -my_admin_site.register(PointDist) + def delete_model(self, request, obj): + if obj.id in (4, 5) or obj.id in list(range(22, 43)): + pass + else: + super().delete_model(request, obj) class PlacementPointAdmin(admin.ModelAdmin): pass +my_admin_site.register(TaskStatus, TaskStatusAdmin) +my_admin_site.register(Post_and_pvzGroup, PostPvzGroupAdmin) +my_admin_site.register(OtherObjectsGroup, OtherObjectsGroupAdmin) +my_admin_site.register(Post_and_pvzCategory, PostPvzCategoryAdmin) +my_admin_site.register(OtherObjectsCategory, OtherObjectsCategoryAdmin) my_admin_site.register(PlacementPoint, PlacementPointAdmin) diff --git a/service/apps.py b/service/apps.py index 8d0ae67..1a6fe55 100644 --- a/service/apps.py +++ b/service/apps.py @@ -4,3 +4,6 @@ from django.apps import AppConfig class ServiceConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' name = 'service' + + def ready(self): + from service import signals diff --git a/service/enums.py b/service/enums.py index 3a8df3c..e517954 100644 --- a/service/enums.py +++ b/service/enums.py @@ -6,3 +6,8 @@ class PointStatus(Enum): Installation = 'Согласование-Установка' Working = 'Работает' Cancelled = 'Отменено' + +class MatchingStatus(Enum): + Error = 'Ошибка' + New = 'Новая' + Matched = 'Совпадение' diff --git a/service/layer_service.py b/service/layer_service.py new file mode 100644 index 0000000..ada5077 --- /dev/null +++ b/service/layer_service.py @@ -0,0 +1,44 @@ +from django.contrib.gis.measure import Distance +from service import models +from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS + + +class LayerService: + def count_post_pvz_for_placementpoint(self, obj): + points = models.PlacementPoint.objects.filter(geometry__distance_lt=(obj.wkt, Distance( + m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).all() + for point in points: + LayerService.count_post_pvz(point) + + @staticmethod + def update_categories(instance): + groups = models.Post_and_pvzGroup.objects.filter(category=instance) + groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible) + for gr in groups: + models.RaschetGroups.objects.get_or_create(obj_id=gr.id) + objects = models.Post_and_pvz.objects.filter(group=gr) + objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) + for obj in objects.all(): + models.RaschetObjects.objects.get_or_create(obj_id=obj.id) + + @staticmethod + def update_groups(instance): + models.RaschetGroups.objects.get_or_create(obj_id=instance.id) + objects = models.Post_and_pvz.objects.filter(group=instance) + objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) + for obj in objects.all(): + models.RaschetObjects.objects.get_or_create(obj_id=obj.id) + + @staticmethod + def get_post_and_pvz_categroies(): + return models.Post_and_pvzCategory.objects.all(), models.Post_and_pvzGroup.objects.all() + + @staticmethod + def count_post_pvz(point): + point.rival_post_cnt = models.Post_and_pvz.objects.filter( + category__name="Постамат", include_in_ml=True, + wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( + category__name="ПВЗ", include_in_ml=True, + wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.save() diff --git a/service/management/commands/create_procedures.py b/service/management/commands/create_procedures.py new file mode 100644 index 0000000..30fcff4 --- /dev/null +++ b/service/management/commands/create_procedures.py @@ -0,0 +1,63 @@ +from django.core.management.base import BaseCommand +from service.utils import run_sql_command, log_to_telegram + +CMD_PIVOT_DIST = """CREATE OR REPLACE VIEW compact_placementpoint AS +SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpoint; +CREATE OR REPLACE procedure pivot_dist() +--RETURNS SET OF record +AS $BODY$ +DECLARE columnNames TEXT; +BEGIN +DROP MATERIALIZED VIEW IF EXISTS points_with_dist; +SELECT 'placement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_placementpointpvzdistance order by 1) as asd +INTO columnNames; +EXECUTE format('CREATE MATERIALIZED VIEW points_with_dist AS SELECT * +FROM CROSSTAB( + $$ + SELECT placement_point_id, pvz_postamates_group_id, dist + FROM service_placementpointpvzdistance + ORDER BY 1, 2 + $$ +) AS ct(%s) +LEFT JOIN compact_placementpoint ON placement_point_id=id' + ,columnNames); +END; +$BODY$ +LANGUAGE plpgsql; +""" + +CMD_PIVOT_DIST_PRE = """CREATE OR REPLACE VIEW compact_preplacementpoint AS +SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpoint; +CREATE OR REPLACE procedure prepivot_dist() +AS $BODY$ +DECLARE columnNames TEXT; +BEGIN +DROP MATERIALIZED VIEW IF EXISTS prepoints_with_dist; +SELECT 'preplacement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_preplacementpointpvzdistance order by 1) as asd +INTO columnNames; +EXECUTE format('CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT * +FROM CROSSTAB( + $$ + SELECT placement_point_id, pvz_postamates_group_id, dist + FROM service_preplacementpointpvzdistance + ORDER BY 1, 2 + $$ +) AS ct(%s) +LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id' + ,columnNames); +END; +$BODY$ +LANGUAGE plpgsql; +""" +class Command(BaseCommand): + help = 'Create procedures' + + def handle(self, *args, **kwargs): + try: + log_to_telegram('Creating procedures') + run_sql_command(CMD_PIVOT_DIST) + log_to_telegram('pivot_dist created') + run_sql_command(CMD_PIVOT_DIST_PRE) + log_to_telegram('prepivot_dist created') + except Exception as e: + log_to_telegram('Error creating views: ' + str(e)) \ No newline at end of file diff --git a/service/management/commands/create_views.py b/service/management/commands/create_views.py new file mode 100644 index 0000000..bc6c1e8 --- /dev/null +++ b/service/management/commands/create_views.py @@ -0,0 +1,18 @@ +from django.core.management.base import BaseCommand +from service.utils import run_sql_command, log_to_telegram + +CMD_PIVOT_DIST = """CALL public.pivot_dist();""" + +CMD_PIVOT_DIST_PRE = """CALL public.prepivot_dist();""" +class Command(BaseCommand): + help = 'Create views' + + def handle(self, *args, **kwargs): + try: + log_to_telegram('Creating views') + run_sql_command(CMD_PIVOT_DIST) + log_to_telegram('pivot_dist created') + run_sql_command(CMD_PIVOT_DIST_PRE) + log_to_telegram('prepivot_dist created') + except Exception as e: + log_to_telegram('Error creating views: ' + str(e)) \ No newline at end of file diff --git a/service/management/commands/delete_views.py b/service/management/commands/delete_views.py new file mode 100644 index 0000000..eb7ff35 --- /dev/null +++ b/service/management/commands/delete_views.py @@ -0,0 +1,18 @@ +from django.core.management.base import BaseCommand +from service.utils import run_sql_command, log_to_telegram + +CMD_PIVOT_DIST = """DROP MATERIALIZED VIEW IF EXISTS public.points_with_dist;""" + +CMD_PIVOT_DIST_PRE = """DROP MATERIALIZED VIEW IF EXISTS public.prepoints_with_dist;""" +class Command(BaseCommand): + help = 'delete views' + + def handle(self, *args, **kwargs): + try: + log_to_telegram('Deleting views') + run_sql_command(CMD_PIVOT_DIST) + log_to_telegram('pivot_dist deleted') + run_sql_command(CMD_PIVOT_DIST_PRE) + log_to_telegram('prepivot_dist deleted') + except Exception as e: + log_to_telegram('Error deleting views: ' + str(e)) diff --git a/service/migrations/0022_lastmlcall.py b/service/migrations/0022_lastmlcall.py new file mode 100644 index 0000000..68fa331 --- /dev/null +++ b/service/migrations/0022_lastmlcall.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2023-07-15 13:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0021_auto_20230402_2129'), + ] + + operations = [ + migrations.CreateModel( + name='LastMLCall', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dt', models.DateTimeField(auto_now_add=True)), + ], + ), + ] diff --git a/service/migrations/0023_auto_20230801_1654.py b/service/migrations/0023_auto_20230801_1654.py new file mode 100644 index 0000000..25a50d8 --- /dev/null +++ b/service/migrations/0023_auto_20230801_1654.py @@ -0,0 +1,166 @@ +# Generated by Django 3.2 on 2023-08-01 13:54 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0022_lastmlcall'), + ] + + operations = [ + migrations.CreateModel( + name='OtherObjects', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('wkt', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326)), + ('param1', models.FloatField(blank=True, null=True)), + ('param2', models.TextField(blank=True, null=True)), + ('visible', models.BooleanField(default=True)), + ], + options={ + 'verbose_name': 'Прочий объект', + 'verbose_name_plural': 'Прочие объекты', + }, + ), + migrations.CreateModel( + name='OtherObjectsCategory', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(verbose_name='Название слоя')), + ('image', models.ImageField(blank=True, default=None, null=True, upload_to='other_objects_category_images/', verbose_name='Картинка')), + ('visible', models.BooleanField(default=True)), + ], + options={ + 'verbose_name': 'категория прочих объектов', + 'verbose_name_plural': 'Категории прочих объектов', + }, + ), + migrations.CreateModel( + name='OtherObjectsGroup', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(verbose_name='Название группы')), + ('image', models.ImageField(blank=True, null=True, upload_to='other_objects_group_images/', verbose_name='Картинка')), + ('visible', models.BooleanField(default=True)), + ('category', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='otherobjectsgroup', to='service.otherobjectscategory')), + ], + options={ + 'verbose_name': 'группа прочих объектов', + 'verbose_name_plural': 'Группы прочих объектов', + }, + ), + migrations.CreateModel( + name='PlacementPointPVZDistance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dist', models.FloatField(blank=True, default=None, null=True)), + ], + ), + migrations.CreateModel( + name='Post_and_pvz', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('wkt', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326)), + ('visible', models.BooleanField(default=True)), + ('inlude_in_ml', models.BooleanField(default=True)), + ], + options={ + 'verbose_name': 'Постамат или ПВЗ', + 'verbose_name_plural': 'Постаматы и ПВЗ', + }, + ), + migrations.CreateModel( + name='Post_and_pvzCategory', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(verbose_name='Название слоя')), + ('image', models.ImageField(blank=True, default=None, null=True, upload_to='post_and_pvz_category_images/', verbose_name='Картинка')), + ('visible', models.BooleanField(default=True)), + ('inlude_in_ml', models.BooleanField(default=True)), + ], + options={ + 'verbose_name': 'категория постаматов и ПВЗ', + 'verbose_name_plural': 'Категории постаматов и ПВЗ', + }, + ), + migrations.CreateModel( + name='Post_and_pvzGroup', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(verbose_name='Название группы')), + ('image', models.ImageField(blank=True, null=True, upload_to='post_and_pvz_group_images/', verbose_name='Картинка')), + ('visible', models.BooleanField(default=True)), + ('inlude_in_ml', models.BooleanField(default=True)), + ('category', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='post_and_pvz_group', to='service.post_and_pvzcategory')), + ], + options={ + 'verbose_name': 'группа постаматов и ПВЗ', + 'verbose_name_plural': 'Группы постаматов и ПВЗ', + }, + ), + migrations.CreateModel( + name='TaskStatus', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('task_name', models.TextField(unique=True, verbose_name='Название задачи')), + ('status', models.TextField(blank=True, null=True, verbose_name='Статус выполнения')), + ], + options={ + 'verbose_name': 'Статус фоновых задач', + 'verbose_name_plural': 'Статус фоновых задач', + }, + ), + migrations.DeleteModel( + name='Rivals', + ), + migrations.AlterModelOptions( + name='ao', + options={'verbose_name': 'АО', 'verbose_name_plural': 'АО'}, + ), + migrations.AlterModelOptions( + name='placementpoint', + options={'verbose_name': 'Точка', 'verbose_name_plural': 'Точки'}, + ), + migrations.AlterModelOptions( + name='pointdist', + options={'verbose_name_plural': 'Расстояния между точками'}, + ), + migrations.AlterModelOptions( + name='rayon', + options={'verbose_name': 'Район', 'verbose_name_plural': 'Районы'}, + ), + migrations.AddField( + model_name='post_and_pvz', + name='category', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='post_and_pvz', to='service.post_and_pvzcategory'), + ), + migrations.AddField( + model_name='post_and_pvz', + name='group', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='post_and_pvz', to='service.post_and_pvzgroup'), + ), + migrations.AddField( + model_name='placementpointpvzdistance', + name='placement_point', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.placementpoint'), + ), + migrations.AddField( + model_name='placementpointpvzdistance', + name='pvz_postamates_group', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup'), + ), + migrations.AddField( + model_name='otherobjects', + name='category', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='other_objects', to='service.otherobjectscategory'), + ), + migrations.AddField( + model_name='otherobjects', + name='group', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='other_objects', to='service.otherobjectsgroup'), + ), + ] diff --git a/service/migrations/0024_tempfiles.py b/service/migrations/0024_tempfiles.py new file mode 100644 index 0000000..6b2ec1d --- /dev/null +++ b/service/migrations/0024_tempfiles.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2023-08-03 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0023_auto_20230801_1654'), + ] + + operations = [ + migrations.CreateModel( + name='TempFiles', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data', models.TextField()), + ], + ), + ] diff --git a/service/migrations/0025_auto_20230804_1349.py b/service/migrations/0025_auto_20230804_1349.py new file mode 100644 index 0000000..6b07a2d --- /dev/null +++ b/service/migrations/0025_auto_20230804_1349.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2 on 2023-08-04 10:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0024_tempfiles'), + ] + + operations = [ + migrations.AlterField( + model_name='otherobjectsgroup', + name='category', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.otherobjectscategory'), + ), + migrations.AlterField( + model_name='post_and_pvzgroup', + name='category', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.post_and_pvzcategory'), + ), + ] diff --git a/service/migrations/0026_auto_20230823_2016.py b/service/migrations/0026_auto_20230823_2016.py new file mode 100644 index 0000000..0d2205b --- /dev/null +++ b/service/migrations/0026_auto_20230823_2016.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2 on 2023-08-23 17:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0025_auto_20230804_1349'), + ] + + operations = [ + migrations.RenameField( + model_name='post_and_pvz', + old_name='inlude_in_ml', + new_name='include_in_ml', + ), + migrations.RenameField( + model_name='post_and_pvzcategory', + old_name='inlude_in_ml', + new_name='include_in_ml', + ), + migrations.RenameField( + model_name='post_and_pvzgroup', + old_name='inlude_in_ml', + new_name='include_in_ml', + ), + migrations.DeleteModel( + name='PointDist', + ), + ] diff --git a/service/migrations/0027_postmlchecker.py b/service/migrations/0027_postmlchecker.py new file mode 100644 index 0000000..8abf586 --- /dev/null +++ b/service/migrations/0027_postmlchecker.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2 on 2023-08-24 17:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0026_auto_20230823_2016'), + ] + + operations = [ + migrations.CreateModel( + name='PostMLChecker', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('current', models.IntegerField(default=0)), + ('target', models.IntegerField()), + ], + ), + ] diff --git a/service/migrations/0028_delete_postmlchecker.py b/service/migrations/0028_delete_postmlchecker.py new file mode 100644 index 0000000..95d3625 --- /dev/null +++ b/service/migrations/0028_delete_postmlchecker.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2 on 2023-08-31 12:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0027_postmlchecker'), + ] + + operations = [ + migrations.DeleteModel( + name='PostMLChecker', + ), + ] diff --git a/service/migrations/0029_alter_placementpoint_postamat_id.py b/service/migrations/0029_alter_placementpoint_postamat_id.py new file mode 100644 index 0000000..0e36bb5 --- /dev/null +++ b/service/migrations/0029_alter_placementpoint_postamat_id.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2023-08-31 18:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0028_delete_postmlchecker'), + ] + + operations = [ + migrations.AlterField( + model_name='placementpoint', + name='postamat_id', + field=models.IntegerField(blank=True, null=True, unique=True, verbose_name='ID постамата'), + ), + ] diff --git a/service/migrations/0030_auto_20230903_2006.py b/service/migrations/0030_auto_20230903_2006.py new file mode 100644 index 0000000..49ea2d8 --- /dev/null +++ b/service/migrations/0030_auto_20230903_2006.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2 on 2023-09-03 17:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0029_alter_placementpoint_postamat_id'), + ] + + operations = [ + migrations.AlterModelOptions( + name='otherobjects', + options={'ordering': ('id',), 'verbose_name': 'Прочий объект', 'verbose_name_plural': 'Прочие объекты'}, + ), + migrations.AlterModelOptions( + name='placementpoint', + options={'ordering': ('id',), 'verbose_name': 'Точка', 'verbose_name_plural': 'Точки'}, + ), + migrations.AlterModelOptions( + name='post_and_pvz', + options={'ordering': ('id',), 'verbose_name': 'Постамат или ПВЗ', 'verbose_name_plural': 'Постаматы и ПВЗ'}, + ), + migrations.AddField( + model_name='otherobjects', + name='param3', + field=models.FloatField(blank=True, null=True), + ), + migrations.AddField( + model_name='otherobjects', + name='param4', + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/service/migrations/0031_preplacementpoint.py b/service/migrations/0031_preplacementpoint.py new file mode 100644 index 0000000..431e3da --- /dev/null +++ b/service/migrations/0031_preplacementpoint.py @@ -0,0 +1,94 @@ +# Generated by Django 3.2 on 2023-09-05 17:56 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0030_auto_20230903_2006'), + ] + + operations = [ + migrations.CreateModel( + name='PrePlacementPoint', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('address', models.TextField(blank=True, null=True, verbose_name='Адрес')), + ('name', models.TextField(blank=True, null=True, verbose_name='Название')), + ('postamat_id', models.IntegerField(blank=True, null=True, unique=True, verbose_name='ID постамата')), + ('category', models.TextField(blank=True, null=True, verbose_name='Категория')), + ('status', models.TextField(blank=True, choices=[('Pending', 'К рассмотрению'), ('Installation', 'Согласование-Установка'), ('Working', 'Работает'), ('Cancelled', 'Отменено')], null=True, verbose_name='Статус')), + ('start_date', models.DateTimeField(blank=True, null=True)), + ('age_day', models.IntegerField(blank=True, null=True, verbose_name='Возраст')), + ('prediction_first', models.IntegerField(blank=True, null=True, verbose_name='Прогноз начальный')), + ('prediction_current', models.IntegerField(blank=True, null=True, verbose_name='Прогноз текущий')), + ('plan_first', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель начальный')), + ('plan_current', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель текущий')), + ('fact', models.IntegerField(blank=True, null=True, verbose_name='Фактический показатель')), + ('fact_raw', models.IntegerField(blank=True, null=True)), + ('delta_first', models.IntegerField(blank=True, null=True, verbose_name='Разница начальная')), + ('delta_current', models.IntegerField(blank=True, null=True, verbose_name='Разница текущая')), + ('sample_trn', models.BooleanField(blank=True, null=True)), + ('flat_cnt', models.IntegerField(blank=True, null=True, verbose_name='Количество квартир')), + ('year_bld', models.IntegerField(blank=True, null=True, verbose_name='Год постройки')), + ('levels', models.IntegerField(blank=True, null=True)), + ('enrg_cls', models.TextField(blank=True, null=True)), + ('mat_nes', models.TextField(blank=True, null=True)), + ('doors', models.IntegerField(blank=True, null=True)), + ('flats_cnt', models.IntegerField(blank=True, null=True)), + ('popul_home', models.IntegerField(blank=True, null=True)), + ('popul_job', models.IntegerField(blank=True, null=True)), + ('other_post_cnt', models.IntegerField(blank=True, null=True)), + ('target_post_cnt', models.IntegerField(blank=True, null=True)), + ('yndxfood_cnt', models.IntegerField(blank=True, null=True)), + ('yndxfood_sum', models.IntegerField(blank=True, null=True)), + ('yndxfood_cnt_cst', models.IntegerField(blank=True, null=True)), + ('geometry', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326, verbose_name='Координаты')), + ('is_vis', models.BooleanField(blank=True, null=True)), + ('subject_rf', models.TextField(blank=True, null=True)), + ('city', models.TextField(blank=True, null=True)), + ('street', models.TextField(blank=True, null=True)), + ('house_number', models.TextField(blank=True, null=True)), + ('entrance', models.TextField(blank=True, null=True)), + ('post_code', models.TextField(blank=True, null=True)), + ('metro_dist', models.FloatField(blank=True, null=True)), + ('target_dist', models.FloatField(blank=True, null=True)), + ('property_price_bargains', models.FloatField(blank=True, null=True)), + ('property_price_offers', models.FloatField(blank=True, null=True)), + ('property_mean_floor', models.FloatField(blank=True, null=True)), + ('property_era', models.TextField(blank=True, null=True)), + ('business_activity', models.IntegerField(blank=True, null=True)), + ('bc_cnt', models.IntegerField(blank=True, null=True)), + ('tc_cnt', models.IntegerField(blank=True, null=True)), + ('rival_pvz_cnt', models.IntegerField(blank=True, null=True)), + ('rival_post_cnt', models.IntegerField(blank=True, null=True)), + ('flats_cnt_2', models.IntegerField(blank=True, null=True)), + ('school_cnt', models.IntegerField(blank=True, null=True)), + ('kindergar_cnt', models.IntegerField(blank=True, null=True)), + ('public_stop_cnt', models.IntegerField(blank=True, null=True)), + ('sport_center_cnt', models.IntegerField(blank=True, null=True)), + ('pharmacy_cnt', models.IntegerField(blank=True, null=True)), + ('supermarket_cnt', models.IntegerField(blank=True, null=True)), + ('supermarket_premium_cnt', models.IntegerField(blank=True, null=True)), + ('clinic_cnt', models.IntegerField(blank=True, null=True)), + ('bank_cnt', models.IntegerField(blank=True, null=True)), + ('reca_cnt', models.IntegerField(blank=True, null=True)), + ('lab_cnt', models.IntegerField(blank=True, null=True)), + ('culture_cnt', models.IntegerField(blank=True, null=True)), + ('attraction_cnt', models.IntegerField(blank=True, null=True)), + ('mfc_cnt', models.IntegerField(blank=True, null=True)), + ('target_cnt_ao_mean', models.FloatField(blank=True, null=True)), + ('target_cnt_nearby_mean', models.FloatField(blank=True, null=True)), + ('target_age_nearby_mean', models.FloatField(blank=True, null=True)), + ('matching_status', models.TextField(blank=True, choices=[('Error', 'Ошибка'), ('New', 'Новая'), ('Matched', 'Совпадение')], null=True)), + ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.rayon')), + ('district', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.ao')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/service/migrations/0032_auto_20230912_1616.py b/service/migrations/0032_auto_20230912_1616.py new file mode 100644 index 0000000..cbcaed3 --- /dev/null +++ b/service/migrations/0032_auto_20230912_1616.py @@ -0,0 +1,27 @@ +# Generated by Django 3.2 on 2023-09-12 13:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0031_preplacementpoint'), + ] + + operations = [ + migrations.AlterModelOptions( + name='preplacementpoint', + options={'ordering': ('id',)}, + ), + migrations.CreateModel( + name='PrePlacementPointPVZDistance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dist', models.FloatField(blank=True, default=None, null=True)), + ('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.preplacementpoint')), + ('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')), + ], + ), + ] diff --git a/service/migrations/0033_raschetobjects.py b/service/migrations/0033_raschetobjects.py new file mode 100644 index 0000000..0adecf0 --- /dev/null +++ b/service/migrations/0033_raschetobjects.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2023-09-16 09:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0032_auto_20230912_1616'), + ] + + operations = [ + migrations.CreateModel( + name='RaschetObjects', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obj_id', models.IntegerField()), + ], + ), + ] diff --git a/service/migrations/0034_raschetgroups.py b/service/migrations/0034_raschetgroups.py new file mode 100644 index 0000000..e312314 --- /dev/null +++ b/service/migrations/0034_raschetgroups.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2023-09-16 11:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0033_raschetobjects'), + ] + + operations = [ + migrations.CreateModel( + name='RaschetGroups', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obj_id', models.IntegerField()), + ], + ), + ] diff --git a/service/migrations/0035_auto_20230928_1519.py b/service/migrations/0035_auto_20230928_1519.py new file mode 100644 index 0000000..582dc17 --- /dev/null +++ b/service/migrations/0035_auto_20230928_1519.py @@ -0,0 +1,333 @@ +# Generated by Django 3.2 on 2023-09-28 12:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0034_raschetgroups'), + ] + + operations = [ + migrations.AddField( + model_name='placementpoint', + name='attraction_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='bank_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='bc_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='business_activity_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='clinic_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='culture_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='flats_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='kindergar_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='lab_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='metro_dist_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='mfc_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='pharmacy_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='popul_home_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='popul_job_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='property_era_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='property_mean_floor_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='property_price_bargains_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='property_price_offers_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='public_stop_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='reca_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='rival_post_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='rival_pvz_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='school_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='sport_center_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='supermarket_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='supermarket_premium_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='target_cnt_ao_mean_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='target_dist_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='target_post_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='tc_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='yndxfood_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='placementpoint', + name='yndxfood_sum_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='attraction_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='bank_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='bc_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='business_activity_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='clinic_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='culture_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='flats_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='kindergar_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='lab_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='metro_dist_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='mfc_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='pharmacy_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='popul_home_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='popul_job_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='property_era_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='property_mean_floor_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='property_price_bargains_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='property_price_offers_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='public_stop_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='reca_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='rival_post_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='rival_pvz_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='school_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='sport_center_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='supermarket_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='supermarket_premium_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='target_cnt_ao_mean_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='target_dist_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='target_post_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='tc_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='yndxfood_cnt_shap', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='preplacementpoint', + name='yndxfood_sum_shap', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/service/migrations/0036_house.py b/service/migrations/0036_house.py new file mode 100644 index 0000000..8b1845a --- /dev/null +++ b/service/migrations/0036_house.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2 on 2023-10-02 09:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0035_auto_20230928_1519'), + ] + + operations = [ + migrations.CreateModel( + name='House', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('year_bld', models.IntegerField(blank=True, null=True)), + ('mat_nes', models.TextField(blank=True, null=True)), + ('flat_cnt', models.IntegerField(blank=True, null=True)), + ('levels', models.TextField(blank=True, null=True)), + ('doors', models.IntegerField(blank=True, null=True)), + ('enrg_cls', models.TextField(blank=True, null=True)), + ('street', models.TextField(blank=True, null=True)), + ('house_number', models.TextField(blank=True, null=True)), + ], + options={ + 'verbose_name': 'Дом', + 'verbose_name_plural': 'Дома', + 'ordering': ('id',), + }, + ), + ] diff --git a/service/migrations/0037_auto_20231002_1349.py b/service/migrations/0037_auto_20231002_1349.py new file mode 100644 index 0000000..f18ef38 --- /dev/null +++ b/service/migrations/0037_auto_20231002_1349.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2 on 2023-10-02 10:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0036_house'), + ] + + operations = [ + migrations.AlterField( + model_name='placementpoint', + name='levels', + field=models.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='preplacementpoint', + name='levels', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/service/migrations/0038_auto_20231008_1141.py b/service/migrations/0038_auto_20231008_1141.py new file mode 100644 index 0000000..e9b88d7 --- /dev/null +++ b/service/migrations/0038_auto_20231008_1141.py @@ -0,0 +1,135 @@ +# Generated by Django 3.2 on 2023-10-08 08:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0037_auto_20231002_1349'), + ] + + operations = [ + migrations.RemoveField( + model_name='preplacementpointpvzdistance', + name='placement_point', + ), + migrations.RemoveField( + model_name='preplacementpointpvzdistance', + name='pvz_postamates_group', + ), + migrations.RemoveField( + model_name='otherobjectscategory', + name='image', + ), + migrations.RemoveField( + model_name='post_and_pvzcategory', + name='image', + ), + migrations.AddField( + model_name='placementpoint', + name='d10', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата СДЭК'), + ), + migrations.AddField( + model_name='placementpoint', + name='d12', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Ozon'), + ), + migrations.AddField( + model_name='placementpoint', + name='d17', + field=models.FloatField(null=True, verbose_name='Расстояние до ПВЗ Wildberries'), + ), + migrations.AddField( + model_name='placementpoint', + name='d19', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Yandex'), + ), + migrations.AddField( + model_name='placementpoint', + name='d20', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Yandex'), + ), + migrations.AddField( + model_name='placementpoint', + name='d24', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Почта России'), + ), + migrations.AddField( + model_name='placementpoint', + name='d25', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Почта России'), + ), + migrations.AddField( + model_name='placementpoint', + name='d7', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Ozon'), + ), + migrations.AddField( + model_name='placementpoint', + name='d8', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ СДЭК'), + ), + migrations.AddField( + model_name='placementpoint', + name='d9', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Халва'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d10', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата СДЭК'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d12', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Ozon'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d17', + field=models.FloatField(null=True, verbose_name='Расстояние до ПВЗ Wildberries'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d19', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Yandex'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d20', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Yandex'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d24', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Почта России'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d25', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Почта России'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d7', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Ozon'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d8', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ СДЭК'), + ), + migrations.AddField( + model_name='preplacementpoint', + name='d9', + field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Халва'), + ), + migrations.DeleteModel( + name='PlacementPointPVZDistance', + ), + migrations.DeleteModel( + name='PrePlacementPointPVZDistance', + ), + ] diff --git a/service/migrations/0039_auto_20231011_2120.py b/service/migrations/0039_auto_20231011_2120.py new file mode 100644 index 0000000..049c1ec --- /dev/null +++ b/service/migrations/0039_auto_20231011_2120.py @@ -0,0 +1,122 @@ +# Generated by Django 3.2 on 2023-10-11 18:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0038_auto_20231008_1141'), + ] + + operations = [ + migrations.RemoveField( + model_name='placementpoint', + name='d10', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d12', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d17', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d19', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d20', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d24', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d25', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d7', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d8', + ), + migrations.RemoveField( + model_name='placementpoint', + name='d9', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d10', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d12', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d17', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d19', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d20', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d24', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d25', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d7', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d8', + ), + migrations.RemoveField( + model_name='preplacementpoint', + name='d9', + ), + migrations.AddField( + model_name='otherobjectscategory', + name='image', + field=models.ImageField(blank=True, default=None, null=True, upload_to='other_objects_category_images/', verbose_name='Картинка'), + ), + migrations.AddField( + model_name='post_and_pvzcategory', + name='image', + field=models.ImageField(blank=True, default=None, null=True, upload_to='post_and_pvz_category_images/', verbose_name='Картинка'), + ), + migrations.CreateModel( + name='PrePlacementPointPVZDistance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dist', models.FloatField(blank=True, default=None, null=True)), + ('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.preplacementpoint')), + ('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')), + ], + ), + migrations.CreateModel( + name='PlacementPointPVZDistance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('dist', models.FloatField(blank=True, default=None, null=True)), + ('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.placementpoint')), + ('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')), + ], + ), + ] diff --git a/service/models.py b/service/models.py index fe1a4f8..578d0f4 100644 --- a/service/models.py +++ b/service/models.py @@ -1,19 +1,20 @@ from django.contrib.auth.models import User from django.contrib.gis.db import models as gis_models from django.db import models - from postamates.settings import SRID -from service.enums import PointStatus -from service.signals import * +from service.enums import PointStatus, MatchingStatus User._meta.get_field('email')._unique = True -class PlacementPoint(models.Model): +class AbstractPlacementPoint(models.Model): + class Meta: + abstract = True + STATUS_CHOICES = [(tag.name, tag.value) for tag in PointStatus] address = models.TextField(null=True, blank=True, verbose_name='Адрес') name = models.TextField(null=True, blank=True, verbose_name='Название') - postamat_id = models.TextField(unique=True, null=True, blank=True, verbose_name='ID постамата') + postamat_id = models.IntegerField(unique=True, null=True, blank=True, verbose_name='ID постамата') category = models.TextField(null=True, blank=True, verbose_name='Категория') status = models.TextField(choices=STATUS_CHOICES, null=True, blank=True, verbose_name='Статус') start_date = models.DateTimeField(null=True, blank=True) @@ -29,7 +30,7 @@ class PlacementPoint(models.Model): sample_trn = models.BooleanField(null=True, blank=True) flat_cnt = models.IntegerField(null=True, blank=True, verbose_name='Количество квартир') year_bld = models.IntegerField(null=True, blank=True, verbose_name='Год постройки') - levels = models.IntegerField(null=True, blank=True) + levels = models.TextField(null=True, blank=True) enrg_cls = models.TextField(null=True, blank=True) mat_nes = models.TextField(null=True, blank=True) doors = models.IntegerField(null=True, blank=True) @@ -80,27 +81,216 @@ class PlacementPoint(models.Model): target_cnt_ao_mean = models.FloatField(null=True, blank=True) target_cnt_nearby_mean = models.FloatField(null=True, blank=True) target_age_nearby_mean = models.FloatField(null=True, blank=True) + target_dist_shap = models.IntegerField(null=True, blank=True) + target_post_cnt_shap = models.IntegerField(null=True, blank=True) + target_cnt_ao_mean_shap = models.IntegerField(null=True, blank=True) + rival_pvz_cnt_shap = models.IntegerField(null=True, blank=True) + rival_post_cnt_shap = models.IntegerField(null=True, blank=True) + metro_dist_shap = models.IntegerField(null=True, blank=True) + property_price_bargains_shap = models.IntegerField(null=True, blank=True) + property_price_offers_shap = models.IntegerField(null=True, blank=True) + property_mean_floor_shap = models.IntegerField(null=True, blank=True) + property_era_shap = models.IntegerField(null=True, blank=True) + flats_cnt_shap = models.IntegerField(null=True, blank=True) + popul_home_shap = models.IntegerField(null=True, blank=True) + popul_job_shap = models.IntegerField(null=True, blank=True) + yndxfood_sum_shap = models.IntegerField(null=True, blank=True) + yndxfood_cnt_shap = models.IntegerField(null=True, blank=True) + school_cnt_shap = models.IntegerField(null=True, blank=True) + kindergar_cnt_shap = models.IntegerField(null=True, blank=True) + public_stop_cnt_shap = models.IntegerField(null=True, blank=True) + sport_center_cnt_shap = models.IntegerField(null=True, blank=True) + pharmacy_cnt_shap = models.IntegerField(null=True, blank=True) + supermarket_cnt_shap = models.IntegerField(null=True, blank=True) + supermarket_premium_cnt_shap = models.IntegerField(null=True, blank=True) + clinic_cnt_shap = models.IntegerField(null=True, blank=True) + bank_cnt_shap = models.IntegerField(null=True, blank=True) + reca_cnt_shap = models.IntegerField(null=True, blank=True) + lab_cnt_shap = models.IntegerField(null=True, blank=True) + culture_cnt_shap = models.IntegerField(null=True, blank=True) + attraction_cnt_shap = models.IntegerField(null=True, blank=True) + mfc_cnt_shap = models.IntegerField(null=True, blank=True) + bc_cnt_shap = models.IntegerField(null=True, blank=True) + tc_cnt_shap = models.IntegerField(null=True, blank=True) + business_activity_shap = models.IntegerField(null=True, blank=True) + + +class PlacementPoint(AbstractPlacementPoint): + class Meta: + verbose_name = 'Точка' + verbose_name_plural = 'Точки' + ordering = ('id',) + + +class PrePlacementPoint(AbstractPlacementPoint): + class Meta: + ordering = ('id',) + + MATCHING_CHOICES = [(tag.name, tag.value) for tag in MatchingStatus] + matching_status = models.TextField(null=True, blank=True, choices=MATCHING_CHOICES) class AO(models.Model): + class Meta: + verbose_name = 'АО' + verbose_name_plural = 'АО' + name = models.TextField(null=True, blank=True, verbose_name='Округ') polygon = gis_models.MultiPolygonField(null=True, srid=SRID) class Rayon(models.Model): + class Meta: + verbose_name = 'Район' + verbose_name_plural = 'Районы' + name = models.TextField(null=True, blank=True, verbose_name='Район') AO = models.ForeignKey('AO', related_name='rayons', on_delete=models.CASCADE) polygon = gis_models.MultiPolygonField(null=True, srid=SRID) -class Rivals(models.Model): +class Post_and_pvz(models.Model): + class Meta: + verbose_name = 'Постамат или ПВЗ' + verbose_name_plural = 'Постаматы и ПВЗ' + ordering = ('id',) + + wkt = gis_models.PointField(srid=SRID, null=True) + category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='post_and_pvz', + on_delete=models.CASCADE) + group = models.ForeignKey('Post_and_pvzGroup', default=None, related_name='post_and_pvz', on_delete=models.CASCADE) + visible = models.BooleanField(default=True) + include_in_ml = models.BooleanField(default=True) + + +class OtherObjects(models.Model): + class Meta: + verbose_name = 'Прочий объект' + verbose_name_plural = 'Прочие объекты' + ordering = ('id',) + wkt = gis_models.PointField(srid=SRID, null=True) - info = models.TextField(null=True, blank=True) - type = models.TextField(null=True, blank=True) - source = models.TextField(null=True, blank=True) + category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='other_objects', + on_delete=models.CASCADE) + group = models.ForeignKey('OtherObjectsGroup', default=None, related_name='other_objects', on_delete=models.CASCADE) + param1 = models.FloatField(blank=True, null=True) + param2 = models.TextField(blank=True, null=True) + param3 = models.FloatField(blank=True, null=True) + param4 = models.FloatField(blank=True, null=True) + visible = models.BooleanField(default=True) + + +class Post_and_pvzCategory(models.Model): + class Meta: + verbose_name = 'категория постаматов и ПВЗ' + verbose_name_plural = 'Категории постаматов и ПВЗ' + + def __str__(self): + return self.name + + name = models.TextField(null=False, blank=False, verbose_name='Название слоя') + image = models.ImageField(blank=True, null=True, default=None, upload_to='post_and_pvz_category_images/', + verbose_name='Картинка') + visible = models.BooleanField(default=True) + include_in_ml = models.BooleanField(default=True) + + +class Post_and_pvzGroup(models.Model): + class Meta: + verbose_name = 'группа постаматов и ПВЗ' + verbose_name_plural = 'Группы постаматов и ПВЗ' + + def __str__(self): + return self.category.name + ' ' + self.name + + name = models.TextField(null=False, blank=False, verbose_name='Название группы') + category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='groups', + on_delete=models.CASCADE) + image = models.ImageField(blank=True, null=True, upload_to='post_and_pvz_group_images/', verbose_name='Картинка') + visible = models.BooleanField(default=True) + include_in_ml = models.BooleanField(default=True) + + +class OtherObjectsCategory(models.Model): + class Meta: + verbose_name = 'категория прочих объектов' + verbose_name_plural = 'Категории прочих объектов' + + def __str__(self): + return self.name + + name = models.TextField(null=False, blank=False, verbose_name='Название слоя') + image = models.ImageField(blank=True, null=True, default=None, upload_to='other_objects_category_images/', + verbose_name='Картинка') + visible = models.BooleanField(default=True) + + +class OtherObjectsGroup(models.Model): + class Meta: + verbose_name = 'группа прочих объектов' + verbose_name_plural = 'Группы прочих объектов' + + def __str__(self): + return self.category.name + ' ' + self.name + + name = models.TextField(null=False, blank=False, verbose_name='Название группы') + category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='groups', + on_delete=models.CASCADE) + image = models.ImageField(blank=True, null=True, upload_to='other_objects_group_images/', verbose_name='Картинка') + visible = models.BooleanField(default=True) + + +class PlacementPointPVZDistance(models.Model): + placement_point = models.ForeignKey('PlacementPoint', default=None, on_delete=models.CASCADE) + pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE) + dist = models.FloatField(blank=True, null=True, default=None) + + +class PrePlacementPointPVZDistance(models.Model): + placement_point = models.ForeignKey('PrePlacementPoint', default=None, on_delete=models.CASCADE) + pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE) + dist = models.FloatField(blank=True, null=True, default=None) + + +class TaskStatus(models.Model): + class Meta: + verbose_name = 'Статус фоновых задач' + verbose_name_plural = 'Статус фоновых задач' + + def __str__(self): + return self.task_name + + task_name = models.TextField(blank=False, unique=True, verbose_name='Название задачи') + status = models.TextField(blank=True, null=True, verbose_name='Статус выполнения') + + +class LastMLCall(models.Model): + dt = models.DateTimeField(auto_now_add=True) + + +class TempFiles(models.Model): + data = models.TextField(blank=False, null=False) + + +class RaschetGroups(models.Model): + obj_id = models.IntegerField(null=False, blank=False) + + +class RaschetObjects(models.Model): + obj_id = models.IntegerField(null=False, blank=False) + +class House(models.Model): + class Meta: + verbose_name = 'Дом' + verbose_name_plural = 'Дома' + ordering = ('id',) -class PointDist(models.Model): - id1 = models.ForeignKey('PlacementPoint', on_delete=models.CASCADE, null=False, related_name='placement_point_id1') - id2 = models.ForeignKey('PlacementPoint', on_delete=models.CASCADE, null=False, related_name='placement_point_id2') - distance = models.FloatField(null=False) + year_bld = models.IntegerField(blank=True,null=True) + mat_nes = models.TextField(blank=True,null=True) + flat_cnt = models.IntegerField(blank=True, null=True) + levels = models.TextField(blank=True,null=True) + doors = models.IntegerField(blank=True, null=True) + enrg_cls = models.TextField(blank=True,null=True) + street = models.TextField(blank=True,null=True) + house_number = models.TextField(blank=True,null=True) \ No newline at end of file diff --git a/service/serializers.py b/service/serializers.py index 1efb341..e89d809 100644 --- a/service/serializers.py +++ b/service/serializers.py @@ -1,6 +1,7 @@ from rest_framework import serializers from service import models +from service.service import PointService class PlacementPointSerializer(serializers.ModelSerializer): @@ -8,6 +9,46 @@ class PlacementPointSerializer(serializers.ModelSerializer): model = models.PlacementPoint fields = '__all__' + def to_representation(self, instance): + representation = super().to_representation(instance) + min_distances = PointService.get_min_distances_to_group(instance.id) + representation['min_distance_to_group'] = min_distances + return representation + +class PrePlacementPointSerializer(PlacementPointSerializer): + class Meta: + model = models.PrePlacementPoint + fields = '__all__' + + + +class PostAndPVZGroupSerializer(serializers.ModelSerializer): + class Meta: + model = models.Post_and_pvzGroup + fields = '__all__' + + +class PostAndPVZCategorySerializer(serializers.ModelSerializer): + groups = PostAndPVZGroupSerializer(read_only=True, many=True) + + class Meta: + model = models.Post_and_pvzCategory + fields = '__all__' + + +class OtherObjectsGroupSerializer(serializers.ModelSerializer): + class Meta: + model = models.OtherObjectsGroup + fields = '__all__' + + +class OtherObjectsCategorySerializer(serializers.ModelSerializer): + groups = OtherObjectsGroupSerializer(many=True, read_only=True) + + class Meta: + model = models.OtherObjectsCategory + fields = '__all__' + class RayonSerializer(serializers.ModelSerializer): class Meta: diff --git a/service/service.py b/service/service.py index 644ce9d..b7b5ecb 100644 --- a/service/service.py +++ b/service/service.py @@ -5,13 +5,20 @@ import pandas as pd from django.contrib.gis.measure import Distance from django.db.models import F -from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS +from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT from service import models from service.enums import PointStatus -from service.tasks import raschet +from service.utils import create_columns_dist, run_psql_command +import base64 +import requests +from postamates.settings import GEOCODER_API_KEY +from service.enums import MatchingStatus +from django.contrib.gis.db.models.functions import Distance as Dist +from django.db.models import Avg, Sum, Count class PointService: + def update_fact(self, postamat_id: str, fact: int): qs = self.get_point_by_postamat_id(postamat_id) qs.update(**{'fact': fact}) @@ -20,9 +27,249 @@ class PointService: qs = self.get_point_by_id(point_id) qs.update(**{'postamat_id': postamat_id}) + def start_mathing(self, obj_id: int): + file = models.TempFiles.objects.get(id=obj_id) + excel_file = base64.b64decode(file.data) + df = pd.read_excel(excel_file) + total = df.shape[0] + matched = 0 + problem = 0 + for _i, row in df.iterrows(): + addr = row['Адрес'] + cat = row['Категория объекта'] + req_url = f"https://geocode.search.hereapi.com/v1/geocode?q={addr}&apiKey={GEOCODER_API_KEY}" + response = requests.get(req_url).json().get('items') + if not response: + models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, + status=PointStatus.Pending.name) + problem += 1 + continue + coords = response[0].get('position') + if not coords: + models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, + status=PointStatus.Pending.name) + problem += 1 + continue + wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")" + response = response[0]['address'] + obj = models.PlacementPoint.objects.filter(street=response.get('street'), + house_number=response.get('houseNumber'), + category=cat).values() + rayon = models.Rayon.objects.filter(polygon__intersects=wkt).first() + if obj: + if cat == 'Подъезд жилого дома': + objs = obj.all() + else: + objs = [obj.first()] + for o in objs: + distances = models.PlacementPointPVZDistance.objects.filter(placement_point=o.get('id')).all() + o.pop('id') + street = o.pop('street') + house_number = o.pop('house_number') + if o.get('name'): + name = o.pop('name') + pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, + house_number=house_number,name=name, + matching_status=MatchingStatus.Matched.name, + defaults=o) + else: + pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, + house_number=house_number, + matching_status=MatchingStatus.Matched.name, + defaults=o) + for d in distances: + models.PrePlacementPointPVZDistance.objects.get_or_create(placement_point=pre_obj, + pvz_postamates_group=d.pvz_postamates_group, + dist=d.dist) + + matched += 1 + elif not rayon: + models.PrePlacementPoint.objects.get_or_create(street=response.get('street'), + house_number=response.get('houseNumber'), + address=addr, + subject_rf=response.get('state'), + city=response.get('city'), + category=cat, geometry=wkt, sample_trn=False, + is_vis=True, + matching_status=MatchingStatus.Error.name, + status=PointStatus.Pending.name) + problem += 1 + elif cat == 'Подъезд жилого дома' and models.House.objects.filter(street=response.get('street'), + house_number=response.get( + 'houseNumber')).first(): + house = models.House.objects.filter(street=response.get('street'), + house_number=response.get('houseNumber')).values().first() + house.pop('id') + if house.get('doors') and house.get('doors') > 1: + house['flat_cnt'] = int(house['flat_cnt'] / house['doors']) + for _num in range(house['doors']): + models.PrePlacementPoint.objects.get_or_create(address=addr, + name=f'Подъезд {_num + 1}', + matching_status=MatchingStatus.New.name, + sample_trn=False, + is_vis=True, category=cat, geometry=wkt, + status=PointStatus.Pending.name, area=rayon, + district=rayon.AO, defaults=house) + else: + models.PrePlacementPoint.objects.get_or_create(address=addr, + matching_status=MatchingStatus.New.name, + sample_trn=False, + is_vis=True, category=cat, geometry=wkt, + status=PointStatus.Pending.name, area=rayon, + district=rayon.AO, defaults=house) + else: + models.PrePlacementPoint.objects.get_or_create(address=addr, street=response.get('street'), + house_number=response.get('houseNumber'), + subject_rf=response.get('state'), + city=response.get('city'), + category=cat, geometry=wkt, sample_trn=False, + is_vis=True, + matching_status=MatchingStatus.New.name, + status=PointStatus.Pending.name, area=rayon, + district=rayon.AO) + return total, matched, problem + + def make_enrichment(self): + points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all() + groups = models.Post_and_pvzGroup.objects.all() + for point in points: + origin = point.geometry + qs = models.PlacementPoint.objects.filter(status=PointStatus.Working.name).annotate( + dist=Dist('geometry', origin)).order_by('dist') + if qs: + point.target_dist = qs[0].dist.m + point.target_post_cnt = qs.filter( + dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS) + ).count() + point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean + point.rival_post_cnt = models.Post_and_pvz.objects.filter( + category__name="Постаматы прочих сетей", include_in_ml=True, + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( + category__name="ПВЗ", include_in_ml=True, + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate( + dist=Dist('wkt', origin)).order_by('dist')[0].dist.m + point.property_price_bargains = models.OtherObjects.objects.filter( + group__name="bargains", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[ + 'param1__avg'] + offers_estate = models.OtherObjects.objects.filter( + group__name="offers_estate", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + param1__avg=Avg('param1'), param3__avg=Avg('param3')) + point.property_price_offers = offers_estate['param1__avg'] + point.property_mean_floor = offers_estate['param3__avg'] + point.property_era = models.OtherObjects.objects.filter( + group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[ + 'param2'] + point.flats_cnt = models.OtherObjects.objects.filter( + group__name="flats_cnt", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + param1__sum=Sum('param1'))['param1__sum'] + popul_home_job = models.OtherObjects.objects.filter( + group__name="popul_home_job", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + param1__sum=Sum('param1'), param3__sum=Sum('param3')) + point.popul_home = popul_home_job['param1__sum'] + point.popul_job = popul_home_job['param3__sum'] + yndx_food_cnt_amt = models.OtherObjects.objects.filter( + group__name="yndx_food_cnt_amt", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + param1__sum=Sum('param1'), param3__sum=Sum('param3')) + point.yndxfood_sum = yndx_food_cnt_amt['param1__sum'] + point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum'] + point.school_cnt = models.OtherObjects.objects.filter( + group__name="schools", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.kindergar_cnt = models.OtherObjects.objects.filter( + group__name="kindergar", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.public_stop_cnt = models.OtherObjects.objects.filter( + group__name="stops", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.sport_center_cnt = models.OtherObjects.objects.filter( + group__name="sport_centers", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.pharmacy_cnt = models.OtherObjects.objects.filter( + group__name="pharmacies", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.supermarket_cnt = models.OtherObjects.objects.filter( + group__name="supermarkets", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.supermarket_premium_cnt = models.OtherObjects.objects.filter( + group__name="supermarkets_premium", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.clinic_cnt = models.OtherObjects.objects.filter( + group__name="clinics", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.bank_cnt = models.OtherObjects.objects.filter( + group__name="banks", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.reca_cnt = models.OtherObjects.objects.filter( + group__name="recas", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.lab_cnt = models.OtherObjects.objects.filter( + group__name="labs", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.culture_cnt = models.OtherObjects.objects.filter( + group__name="cultures", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.attraction_cnt = models.OtherObjects.objects.filter( + group__name="attractions", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.mfc_cnt = models.OtherObjects.objects.filter( + group__name="public_services", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.bc_cnt = models.OtherObjects.objects.filter( + group__name="BC", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.tc_cnt = models.OtherObjects.objects.filter( + group__name="TC", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + point.business_activity = models.OtherObjects.objects.filter( + group__name="business_activity", + wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + param1__sum=Sum('param1'))['param1__sum'] + point.age_day = AGE_DAY_LIMIT + placement_point = models.PlacementPoint.objects.annotate( + dist=Dist('geometry', origin)).order_by('dist') + if placement_point: + placement_point = placement_point[0] + point.target_cnt_ao_mean = placement_point.target_cnt_ao_mean + point.save() + for group in groups: + self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance) + run_psql_command() + + @staticmethod + def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance): + post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( + distance=Dist("wkt", point.geometry)).order_by('distance').first() + d = instance_type.objects.filter(placement_point=point, + pvz_postamates_group=group).first() + if post_object: + if d: + if d.dist > post_object.distance.m: + d.dist = post_object.distance.m + d.save() + else: + instance_type.objects.create(placement_point=point, + pvz_postamates_group=group, + dist=post_object.distance.m) + + @staticmethod + def delete_preplacement_points(ids: list): + models.PrePlacementPoint.objects.filter(id__in=ids).all().delete() + + @staticmethod + def get_min_distances_to_group(postamat_id: str): + return {d['pvz_postamates_group']: d['dist'] for d in list( + models.PlacementPointPVZDistance.objects.filter(placement_point=postamat_id).values( + 'pvz_postamates_group', 'dist'))} + @staticmethod def update_points_in_radius(qs: models.PlacementPoint, new_status: str): - triggers = False for point in qs: if new_status == PointStatus.Installation.name: if point.status == PointStatus.Pending.name: @@ -30,18 +277,12 @@ class PointService: geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)), ) pnts.update(prediction_first=F('prediction_current'), target_post_cnt=F('target_post_cnt') + 1) - triggers = True elif new_status == PointStatus.Cancelled.name or new_status == PointStatus.Pending.name: if point.status == PointStatus.Installation.name: pnts = models.PlacementPoint.objects.filter( geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)), ) pnts.update(target_post_cnt=F('target_post_cnt') - 1 if F('target_post_cnt') != 0 else 0) - triggers = True - elif new_status == PointStatus.Working.name and point.status == PointStatus.Pending.name: - triggers = True - if triggers: - raschet.delay() @staticmethod def update_status(qs: models.PlacementPoint, new_status: str) -> models.PlacementPoint: @@ -60,13 +301,26 @@ class PointService: return models.PlacementPoint.objects.filter(postamat_id=postamat_id) @staticmethod - def to_excel(qs: models.PlacementPoint): - data = pd.DataFrame(list(qs.values())) + def to_excel(serializer): + data = pd.DataFrame(serializer.data) if not data.empty: if data['start_date'].any(): - data['start_date'] = data.get('start_date').dt.tz_localize(None) - data['sample_trn'] = data['sample_trn'].astype(int) + data['start_date'] = data['start_date'].apply(lambda x : pd.to_datetime(x).tz_localize(None) if x else None) + if data['sample_trn'].any(): + data['sample_trn'] = data['sample_trn'].astype(int) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) + data['min_distance_to_group'] = data['min_distance_to_group'].apply(lambda x: list(x.items())) + new_columns = data.apply(create_columns_dist, axis=1) + for ind in new_columns.columns: + expanded = new_columns[ind].apply(pd.Series) + group = models.Post_and_pvzGroup.objects.get(id=int(expanded.loc[0, 0])) + expanded[[f"group_{ind + 1}_name", f"group_{ind + 1}_category"]] = group.name, group.category.name + expanded = expanded.rename(columns={1: f"dist_to_group_{ind + 1}"}) + expanded = expanded.drop(0, axis=1) + new_columns = pd.concat([new_columns, expanded], axis=1) + new_columns = new_columns.drop(ind, axis=1) + data.drop('min_distance_to_group', axis=1, inplace=True) + data = pd.concat([data, new_columns], axis=1) with BytesIO() as b: with pd.ExcelWriter(b) as writer: data.to_excel( @@ -76,13 +330,25 @@ class PointService: return b.getvalue() @staticmethod - def to_json(qs: models.PlacementPoint): - data = pd.DataFrame(list(qs.values())) + def to_json(serializer): + data = pd.DataFrame(serializer.data) data['start_date'] = pd.to_datetime(data['start_date'], errors='coerce') data['start_date'] = data['start_date'].dt.tz_localize(None) data['sample_trn'] = data['sample_trn'].astype(int) data['geometry'] = data['geometry'].apply(lambda x: {'latitude': x[1], 'longtitude': x[0]}) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) + data['min_distance_to_group'] = data['min_distance_to_group'].apply(lambda x: list(x.items())) + new_columns = data.apply(create_columns_dist, axis=1) + for ind in new_columns.columns: + expanded = new_columns[ind].apply(pd.Series) + group = models.Post_and_pvzGroup.objects.get(id=int(expanded.loc[0, 0])) + expanded[[f"group_{ind + 1}_name", f"group_{ind + 1}_category"]] = group.name, group.category.name + expanded = expanded.rename(columns={1: f"dist_to_group_{ind + 1}"}) + expanded = expanded.drop(0, axis=1) + new_columns = pd.concat([new_columns, expanded], axis=1) + new_columns = new_columns.drop(ind, axis=1) + data.drop('min_distance_to_group', axis=1, inplace=True) + data = pd.concat([data, new_columns], axis=1) return data.to_json(orient='records') @staticmethod diff --git a/service/signals.py b/service/signals.py index 3117610..acab55f 100644 --- a/service/signals.py +++ b/service/signals.py @@ -1,5 +1,9 @@ from django.contrib.auth.models import Group from rest_registration.signals import user_registered +from service.models import OtherObjects, OtherObjectsGroup, \ + OtherObjectsCategory +from django.db.models.signals import post_save +from django.dispatch import receiver def user_created(sender, user, request, **kwargs): @@ -9,3 +13,16 @@ def user_created(sender, user, request, **kwargs): user_registered.connect(user_created) + + +@receiver(post_save, sender=OtherObjectsGroup) +def other_group_handler(sender, instance, **kwargs): + OtherObjects.objects.filter(group=instance).update(visible=instance.visible) + + +@receiver(post_save, sender=OtherObjectsCategory) +def other_category_handler(sender, instance, **kwargs): + objects = OtherObjectsGroup.objects.filter(category=instance) + for obj in objects: + obj.visible = instance.visible + obj.save() diff --git a/service/tasks.py b/service/tasks.py index 4ee9b28..5f80fca 100644 --- a/service/tasks.py +++ b/service/tasks.py @@ -4,6 +4,7 @@ import catboost import geopandas as gpd import numpy as np import pandas as pd +import shap import psycopg2 import sqlalchemy from celery import shared_task @@ -14,21 +15,56 @@ from shapely import wkb from sklearn import metrics from sklearn import model_selection as ms from sqlalchemy import text - -import requests - +from django.contrib.gis.db.models.functions import Distance from postamates.settings import AGE_DAY_LIMIT -from postamates.settings import DB_URL -from service.models import PlacementPoint - - -def log_to_telegram(msg): - requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage', json={"chat_id": "-555238820", "text": msg}) +from postamates.settings import DB_URL, STATUS_TASK_NAME +from service.models import PlacementPoint, LastMLCall +from service import models +from service.utils import log_to_telegram +import base64 +from io import StringIO +from django.core.cache import cache +from service.layer_service import LayerService +from service.service import PointService +from service.utils import run_psql_command @shared_task() -def raschet(): - log_to_telegram('start raschet') +def raschet(table_name='service_placementpoint', need_time=True): + print('start raschet') + status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME) + raschet_objs = models.RaschetObjects.objects.all() + if raschet_objs: + status.status = 'Начало расчета кол-ва ПВЗ вокруг точек' + status.save() + total = raschet_objs.count() + for _i, r_o in enumerate(raschet_objs): + obj = models.Post_and_pvz.objects.get(id=r_o.obj_id) + LayerService().count_post_pvz_for_placementpoint(obj) + status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%" + status.save() + status.status = 'Расчет кол-ва ПВЗ вокруг точек завершен' + status.save() + group_objects = models.RaschetGroups.objects.all() + group_total = group_objects.count() + if group_objects: + status.status = 'Начало расчета расстояний' + status.save() + qs = models.PlacementPoint.objects.all() + for _k, g_o in enumerate(group_objects): + g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id) + for q in qs: + PointService.calculate_dist_for_group(point=q, group=g) + status.status = "Подсчет расстояний: " + str(int(_k / group_total * 100)) + "%" + status.save() + status.status = "Подсчет расстояний завершен" + status.save() + models.RaschetObjects.objects.all().delete() + models.RaschetGroups.objects.all().delete() + # Запуск ML + status.status = 'Запуск ML' + status.save() + log_to_telegram(f'{table_name} start raschet') try: log_to_telegram('try connect to db') conn = sqlalchemy.create_engine( @@ -44,23 +80,16 @@ def raschet(): pts['geometry'] = pts['geometry'].apply(wkb.loads, hex=True) pts = gpd.GeoDataFrame(pts, geometry='geometry', crs='epsg:4326') pts = pts.to_crs('epsg:32637') - pts = pts.rename( - columns={ - 'target_cnt_nearby_mean': 'target_dist1', - 'target_age_nearby_mean': 'target_dist2', - 'yndxfood_cnt_cst': 'target_dist3', - }, - ) feats = [ 'id', 'metro_dist', 'target_dist', 'property_price_bargains', 'property_price_offers', 'property_mean_floor', - 'property_era', 'flats_cnt_2', 'flats_cnt', 'popul_home', 'popul_job', 'other_post_cnt', 'yndxfood_sum', + 'property_era', 'flats_cnt', 'popul_home', 'popul_job', 'yndxfood_sum', 'yndxfood_cnt', 'school_cnt', 'kindergar_cnt', 'target_post_cnt', 'public_stop_cnt', 'sport_center_cnt', 'pharmacy_cnt', 'supermarket_cnt', 'supermarket_premium_cnt', 'clinic_cnt', 'bank_cnt', 'reca_cnt', 'lab_cnt', 'culture_cnt', 'attraction_cnt', 'mfc_cnt', 'bc_cnt', 'tc_cnt', 'rival_pvz_cnt', 'rival_post_cnt', - 'business_activity', 'age_day', 'target_cnt_ao_mean', 'target_dist1', 'target_dist2', 'target_dist3', + 'business_activity', 'age_day', 'target_cnt_ao_mean' ] # Записи для обучения @@ -81,24 +110,6 @@ def raschet(): ) pts_trn.loc[pts_trn.target_dist > 700, 'target_dist'] = 700 - pts_trn['target_dist1'] = pts_trn.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[2])), - axis=1, - ) - pts_trn.loc[pts_trn.target_dist1 > 700, 'target_dist1'] = 700 - - pts_trn['target_dist2'] = pts_trn.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[3])), - axis=1, - ) - pts_trn.loc[pts_trn.target_dist2 > 700, 'target_dist2'] = 700 - - pts_trn['target_dist3'] = pts_trn.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[4])), - axis=1, - ) - pts_trn.loc[pts_trn.target_dist3 > 700, 'target_dist3'] = 700 - pts_trn['buf'] = pts_trn.buffer(500) pts_trn = gpd.GeoDataFrame(pts_trn, geometry='buf', crs='epsg:32637') target_post = gpd.sjoin(pts_trn, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) @@ -110,213 +121,393 @@ def raschet(): X_trn = pts_trn[feats].drop(columns=['id']) Y_trn = pts_trn[['fact']] - # Записи для инференса - pts_inf = pts.loc[(pts.status == 'Pending') | - (pts.status == 'Installation') | - (pts.status == 'Cancelled') | - ((pts.status == 'Working') & (pts.sample_trn == False))].reset_index(drop=True) - pts_inf = gpd.GeoDataFrame(pts_inf, geometry='geometry', crs='epsg:32637') - - pts_inf['buf'] = pts_inf.buffer(500) - pts_inf = gpd.GeoDataFrame(pts_inf, geometry='buf', crs='epsg:32637') - pts_target = pts.loc[(pts.status == 'Working') | - (pts.status == 'Installation') | - (pts.sample_trn == True)].reset_index(drop=True) - pts_target = pts_target[['geometry']] - pts_target['cnt'] = 1 - pts_target = gpd.GeoDataFrame(pts_target, geometry='geometry', crs='epsg:32637') - - target_feature_coords = [] - for i in range(0, len(pts_target)): - target_feature_coords.append((pts_target.geometry.x[i], pts_target.geometry.y[i])) - target_feature_coords = np.array(target_feature_coords) - - pts_inf['target_dist'] = pts_inf.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])), - axis=1, - ) - pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700 - - pts_inf['target_dist1'] = pts_inf.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[1])), - axis=1, - ) - pts_inf.loc[pts_inf.target_dist1 > 700, 'target_dist1'] = 700 - - pts_inf['target_dist2'] = pts_inf.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[2])), - axis=1, - ) - pts_inf.loc[pts_inf.target_dist2 > 700, 'target_dist2'] = 700 - pts_inf['target_dist3'] = pts_inf.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[3])), - axis=1, - ) - pts_inf.loc[pts_inf.target_dist3 > 700, 'target_dist3'] = 700 - - pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True) - target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) - target_post = target_post.rename(columns={'cnt': 'target_post_cnt'}) - pts_inf = pts_inf.drop(columns=['target_post_cnt']) - pts_inf = pts_inf.join(target_post.set_index('id'), on='id') - pts_inf['age_day_init'] = pts_inf['age_day'] - pts_inf['age_day'] = 240 - X_inf = pts_inf[feats] - - seeds = [3, 99, 87, 21, 15] - - # Обучение, инференс - r2_scores = [] - mapes = [] - y_infers = [] - - for i in seeds: - x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i) - model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i) - model.fit(x_trn, y_trn, verbose=False) - r2_score = metrics.r2_score(y_test, model.predict(x_test)) - mape = metrics.mean_absolute_percentage_error(y_test, model.predict(x_test)) - if ((r2_score > 0.45) & (mape < 0.25)): - r2_scores.append(r2_score) - mapes.append(mape) - y_infers.append(model.predict(X_inf.drop(columns=['id']))) - - current_pred = sum(y_infers) / 5 - - # Обновление полей по результатам работы модели - update_fields = pts_inf[ - [ - 'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first', - 'prediction_first', - ] - ] - update_fields = update_fields.join( - pd.concat( + status.status = 'Записи для инференса' + status.save() + # Записи для инференса + if table_name == 'service_placementpoint': + pts_inf = pts.loc[(pts.status == 'Pending') | + (pts.status == 'Installation') | + (pts.status == 'Cancelled') | + ((pts.status == 'Working') & (pts.sample_trn == False))].reset_index(drop=True) + elif table_name == 'service_preplacementpoint': + pts_inf = pd.DataFrame(conn.connect().execute(text(f"select * from {table_name}"))) + pts_inf = pts_inf.loc[pts_inf.matching_status == 'New'].reset_index(drop=True) + pts_inf['geometry'] = pts_inf['geometry'].apply(wkb.loads, hex=True) + pts_inf = gpd.GeoDataFrame(pts_inf, geometry='geometry', crs='epsg:4326') + if len(pts_inf) > 0: + pts_inf = pts_inf.to_crs('epsg:32637') + + pts_inf['buf'] = pts_inf.buffer(500) + pts_inf = gpd.GeoDataFrame(pts_inf, geometry='buf', crs='epsg:32637') + pts_target = pts.loc[(pts.status == 'Working') | + (pts.status == 'Installation')].reset_index(drop=True) + pts_target = pts_target[['geometry']] + pts_target['cnt'] = 1 + pts_target = gpd.GeoDataFrame(pts_target, geometry='geometry', crs='epsg:32637') + + target_feature_coords = [] + for i in range(0, len(pts_target)): + target_feature_coords.append((pts_target.geometry.x[i], pts_target.geometry.y[i])) + target_feature_coords = np.array(target_feature_coords) + + pts_inf['target_dist'] = pts_inf.apply( + lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])), + axis=1, + ) + pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700 + + pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True) + target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) + target_post = target_post.rename(columns={'cnt': 'target_post_cnt'}) + pts_inf = pts_inf.drop(columns=['target_post_cnt']) + pts_inf = pts_inf.join(target_post.set_index('id'), on='id') + pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0) + pts_inf['age_day_init'] = pts_inf['age_day'] + pts_inf['age_day'] = 240 + X_inf = pts_inf[feats] + + seeds = [3, 99, 87, 21, 15] + + # Обучение, инференс + r2_scores = [] + mapes = [] + y_infers = [] + status.status = 'Обучение inference 0%' + status.save() + for i in seeds: + status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%' + status.save() + x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i) + model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i) + model.fit(x_trn, y_trn, verbose=False) + r2_score = metrics.r2_score(y_test, model.predict(x_test)) + mape = metrics.mean_absolute_percentage_error(y_test, model.predict(x_test)) + if ((r2_score > 0.45) & (mape < 0.25)): + r2_scores.append(r2_score) + mapes.append(mape) + y_infers.append(model.predict(X_inf.drop(columns=['id']))) + status.status = 'Обучение inference 100%' + current_pred = sum(y_infers) / 5 + + # расчет шапов + explainer = shap.TreeExplainer(model) + shap_values = explainer(X_inf.drop(columns=['id'])) + shap_fields = pd.DataFrame(shap_values.values) + shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap' + shap_fields = shap_fields.drop(columns = ['age_day_shap']) + shap_fields['sum'] = abs(shap_fields).sum(axis=1) + shap_fields = round(shap_fields.iloc[:,:32].div(shap_fields['sum'], axis=0)*100, 2) + + # Обновление полей по результатам работы модели + update_fields = pts_inf[ [ - X_inf[['id']], - pd.DataFrame({'prediction_current': current_pred}), - ], + 'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first', + 'prediction_first', 'target_post_cnt', 'target_dist' + ] + ] + update_fields = update_fields.join( + pd.concat( + [ + X_inf[['id']], + pd.DataFrame({'prediction_current': current_pred}), + ], + axis=1, + ).set_index('id'), + on='id', + ) + update_fields['prediction_current'] = update_fields['prediction_current'].astype(int) + + days_x = np.array([0, 30, 60, 90, 120, 150, 180, 210, 240, 270]) + perc_y = np.array([0, 0.15, 0.20, 0.30, 0.60, 0.70, 0.70, 0.75, 0.75, 0.80]) + spl = interpolate.splrep(days_x, perc_y) + + update_fields['plan_first'] = update_fields.apply( + lambda x: (x.prediction_first * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), axis=1, - ).set_index('id'), - on='id', - ) - update_fields['prediction_current'] = update_fields['prediction_current'].astype(int) - - days_x = np.array([0, 30, 60, 90, 120, 150, 180, 210, 240, 270]) - perc_y = np.array([0, 0.15, 0.20, 0.30, 0.60, 0.70, 0.70, 0.75, 0.75, 0.80]) - spl = interpolate.splrep(days_x, perc_y) - - update_fields['plan_first'] = update_fields.apply( - lambda x: (x.prediction_first * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), - axis=1, - ) - update_fields['plan_current'] = update_fields.apply( - lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), - axis=1, - ) - update_fields['delta_first'] = update_fields.apply( - lambda x: ((x.fact - x.plan_first) / x.plan_first * 100 if x.status == 'Working' else 0), - axis=1, - ) - update_fields['delta_current'] = update_fields.apply( - lambda x: ((x.fact - x.plan_current) / x.plan_current * 100 if x.status == 'Working' else 0), - axis=1, - ) + ) + update_fields['plan_current'] = update_fields.apply( + lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), + axis=1, + ) + update_fields['delta_first'] = update_fields.apply( + lambda x: ((x.fact - x.plan_first) / x.plan_first * 100 if x.status == 'Working' else 0), + axis=1, + ) + update_fields['delta_current'] = update_fields.apply( + lambda x: ((x.fact - x.plan_current) / x.plan_current * 100 if x.status == 'Working' else 0), + axis=1, + ) - update_fields_working = update_fields.loc[update_fields.status == 'Working'].reset_index(drop=True) - update_fields_working = update_fields_working.fillna(0) + update_fields_working = update_fields.loc[update_fields.status == 'Working'].reset_index(drop=True) + update_fields_working = update_fields_working.fillna(0) + connection.close() except Exception as e: log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}') log_to_telegram('Начинается обновление полей в базе') - # Загрузка в базу обновленных значений - try: - log_to_telegram('Подключение к базе данных 2') - conn2 = psycopg2.connect( - database=os.getenv('POSTGRES_DB', 'postgres'), user=os.getenv('POSTGRES_USER', 'postgres'), - password=os.getenv('POSTGRES_PASSWORD', 'postgres'), - host=os.getenv('POSTGRES_HOST', 'postgres'), port=os.getenv('POSTGRES_PORT', 'postgres'), - options='-c search_path=public', - ) - cursor = conn2.cursor() - except: - log_to_telegram('Не удалось подключиться к базе данных') - - # prediction_current - update_records1 = [] - for i in range(0, len(update_fields)): - update_records1.append((int(update_fields.prediction_current[i]), int(update_fields.id[i]))) - sql_update_query = """Update service_placementpoint set prediction_current = %s where id = %s""" - try: - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) - conn2.commit() - except Exception: - cursor.execute('ROLLBACK') - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) - conn2.commit() - - # plan_first - update_records2 = [] - for i in range(0, len(update_fields_working)): - update_records2.append((int(update_fields_working.plan_first[i]), int(update_fields_working.id[i]))) - sql_update_query = """Update service_placementpoint set plan_first = %s where id = %s""" - try: - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2) - conn2.commit() - except Exception: - cursor.execute('ROLLBACK') - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2) - conn2.commit() - - # plan_current - update_records3 = [] - for i in range(0, len(update_fields_working)): - update_records3.append((int(update_fields_working.plan_current[i]), int(update_fields_working.id[i]))) - sql_update_query = """Update service_placementpoint set plan_current = %s where id = %s""" - try: - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3) - conn2.commit() - except Exception: - cursor.execute('ROLLBACK') - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3) - conn2.commit() - - # delta_first - update_records4 = [] - for i in range(0, len(update_fields_working)): - update_records4.append((int(update_fields_working.delta_first[i]), int(update_fields_working.id[i]))) - sql_update_query = """Update service_placementpoint set delta_first = %s where id = %s""" - try: - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4) - conn2.commit() - except Exception: - cursor.execute('ROLLBACK') - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4) - conn2.commit() - - # delta_current - update_records5 = [] - for i in range(0, len(update_fields_working)): - update_records5.append((int(update_fields_working.delta_current[i]), int(update_fields_working.id[i]))) - sql_update_query = """Update service_placementpoint set delta_current = %s where id = %s""" - try: - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5) - conn2.commit() - except Exception: - cursor.execute('ROLLBACK') - psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5) - conn2.commit() + if len(pts_inf) > 0: + status.status = 'Перерасчет ML: 50%' + status.save() + # Загрузка в базу обновленных значений + try: + log_to_telegram('Подключение к базе данных 2') + conn2 = psycopg2.connect( + database=os.getenv('POSTGRES_DB', 'postgres'), user=os.getenv('POSTGRES_USER', 'postgres'), + password=os.getenv('POSTGRES_PASSWORD', 'postgres'), + host=os.getenv('POSTGRES_HOST', 'postgres'), port=os.getenv('POSTGRES_PORT', 'postgres'), + options='-c search_path=public', + ) + cursor = conn2.cursor() + except: + conn2 = None + log_to_telegram('Не удалось подключиться к базе данных') + + if conn2 is not None: + # апдейт шапов + update_fields_shap = pd.concat([shap_fields, update_fields[['id']]], axis=1) + update_records0 = [] + for i in range(0, len(update_fields_shap)): + update_records1 = [] + for n in list(update_fields_shap): + update_records1.append(int(update_fields_shap[n][i])) + update_records0.append(tuple(update_records1)) + shap_fields_name = str(list(shap_fields))[1:-1].replace("'", "").replace(',', '=%s,') + sql_update_query = f"""Update {table_name} set {shap_fields_name} = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records0) + conn2.commit() + except: + cursor.execute("ROLLBACK") + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records0) + conn2.commit() + + # target_post_cnt + update_records1 = [] + for i in range(0, len(update_fields)): + update_records1.append((int(update_fields.target_post_cnt[i]), int(update_fields.id[i]))) + sql_update_query = f"""Update {table_name} set target_post_cnt = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) + conn2.commit() + except: + cursor.execute("ROLLBACK") + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) + conn2.commit() + + # target_dist + update_records1 = [] + for i in range(0, len(update_fields)): + update_records1.append((int(update_fields.target_dist[i]), int(update_fields.id[i]))) + sql_update_query = f"""Update {table_name} set target_dist = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) + conn2.commit() + except: + cursor.execute("ROLLBACK") + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) + conn2.commit() + + # prediction_current + update_records1 = [] + for i in range(0, len(update_fields)): + update_records1.append((int(update_fields.prediction_current[i]), int(update_fields.id[i]))) + sql_update_query = f"""Update {table_name} set prediction_current = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) + conn2.commit() + except Exception: + cursor.execute('ROLLBACK') + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) + conn2.commit() + + # plan_first + update_records2 = [] + for i in range(0, len(update_fields_working)): + update_records2.append((int(update_fields_working.plan_first[i]), int(update_fields_working.id[i]))) + sql_update_query = f"""Update {table_name} set plan_first = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2) + conn2.commit() + except Exception: + cursor.execute('ROLLBACK') + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2) + conn2.commit() + + # plan_current + update_records3 = [] + for i in range(0, len(update_fields_working)): + update_records3.append((int(update_fields_working.plan_current[i]), int(update_fields_working.id[i]))) + sql_update_query = f"""Update {table_name} set plan_current = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3) + conn2.commit() + except Exception: + cursor.execute('ROLLBACK') + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3) + conn2.commit() + + # delta_first + update_records4 = [] + for i in range(0, len(update_fields_working)): + update_records4.append((int(update_fields_working.delta_first[i]), int(update_fields_working.id[i]))) + sql_update_query = f"""Update {table_name} set delta_first = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4) + conn2.commit() + except Exception: + cursor.execute('ROLLBACK') + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4) + conn2.commit() + + # delta_current + update_records5 = [] + for i in range(0, len(update_fields_working)): + update_records5.append((int(update_fields_working.delta_current[i]), int(update_fields_working.id[i]))) + sql_update_query = f"""Update {table_name} set delta_current = %s where id = %s""" + try: + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5) + conn2.commit() + except Exception: + cursor.execute('ROLLBACK') + psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5) + conn2.commit() + conn2.close() + cache.clear() + else: + log_to_telegram('len(pts_inf) <= 0') + run_psql_command() log_to_telegram('end raschet') - + status.status = 'Перерасчет ML завершен' + status.save() + if need_time: + LastMLCall.objects.all().delete() + LastMLCall.objects.create() + + + +@shared_task +def load_post_and_pvz(obj_id: int): + file = models.TempFiles.objects.get(id=obj_id) + status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка ПВЗ и Постаматов') + excel_file = base64.b64decode(file.data) + df = pd.read_excel(excel_file) + df = df.replace(np.nan, None) + df = df.replace('NaT', None) + df.columns = df.columns.str.lower() + data_len = df.shape[0] + for _ind, row in enumerate(df.to_dict('records')): + status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%" + status.save() + category = row.get('category') + group = row.get('group') + if category: + cat, _ = models.Post_and_pvzCategory.objects.get_or_create(name=category) + if group: + gr, _ = models.Post_and_pvzGroup.objects.get_or_create(name=group, category=cat) + row['category'] = cat + row['group'] = gr + lon = str(row.pop('lon')) + lat = str(row.pop("lat")) + row['wkt'] = "POINT(" + lon + " " + lat + ")" + models.Post_and_pvz.objects.get_or_create(**row) + status.status = "Загрузка данных завершена" + status.save() + groups = df[['group', 'category']].drop_duplicates().to_dict(orient='records') + points = models.PlacementPoint.objects.all() + num_points = points.count() + total = len(groups) * num_points + for _i, gr in enumerate(groups): + group = models.Post_and_pvzGroup.objects.get(name=gr['group'], category__name=gr['category']) + for _j, point in enumerate(points): + status.status = "Подсчет расстояний: " + str(int((num_points * _i + _j) / total * 100)) + "%" + status.save() + post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( + distance=Distance("wkt", point.geometry)).order_by('distance').first() + d = models.PlacementPointPVZDistance.objects.filter(placement_point=point, + pvz_postamates_group=group).first() + if d: + if d.dist > post_object.distance.m: + d.dist = post_object.distance.m + d.save() + else: + models.PlacementPointPVZDistance.objects.create(placement_point=point, pvz_postamates_group=group, + dist=post_object.distance.m) + status.status = "Подсчет расстояний завершен" + status.save() + point_qs = models.PlacementPoint.objects.all() + data_len = models.PlacementPoint.objects.count() + for _ind, point in enumerate(point_qs): + status.status = "Пересчет параметров точек: " + str(int(_ind / data_len * 100)) + "%" + status.save() + LayerService.count_post_pvz(point) + status.status = "Завершено" + cache.clear() + status.save() + @shared_task() def add_age_day(): - qs = PlacementPoint.objects - c1 = qs.filter(sample_trn=True).count() + qs = PlacementPoint.objects.filter(status='Working') + # c1 = qs.filter(sample_trn=True).count() qs.update(age_day=F('age_day') + 1) qs2 = qs.filter(age_day__gt=AGE_DAY_LIMIT) qs2.update(sample_trn=True) - c2 = PlacementPoint.objects.filter(sample_trn=True).count() - if c2 - c1 != 0: - raschet.delay() + # c2 = PlacementPoint.objects.filter(sample_trn=True).count() + # if c2 - c1 != 0: + # raschet.delay() + + +@shared_task() +def load_other_objects(obj_id: int): + file = models.TempFiles.objects.get(id=obj_id) + status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Прочих объектов') + excel_file = base64.b64decode(file.data) + df = pd.read_excel(excel_file) + df = df.replace(np.nan, None) + df = df.replace('NaT', None) + df.columns = df.columns.str.lower() + data_len = df.shape[0] + for _ind, row in enumerate(df.to_dict('records')): + status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%" + status.save() + category = row.get('category') + group = row.get('group') + if category: + cat, _ = models.OtherObjectsCategory.objects.get_or_create(name=category) + if group: + gr, _ = models.OtherObjectsGroup.objects.get_or_create(name=group, category=cat) + row['category'] = cat + row['group'] = gr + lon = str(row.pop('lon')) + lat = str(row.pop("lat")) + row['wkt'] = "POINT(" + lon + " " + lat + ")" + models.OtherObjects.objects.get_or_create(**row) + status.status = "Загрузка данных завершена" + cache.clear() + status.save() + + +@shared_task() +def load_data(obj_id: int): + status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Точек') + file = models.TempFiles.objects.get(id=obj_id) + csv_file = base64.b64decode(file.data) + models.PlacementPoint.objects.all().delete() + s = str(csv_file, 'utf-8') + data = StringIO(s) + df = pd.read_csv(data, delimiter=';') + df = df.replace(np.nan, None) + df = df.replace('NaT', None) + data_len = df.shape[0] + for _ind, row in enumerate(df.to_dict('records')): + status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%" + status.save() + data = { + k: row[k] for k in row.keys() if + k not in ['id', 'location_id', 'area', 'district', 'age_month'] + } + models.PlacementPoint.objects.create(**data) + status.status = "Загрузка данных завершена" + status.save() + models.TempFiles.objects.all().delete() diff --git a/service/urls.py b/service/urls.py index 4736770..deadce0 100644 --- a/service/urls.py +++ b/service/urls.py @@ -1,42 +1,25 @@ from django.conf.urls import url -from django.urls import include -from django.urls import path -from django.urls import re_path -from drf_yasg import openapi -from drf_yasg.views import get_schema_view -from rest_framework import permissions from rest_framework import routers - -from postamates.settings import CACHE_TIMEOUT from service import views router = routers.DefaultRouter() -router.register('', views.PlacementPointViewSet) -info = openapi.Info( - title='Snippets API', - default_version='v1', - description='Test description', - terms_of_service='https://www.google.com/policies/terms/', - contact=openapi.Contact(email='contact@snippets.local'), - license=openapi.License(name='BSD License'), -) -schema_view = get_schema_view( - info, - url='https://postamates.spatiality.website/', - public=True, - permission_classes=[permissions.AllowAny], -) +router.register('placement_points', views.PlacementPointViewSet) +router.register('pre_placement_points', views.PrePlacementPointViewSet) +router.register('ao_rayons', views.AOViewSet) +router.register('postamate_and_pvz_groups', views.PostAndPVZCategoryViewSet) +router.register('other_object_groups', views.OtherObjectsCategoryViewSet) + +urlpatterns = router.urls -urlpatterns = [ - path('placement_points/', include([*router.urls]), name='placement_points'), - path('ao_rayons', views.AOViewSet.as_view({'get': 'list'}), name='ao_and_rayons'), +urlpatterns += [ url(r'load_csv/', views.refresh_placement_points.as_view(), name='upload_placement_points'), - url(r'upload_ao_and_rayons/', views.load_ao_and_rayons.as_view(), name='upload_ao_and_rayons'), - url(r'upload_rivals/', views.upload_rivals, name='upload_rivals'), - url(r'upload_dist/', views.upload_dist, name='upload_dist'), + url(r'upload_post_and_pvz/', views.upload_post_and_pvz, name='upload_post_and_pvz'), + url(r'upload_other_objects/', views.upload_other_objects, name='upload_other_objects'), + url(r'upload_houses/', views.upload_houses, name='upload_houses'), url(r'me/', views.get_current_user, name='me'), - re_path(r'^swagger(?P\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'), - re_path(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=CACHE_TIMEOUT), name='schema-swagger-ui'), + url('download_pvz_template/', views.download_pvz_template, name='download_pvz_template'), + url('download_other_template/', views.download_other_template, name='download_other_template'), + url('avg_bi_values/', views.AvgBiValuesViewSet.as_view(), name='avg_bi_values'), ] USE_X_FORWARDED_HOST = True SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') diff --git a/service/utils.py b/service/utils.py index 2baaa24..3549510 100644 --- a/service/utils.py +++ b/service/utils.py @@ -1,24 +1,54 @@ +import os + import geojson import numpy as np import pandas as pd from django.contrib.gis.geos import GEOSGeometry from geojson import MultiPolygon -from tqdm import tqdm from service import models +import requests +from tqdm import tqdm +from django.core.cache import cache +from django.conf import settings +from rest_framework.response import Response +from rest_framework.viewsets import ReadOnlyModelViewSet +from django.db.models import Avg, Min, Max -def load_data(filepath: str): - models.PlacementPoint.objects.all().delete() - df = pd.read_csv(filepath) - df = df.replace(np.nan, None) - df = df.replace('NaT', None) - for row in tqdm(df.to_dict('records'), desc='Loading data...'): - data = { - k: row[k] for k in row.keys() if - k not in ['id', 'location_id', 'area', 'district', 'age_month'] - } - models.PlacementPoint.objects.create(**data) +import psycopg2 +from postamates.settings import DB_URL + + +def run_sql_command(command): + connection = psycopg2.connect( + DB_URL + ) + try: + cursor = connection.cursor() + cursor.execute(command) + connection.commit() + except psycopg2.Error as e: + print("Error executing command:", e) + finally: + cursor.close() + connection.close() + + +def run_psql_command(): + connection = psycopg2.connect( + DB_URL + ) + try: + cursor = connection.cursor() + command = "REFRESH MATERIALIZED VIEW public.points_with_dist;REFRESH MATERIALIZED VIEW public.prepoints_with_dist;" + cursor.execute(command) + connection.commit() + except psycopg2.Error as e: + print("Error executing command:", e) + finally: + cursor.close() + connection.close() def load_ao_and_rayons( @@ -43,20 +73,90 @@ def load_ao_and_rayons( models.Rayon.objects.create(**{'name': name, 'polygon': GEOSGeometry(str(MultiPolygon(coords))), 'AO': ao}) -def load_rivals(filepath: str): - models.Rivals.objects.all().delete() - df = pd.read_csv(filepath) - df = df.replace(np.nan, None) - df = df.replace('NaT', None) - df.columns = df.columns.str.lower() - for row in tqdm(df.to_dict('records'), desc='Loading data...'): - models.Rivals.objects.create(**row) - - def load_dist(filepath: str): models.PointDist.objects.all().delete() df = pd.read_csv(filepath) - for row in tqdm(df.to_dict('records'), desc='Loading data...'): + for row in df.to_dict('records'): row['id1'] = models.PlacementPoint.objects.get(pk=row.get('id1')) row['id2'] = models.PlacementPoint.objects.get(pk=row.get('id2')) models.PointDist.objects.create(**row) + + +def log_to_telegram(msg): + requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage', + json={"chat_id": "-555238820", "text": str(settings.DOMAIN) + '\n' + msg}) + + +def cached_func(key, func, timeout=settings.CACHE_TIMEOUT, *args, **kwargs): + d = cache.get(key) + if d is None: + d = func(*args, **kwargs) + cache.set(key, d, timeout) + return d + + +def get_middle_bi_values(): + fields_to_aggregate = [ + 'target_dist_shap', + 'target_post_cnt_shap', + 'target_cnt_ao_mean_shap', + 'rival_pvz_cnt_shap', + 'rival_post_cnt_shap', + 'metro_dist_shap', + 'property_price_bargains_shap', + 'property_price_offers_shap', + 'property_mean_floor_shap', + 'property_era_shap', + 'flats_cnt_shap', + 'popul_home_shap', + 'popul_job_shap', + 'yndxfood_sum_shap', + 'yndxfood_cnt_shap', + 'school_cnt_shap', + 'kindergar_cnt_shap', + 'public_stop_cnt_shap', + 'sport_center_cnt_shap', + 'pharmacy_cnt_shap', + 'supermarket_cnt_shap', + 'supermarket_premium_cnt_shap', + 'clinic_cnt_shap', + 'bank_cnt_shap', + 'reca_cnt_shap', + 'lab_cnt_shap', + 'culture_cnt_shap', + 'attraction_cnt_shap', + 'mfc_cnt_shap', + 'bc_cnt_shap', + 'tc_cnt_shap', + 'business_activity_shap' + ] + + aggregations = {} + for field_name in fields_to_aggregate: + aggregations[f'avg_{field_name}'] = Avg(field_name) + aggregations[f'min_{field_name}'] = Min(field_name) + aggregations[f'max_{field_name}'] = Max(field_name) + result = models.PlacementPoint.objects.aggregate(**aggregations) + return result + + +class CustomReadOnlyModelViewSet(ReadOnlyModelViewSet): + def list(self, request, *args, **kwargs): + def f(): + return ReadOnlyModelViewSet.list(self, request, *args, **kwargs).data + + d = cached_func(self.__class__.__name__, f) + return Response(d) + + +def create_columns_dist(row): + return pd.Series(row['min_distance_to_group']) + + +def load_houses(filepath: str): + models.House.objects.all().delete() + df = pd.read_csv(filepath) + df = df.replace(np.nan, None) + df = df.replace('NaT', None) + for row in df.to_dict('records'): + models.House.objects.create(**row) diff --git a/service/views.py b/service/views.py index 17727b7..60b97ef 100644 --- a/service/views.py +++ b/service/views.py @@ -14,32 +14,109 @@ from rest_framework.viewsets import ReadOnlyModelViewSet from postamates.settings import AGE_DAY_BORDER from postamates.settings import EXCEL_EXPORT_FILENAME -from postamates.settings import JSON_EXPORT_FILENAME +from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME from service import models from service import pagination from service import serializers from service import utils -from service.enums import PointStatus +from service.enums import PointStatus, MatchingStatus from service.permissions import UserPermission from service.service import PointService -from service.tasks import raschet -from service.utils import load_data +from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data from rest_framework.permissions import AllowAny +from django.shortcuts import redirect +from django.contrib import messages +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import filters +from service.utils import CustomReadOnlyModelViewSet +from django.db.models import Min, Max +import os +from django.forms.models import model_to_dict +import base64 -class AOViewSet(ReadOnlyModelViewSet): +class AOViewSet(CustomReadOnlyModelViewSet): serializer_class = serializers.AOSerializer queryset = models.AO.objects permission_classes = [AllowAny] +class PostAndPVZCategoryViewSet(CustomReadOnlyModelViewSet): + serializer_class = serializers.PostAndPVZCategorySerializer + queryset = models.Post_and_pvzCategory.objects + + +class OtherObjectsCategoryViewSet(CustomReadOnlyModelViewSet): + serializer_class = serializers.OtherObjectsCategorySerializer + queryset = models.OtherObjectsCategory.objects + + class PlacementPointViewSet(ReadOnlyModelViewSet): serializer_class = serializers.PlacementPointSerializer queryset = models.PlacementPoint.objects pagination_class = pagination.MyPagination permission_classes = [UserPermission] + filter_backends = [DjangoFilterBackend, filters.OrderingFilter, filters.SearchFilter] + + @property + def filterset_fields(self): + model_cls = self.queryset.model + fieldset = { + field.name: [ + "exact", + "gt", + "gte", + "lt", + "lte", + "in", + "iexact", + "startswith", + "istartswith", + "endswith", + "iendswith", + "regex", + "iregex", + "isnull", + "contains", + "icontains", + ] + for field in [ + field + for field in model_cls._meta.get_fields() + if field.get_internal_type() + not in [ + "JSONField", + "ForeignKey", + "ManyToManyField", + "OneToOneField", + "PointField" + ] + ] + } + + # filters for model relations + for field in [ + field + for field in model_cls._meta.get_fields() + if field.get_internal_type() + in [ + "ForeignKey", + "ManyToManyField", + "OneToOneField", + ] + ]: + fieldset[field.name] = [ + "exact", + "gt", + "gte", + "lt", + "lte", + ] + + return fieldset def get_queryset(self): + basename = self.request.parser_context['view'].basename qs = self.queryset.all().order_by('id') location_ids = self.request.GET.get('location_ids[]') prediction_first = self.request.GET.get('prediction_first[]') @@ -57,6 +134,8 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): delta_current = self.request.GET.get('delta_current[]') rayons = self.request.GET.get('area[]') aos = self.request.GET.get('district[]') + group_dists_lt = self.request.GET.getlist('dist_to_group__lt') + group_dists_gt = self.request.GET.getlist('dist_to_group__gt') if location_ids: location_ids = list(location_ids.split(',')) qs = qs.filter(pk__in=location_ids) @@ -104,33 +183,77 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): qs = qs.filter(~Q(pk__in=excluded)) if included: inclded = list(included.split(',')) - qs2 = models.PlacementPoint.objects.filter(pk__in=inclded).all() + qs2 = self.queryset.filter(pk__in=inclded).all() qs = (qs | qs2).distinct() + if group_dists_lt: + g_d = [list(g.split(',')) for g in group_dists_lt] + for group in g_d: + if basename == 'preplacementpoint': + filtered_points = list( + models.PrePlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]), + dist__lt=int(group[1])).values_list( + 'placement_point__id', flat=True)) + else: + filtered_points = list( + models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]), + dist__lt=int(group[1])).values_list( + 'placement_point__id', flat=True)) + qs = qs.filter(id__in=filtered_points) + if group_dists_gt: + g_d = [list(g.split(',')) for g in group_dists_gt] + for group in g_d: + if basename == 'preplacementpoint': + filtered_points = list( + models.PrePlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]), + dist__gt=int(group[1])).values_list( + 'placement_point__id', flat=True)) + else: + filtered_points = list( + models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]), + dist__gt=int(group[1])).values_list( + 'placement_point__id', flat=True)) + qs = qs.filter(id__in=filtered_points) return qs + @action(methods=['get'], detail=False) + def get_filterset_fields(self, request, *args, **kwargs): + return Response(self.filterset_fields, status=HTTPStatus.OK) + @action(detail=False, methods=['get']) def filters(self, request): - qs = self.get_queryset() - keys = ( - 'age_day', 'prediction_first', 'prediction_current', - 'plan_first', 'plan_current', 'fact', 'delta_first', - 'delta_current', 'flat_cnt', 'year_bld', 'levels', - 'doors', 'flats_cnt', 'popul_home', 'popul_job', - 'other_post_cnt', 'target_post_cnt', 'yndxfood_cnt', - 'yndxfood_sum', 'yndxfood_cnt_cst', - ) - temp_data = { - key: [ - x for x in list(set(qs.values_list(key, flat=True))) if - x is not None - ] - for key in keys - } - data = { - key: [ - min(temp_data[key]), max(temp_data[key]), - ] if temp_data[key] else [0, 100] for key in keys - } + def get_filter_data(): + qs = self.get_queryset() + keys = ( + 'age_day', 'prediction_first', 'prediction_current', + 'plan_first', 'plan_current', 'fact', 'delta_first', + 'delta_current', 'flat_cnt', 'year_bld', 'levels', + 'doors', 'flats_cnt', 'popul_home', 'popul_job', + 'other_post_cnt', 'target_post_cnt', 'yndxfood_cnt', + 'yndxfood_sum', 'yndxfood_cnt_cst', 'rival_post_cnt', 'rival_pvz_cnt', 'tc_cnt', 'culture_cnt', + 'mfc_cnt', 'public_stop_cnt' + , 'supermarket_cnt', 'target_dist', 'metro_dist' + ) + temp_data = { + key: [ + x for x in list(set(qs.values_list(key, flat=True))) if + x is not None + ] + for key in keys + } + data = { + key: [ + min(temp_data[key]), max(temp_data[key]), + ] if temp_data[key] else [0, 100] for key in keys + } + data['dist_to_groups'] = [{'group_id': d['pvz_postamates_group'], 'dist': [d['min_dist'], d['max_dist']]} + for d in list( + models.PlacementPointPVZDistance.objects.values('pvz_postamates_group').annotate( + min_dist=Min('dist'), max_dist=Max('dist')))] + + return data + + data = get_filter_data() + return Response(data, status=HTTPStatus.OK) @action(detail=False, methods=['get']) @@ -194,9 +317,10 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): @action(detail=False, methods=['get']) def to_excel(self, request): qs = self.get_queryset() + serializer = self.serializer_class(qs, many=True) filename = EXCEL_EXPORT_FILENAME res = HttpResponse( - PointService.to_excel(qs), + PointService.to_excel(serializer), content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', ) res['Content-Disposition'] = f'attachment; filename={filename}' @@ -205,8 +329,9 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): @action(detail=False, methods=['get']) def to_json(self, request): qs = self.get_queryset() + serializer = self.serializer_class(qs, many=True) filename = JSON_EXPORT_FILENAME - response = HttpResponse(PointService.to_json(qs), content_type='application/json') + response = HttpResponse(PointService.to_json(serializer), content_type='application/json') response['Content-Disposition'] = f'attachment; filename={filename}' return response @@ -220,14 +345,99 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): raschet.delay() return Response('Sucess', status=HTTPStatus.OK) + @action(detail=False, methods=['get']) + def last_time_ml_run(self, request): + status = models.TaskStatus.objects.filter(task_name=STATUS_TASK_NAME).first() + st = None + if status: + st = status.status + return Response({'last_time': models.LastMLCall.objects.first().dt, 'task_status': st}, status=HTTPStatus.OK) + + +class PrePlacementPointViewSet(PlacementPointViewSet): + queryset = models.PrePlacementPoint.objects + serializer_class = serializers.PrePlacementPointSerializer + + def get_queryset(self): + qs = super().get_queryset() + matching_status = self.request.GET.get('matching_status') + if matching_status: + qs = qs.filter(matching_status=matching_status) + return qs + + @action(detail=False, methods=['post']) + def load_matching_file(self, request): + file = request.FILES['file'].file + file_bytes = file.read() + excel_base64 = base64.b64encode(file_bytes).decode() + obj = models.TempFiles.objects.create(data=excel_base64) + return Response( + {'id': obj.id}, + status=HTTPStatus.OK, + ) + + @action(detail=False, methods=['post']) + def start_matching(self, request): + file_id = request.POST['id'] + total, matched, problem = PointService().start_mathing(file_id) + PointService().make_enrichment() + raschet('service_preplacementpoint', need_time=False) + return Response( + {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}}, + status=HTTPStatus.OK, + ) + + @action(detail=False, methods=['post']) + def move_points(self, request): + qs = self.get_queryset() + qs = qs.filter(matching_status=MatchingStatus.New.name) + for q in qs: + obj = model_to_dict(q) + obj.pop('matching_status') + obj.pop('id') + ao = obj.pop('district') + rayon = obj.pop('area') + obj['district'] = models.AO.objects.get(id=ao) + obj['area'] = models.Rayon.objects.get(id=rayon) + obj['age_day'] = 1 + new_obj = models.PlacementPoint.objects.create(**obj) + dists = models.PrePlacementPointPVZDistance.objects.filter(placement_point=q).all() + for d in dists: + models.PlacementPointPVZDistance.objects.get_or_create(placement_point=new_obj, + pvz_postamates_group=d.pvz_postamates_group, + dist=d.dist) + models.PrePlacementPoint.objects.all().delete() + return Response(status=HTTPStatus.OK, ) + + @action(detail=False, methods=['delete']) + def delete_points(self, request): + ids = request.POST.get('ids') + if ids: + ids = ids.split(',') + PointService.delete_preplacement_points(ids) + else: + models.PrePlacementPoint.objects.all().delete() + return Response(status=HTTPStatus.OK, ) + + @action(detail=False, methods=['get']) + def download_template(self, request): + image_buffer = open('PrePlacementPoints.xlsx', "rb").read() + response = HttpResponse(image_buffer, content_type='xlsx') + response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx') + return response + class refresh_placement_points(APIView): @staticmethod def post(request): warnings.filterwarnings('ignore') - file = request.FILES['file'] - load_data(file) - return Response(status=HTTPStatus.OK) + file = request.FILES['file'].file + file_bytes = file.read() + csv_base64 = base64.b64encode(file_bytes).decode() + obj = models.TempFiles.objects.create(data=csv_base64) + load_data.delay(obj.id) + messages.success(request, 'Файл точек успешно загружен') + return redirect('/admin') class load_ao_and_rayons(APIView): @@ -237,15 +447,40 @@ class load_ao_and_rayons(APIView): file_ao = request.FILES['file_ao'] file_rayon = request.FILES['file_rayon'] utils.load_ao_and_rayons(file_ao, file_rayon) - return Response(status=HTTPStatus.OK) + messages.success(request, 'Файл АО и Районов успешно загружен') + return redirect('/admin') + + +class AvgBiValuesViewSet(APIView): + @staticmethod + def get(request): + data = utils.get_middle_bi_values() + return Response(data, status=HTTPStatus.OK) + + + +@api_view(['POST']) +def upload_post_and_pvz(request): + warnings.filterwarnings('ignore') + file_rivals = request.FILES['file_post_and_pvz'].file + file_bytes = file_rivals.read() + excel_base64 = base64.b64encode(file_bytes).decode() + obj = models.TempFiles.objects.create(data=excel_base64) + load_post_and_pvz.delay(obj.id) + messages.success(request, 'Загрузка ПВЗ и Постаматов началась. Отслеживайте выполнение в Статусе фоновых задач') + return redirect('/admin') @api_view(['POST']) -def upload_rivals(request): +def upload_other_objects(request): warnings.filterwarnings('ignore') - file_rivals = request.FILES['file_rivals'] - utils.load_rivals(file_rivals) - return JsonResponse({'message': 'OK'}) + file = request.FILES['file_other_objects'] + file_bytes = file.read() + excel_base64 = base64.b64encode(file_bytes).decode() + obj = models.TempFiles.objects.create(data=excel_base64) + load_other_objects.delay(obj.id) + messages.success(request, 'Загрузка Прочих объектов началась. Отслеживайте выполнение в Статусе фоновых задач') + return redirect('/admin') @api_view(['POST']) @@ -253,7 +488,17 @@ def upload_dist(request): warnings.filterwarnings('ignore') file_dist = request.FILES['file_dist'] utils.load_dist(file_dist) - return JsonResponse({'message': 'OK'}) + messages.success(request, 'Файл расстояний успешно загружен') + return redirect('/admin') + + +@api_view(['POST']) +def upload_houses(request): + warnings.filterwarnings('ignore') + file_dist = request.FILES['file_houses'] + utils.load_houses(file_dist) + messages.success(request, 'Файл с домами успешно загружен') + return redirect('/admin') @api_view(['GET']) @@ -262,3 +507,17 @@ def get_current_user(request): return JsonResponse( {'groups': [gr.name for gr in request.user.groups.all()]}, ) + + +def download_pvz_template(self): + image_buffer = open('Постоматы и ПВЗ.xlsx', "rb").read() + response = HttpResponse(image_buffer, content_type='xlsx') + response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('pvz_and_postomats.xlsx') + return response + + +def download_other_template(self): + image_buffer = open('Другие объекты.xlsx', "rb").read() + response = HttpResponse(image_buffer, content_type='xlsx') + response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('other.xlsx') + return response diff --git a/templates/admin/delete_selected_confirmation.html b/templates/admin/delete_selected_confirmation.html new file mode 100644 index 0000000..cb162f3 --- /dev/null +++ b/templates/admin/delete_selected_confirmation.html @@ -0,0 +1,51 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls static %} + +{% block extrahead %} + {{ block.super }} + {{ media }} + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% translate 'Delete multiple objects' %} +
+{% endblock %} + +{% block content %} +{% if perms_lacking %} +

{% blocktranslate %}Deleting the selected {{ objects_name }} would result in deleting related objects, but your account doesn't have permission to delete the following types of objects:{% endblocktranslate %}

+ +{% elif protected %} +

{% blocktranslate %}Deleting the selected {{ objects_name }} would require deleting the following protected related objects:{% endblocktranslate %}

+ +{% else %} +

{% blocktranslate %}Are you sure you want to delete the selected {{ objects_name }}? All of the following objects and their related items will be deleted:{% endblocktranslate %}

+ {% include "admin/includes/object_delete_summary.html" %} +

{% translate "Objects" %}

+ {% for deletable_object in deletable_objects %} + {% if 100 < deletable_object|length %} +

{{ deletable_object|length }} objects

+ {% else %} + + {% endif %} + {% endfor %} +
{% csrf_token %} +
+ {% for obj in queryset %} + + {% endfor %} + + + + {% translate "No, take me back" %} +
+
+{% endif %} +{% endblock %} diff --git a/templates/admin/index.html b/templates/admin/index.html index 49190a9..93ebb19 100644 --- a/templates/admin/index.html +++ b/templates/admin/index.html @@ -50,39 +50,26 @@
{% if is_superadmin %} -

Обновить файл точек

-
+

Загрузить файл ПВЗ и Постаматов

+ Скачать шаблон + {% csrf_token %} - +
-

Загрузить районы и АО

-
+

Загрузить файл прочих объектов

+ Скачать шаблон + {% csrf_token %} -

Файл с АО

- -

Файл с Районами

- +
-

Обновить файл полигонов

-
+

Загрузить файл с домами

+ {% csrf_token %} - +
-

Загрузить файл конкурентов

-
- {% csrf_token %} - - -
-

Загрузить файл расстояний

-
- {% csrf_token %} - - -
{% endif %}
diff --git a/Другие объекты.xlsx b/Другие объекты.xlsx new file mode 100644 index 0000000..0752d9a Binary files /dev/null and b/Другие объекты.xlsx differ diff --git a/Постоматы и ПВЗ.xlsx b/Постоматы и ПВЗ.xlsx new file mode 100644 index 0000000..ca001a9 Binary files /dev/null and b/Постоматы и ПВЗ.xlsx differ