Timofey Malinin 3 years ago
parent f8e7e9b833
commit 3058d45789

@ -19,3 +19,4 @@ MARTIN_PORT=3000
# Host domain name (maps in docker-compose to REACT_APP_DOMAIN_URL) # Host domain name (maps in docker-compose to REACT_APP_DOMAIN_URL)
DOMAIN=postnet-dev.selftech.ru DOMAIN=postnet-dev.selftech.ru
GEOCODER_API_KEY = api_key_here

1
.gitignore vendored

@ -134,5 +134,4 @@ nginx/nginx.conf.prod
docker-compose.dev.yml docker-compose.dev.yml
pg_dumps/ pg_dumps/
django_static/ django_static/
django_media/
dit_frontend/ dit_frontend/

@ -35,14 +35,29 @@ build-static-image:
-t ${DOCKER_IMAGE_TAG}-static . -t ${DOCKER_IMAGE_TAG}-static .
- docker push ${DOCKER_IMAGE_TAG}-static - docker push ${DOCKER_IMAGE_TAG}-static
auto-deploy-dev-kuber:
extends: .deploy_base_kuber
variables:
INGRESS_HOST: "postnet.dev.selftech.ru"
tags:
- docker
only:
refs:
- dev
environment:
name: dev
deploy-dev-kuber: deploy-dev-kuber:
extends: .deploy_base_kuber extends: .deploy_base_kuber
variables: variables:
INGRESS_HOST: "postnet.dev.selftech.ru" INGRESS_HOST: "postnet.dev.selftech.ru"
tags: tags:
- docker - docker
except:
- dev
environment: environment:
name: dev name: dev
when: manual
deploy-prod-kuber: deploy-prod-kuber:
extends: .deploy_base_kuber extends: .deploy_base_kuber
@ -52,6 +67,21 @@ deploy-prod-kuber:
- docker-prod - docker-prod
environment: environment:
name: prod name: prod
when: manual
dev_restart_martin:
extends: .restart_martin_base
tags:
- docker
environment:
name: dev
prod_restart_martin:
extends: .restart_martin_base
tags:
- docker-prod
environment:
name: prod
.deploy_base_kuber: .deploy_base_kuber:
image: ${YC_CONTAINER_REGISTRY}/public/helm-kubectl-git:1.0.0 image: ${YC_CONTAINER_REGISTRY}/public/helm-kubectl-git:1.0.0
@ -80,6 +110,17 @@ deploy-prod-kuber:
- ./deploy/beat.yml - ./deploy/beat.yml
- ./deploy/django-static.yml - ./deploy/django-static.yml
expire_in: 1 week expire_in: 1 week
.restart_martin_base:
image: ${YC_CONTAINER_REGISTRY}/public/helm-kubectl-git:1.0.0
stage: .pre
before_script:
- KUBE_CONFIG=`echo ${CI_ENVIRONMENT_NAME}_kubeconfig`
- mkdir -p ${HOME}/.kube
- 'cat ${!KUBE_CONFIG} > ${HOME}/.kube/config'
- chmod -R 700 ${HOME}/.kube
script:
- kubectl rollout restart deployment martin -n ${DEPLOY_KUBER_NAMESPACE}
when: manual when: manual
# variables: # variables:

Binary file not shown.

@ -104,4 +104,7 @@ server {
listen [::]:80 ; listen [::]:80 ;
return 404; return 404;
} }
``` ```

@ -40,6 +40,13 @@ spec:
port: 8888 port: 8888
initialDelaySeconds: 3 initialDelaySeconds: 3
periodSeconds: 3 periodSeconds: 3
volumeMounts:
- mountPath: "/usr/share/nginx/html/django_media"
name: django-nginx-data
volumes:
- name: django-nginx-data
persistentVolumeClaim:
claimName: django-nginx-pvc
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service
@ -71,3 +78,10 @@ spec:
number: 80 number: 80
path: /django_static/ path: /django_static/
pathType: ImplementationSpecific pathType: ImplementationSpecific
- backend:
service:
name: django-static
port:
number: 80
path: /django_media/
pathType: ImplementationSpecific

@ -19,7 +19,7 @@ spec:
containers: containers:
- name: django - name: django
image: DEPLOY_IMAGE_TAG image: DEPLOY_IMAGE_TAG
command: ["sh", "-c", "python manage.py migrate && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"] command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"]
ports: ports:
- containerPort: 8000 - containerPort: 8000
name: django-port name: django-port
@ -28,23 +28,30 @@ spec:
name: postamates-configmap name: postamates-configmap
resources: resources:
requests: requests:
memory: "1500Mi" memory: "3000Mi"
cpu: "300m" cpu: "500m"
limits: limits:
memory: "1500Mi" memory: "3000Mi"
cpu: "300m" cpu: "500m"
readinessProbe: # readinessProbe:
httpGet: # httpGet:
path: /api/ao_rayons # path: /api/ao_rayons
port: django-port # port: django-port
initialDelaySeconds: 3 # initialDelaySeconds: 3
periodSeconds: 3 # periodSeconds: 3
livenessProbe: # livenessProbe:
httpGet: # httpGet:
path: /api/ao_rayons # path: /api/ao_rayons
port: django-port # port: django-port
initialDelaySeconds: 3 # initialDelaySeconds: 3
periodSeconds: 3 # periodSeconds: 3
volumeMounts:
- mountPath: "/code/django_media"
name: django-nginx-data
volumes:
- name: django-nginx-data
persistentVolumeClaim:
claimName: django-nginx-pvc
--- ---
apiVersion: v1 apiVersion: v1
kind: Service kind: Service

@ -2,7 +2,8 @@ ARG YC_CONTAINER_REGISTRY
FROM ${YC_CONTAINER_REGISTRY}/public/python:3.8 FROM ${YC_CONTAINER_REGISTRY}/public/python:3.8
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y binutils libproj-dev gdal-bin apt-get install -y binutils libproj-dev gdal-bin && \
apt-get install -y postgresql-client
ENV PYTHONUNBUFFERED 1 ENV PYTHONUNBUFFERED 1

@ -19,14 +19,14 @@ spec:
containers: containers:
- name: worker - name: worker
image: DEPLOY_IMAGE_TAG image: DEPLOY_IMAGE_TAG
command: ["sh", "-c", "celery -A postamates.celery:app worker"] command: ["sh", "-c", "celery -A postamates.celery:app worker", "-l", "info", "--concurrency=1"]
envFrom: envFrom:
- configMapRef: - configMapRef:
name: postamates-configmap name: postamates-configmap
resources: resources:
requests: requests:
memory: "1500Mi" memory: "3000Mi"
cpu: "300m" cpu: "500m"
limits: limits:
memory: "1500Mi" memory: "3000Mi"
cpu: "300m" cpu: "500m"

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.5 KiB

@ -30,6 +30,10 @@ services:
sh -c "python manage.py migrate && sh -c "python manage.py migrate &&
python manage.py collectstatic --noinput && python manage.py collectstatic --noinput &&
python manage.py loaddata fixtures/groups.json && python manage.py loaddata fixtures/groups.json &&
python manage.py loaddata fixtures/post_and_pvz.json &&
python manage.py loaddata fixtures/post_and_pvz_groups.json &&
python manage.py loaddata fixtures/otherobjectscategorys.json &&
python manage.py loaddata fixtures/otherobjectsgroups.json &&
python manage.py runserver 0.0.0.0:${DJANGO_PORT}" python manage.py runserver 0.0.0.0:${DJANGO_PORT}"
environment: environment:
<<: *postgres-variables <<: *postgres-variables

@ -1 +1 @@
[{"model": "auth.group", "pk": 1, "fields": {"name": "Администратор пользователей", "permissions": [13, 14, 15, 16]}}, {"model": "auth.group", "pk": 2, "fields": {"name": "Зритель", "permissions": [28]}}, {"model": "auth.group", "pk": 3, "fields": {"name": "Редактор", "permissions": [25, 26, 27, 28]}}] [{"model": "auth.group", "pk": 1, "fields": {"name": "Зритель", "permissions": [["view_placementpoint", "service", "placementpoint"]]}}, {"model": "auth.group", "pk": 2, "fields": {"name": "Редактор", "permissions": [["add_placementpoint", "service", "placementpoint"], ["change_placementpoint", "service", "placementpoint"], ["delete_placementpoint", "service", "placementpoint"], ["view_placementpoint", "service", "placementpoint"]]}}, {"model": "auth.group", "pk": 3, "fields": {"name": "Администратор пользователей", "permissions": [["add_user", "auth", "user"], ["change_user", "auth", "user"], ["delete_user", "auth", "user"], ["view_user", "auth", "user"]]}}]

@ -0,0 +1,186 @@
[
{
"model": "service.otherobjectscategory",
"pk": 4,
"fields": {
"name": "business_activity",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 5,
"fields": {
"name": "metro_stations",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 22,
"fields": {
"name": "bargains",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 23,
"fields": {
"name": "BC",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 24,
"fields": {
"name": "flats_cnt",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 25,
"fields": {
"name": "offers_estate",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 26,
"fields": {
"name": "schools",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 27,
"fields": {
"name": "kindergar",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 28,
"fields": {
"name": "stops",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 29,
"fields": {
"name": "pharmacies",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 30,
"fields": {
"name": "sport_centers",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 31,
"fields": {
"name": "supermarkets",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 32,
"fields": {
"name": "supermarkets_premium",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 33,
"fields": {
"name": "banks",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 34,
"fields": {
"name": "recas",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 35,
"fields": {
"name": "labs",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 36,
"fields": {
"name": "clinics",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 37,
"fields": {
"name": "attractions",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 38,
"fields": {
"name": "cultures",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 39,
"fields": {
"name": "public_services",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 40,
"fields": {
"name": "popul_home_job",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 41,
"fields": {
"name": "TC",
"visible": false
}
},
{
"model": "service.otherobjectscategory",
"pk": 42,
"fields": {
"name": "yndx_food_cnt_amt",
"visible": false
}
}
]

@ -0,0 +1,23 @@
[{"model": "service.otherobjectsgroup", "pk": 4, "fields": {"name": "business_activity", "category": 4, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 5, "fields": {"name": "metro_stations", "category": 5, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 23, "fields": {"name": "bargains", "category": 22, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 24, "fields": {"name": "BC", "category": 23, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 25, "fields": {"name": "flats_cnt", "category": 24, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 26, "fields": {"name": "offers_estate", "category": 25, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 27, "fields": {"name": "schools", "category": 26, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 28, "fields": {"name": "kindergar", "category": 27, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 29, "fields": {"name": "stops", "category": 28, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 30, "fields": {"name": "pharmacies", "category": 29, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 31, "fields": {"name": "sport_centers", "category": 30, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 32, "fields": {"name": "supermarkets", "category": 31, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 33, "fields": {"name": "supermarkets_premium", "category": 32, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 34, "fields": {"name": "banks", "category": 33, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 35, "fields": {"name": "recas", "category": 34, "image": "", "visible": false}}, {
"model": "service.otherobjectsgroup", "pk": 36, "fields": {"name": "labs", "category": 35, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 37, "fields": {"name": "clinics", "category": 36, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 38, "fields": {"name": "attractions", "category": 37, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 39, "fields": {"name": "cultures", "category": 38, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 40, "fields": {"name": "public_services", "category": 39, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 41, "fields": {"name": "popul_home_job", "category": 40, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 42, "fields": {"name": "TC", "category": 41, "image": "", "visible": false}},
{"model": "service.otherobjectsgroup", "pk": 43, "fields": {"name": "yndx_food_cnt_amt", "category": 42, "image": "", "visible": false}}]

@ -0,0 +1,20 @@
[
{
"model": "service.post_and_pvzcategory",
"pk": 1,
"fields": {
"name": "ПВЗ",
"visible": true,
"include_in_ml": true
}
},
{
"model": "service.post_and_pvzcategory",
"pk": 2,
"fields": {
"name": "Постаматы прочих сетей",
"visible": true,
"include_in_ml": true
}
}
]

@ -0,0 +1,112 @@
[
{
"model":"service.post_and_pvzgroup",
"pk":7,
"fields":{
"name":"Ozon",
"category":1,
"image":"post_and_pvz_group_images/nameozon_typePVZ_SizeM.png",
"visible":true,
"include_in_ml":true
}
},
{
"model":"service.post_and_pvzgroup",
"pk":8,
"fields":{
"name":"СДЭК",
"category":1,
"image":"post_and_pvz_group_images/namesdek_typePVZ_SizeM.png",
"visible":true,
"include_in_ml":true
}
},
{
"model":"service.post_and_pvzgroup",
"pk":9,
"fields":{
"name":"Халва",
"category":2,
"image":"post_and_pvz_group_images/namekhalva_typepostamat_SizeM.png",
"visible":true,
"include_in_ml":false
}
},
{
"model":"service.post_and_pvzgroup",
"pk":10,
"fields":{
"name":"СДЭК",
"category":2,
"image":"post_and_pvz_group_images/namesdek_typepostamat_SizeM.png",
"visible":true,
"include_in_ml":false
}
},
{
"model":"service.post_and_pvzgroup",
"pk":12,
"fields":{
"name":"Ozon",
"category":2,
"image":"post_and_pvz_group_images/nameozon_typepostamat_SizeM.png",
"visible":true,
"include_in_ml":false
}
},
{
"model":"service.post_and_pvzgroup",
"pk":17,
"fields":{
"name":"WildBerries",
"category":1,
"image":"post_and_pvz_group_images/namewb_typePVZ_SizeM.png",
"visible":true,
"include_in_ml":false
}
},
{
"model":"service.post_and_pvzgroup",
"pk":19,
"fields":{
"name":"Яндекс.Маркет",
"category":1,
"image":"post_and_pvz_group_images/ya.market-PVZ-M_KWksDvo.png",
"visible":true,
"include_in_ml":true
}
},
{
"model":"service.post_and_pvzgroup",
"pk":20,
"fields":{
"name":"Яндекс.Маркет",
"category":2,
"image":"post_and_pvz_group_images/ya.market-postamat-M.png",
"visible":true,
"include_in_ml":true
}
},
{
"model":"service.post_and_pvzgroup",
"pk":24,
"fields":{
"name":"Почта России",
"category":1,
"image":"post_and_pvz_group_images/pochta-PVZ-M.png",
"visible":true,
"include_in_ml":true
}
},
{
"model":"service.post_and_pvzgroup",
"pk":25,
"fields":{
"name":"Почта России",
"category":2,
"image":"post_and_pvz_group_images/pochta-postamat-M_gBJlHTO.png",
"visible":true,
"include_in_ml":true
}
}
]

@ -22,7 +22,9 @@ BASE_DIR = Path(__file__).resolve().parent.parent
SECRET_KEY = 'django-insecure-5czma@e7b(e4v+c*@bkknj(*em%@x52jizednhy6lye)_@ox4@' SECRET_KEY = 'django-insecure-5czma@e7b(e4v+c*@bkknj(*em%@x52jizednhy6lye)_@ox4@'
# SECURITY WARNING: don't run with debug turned on in production! # SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.getenv('DEBUG', False) == 'True' # TODO: change DEBUG to
# DEBUG = os.getenv('DEBUG', False) == 'True'
DEBUG = True
ALLOWED_HOSTS = ['*'] ALLOWED_HOSTS = ['*']
@ -133,7 +135,7 @@ MEDIA_ROOT = os.path.join(BASE_DIR, 'django_media')
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
CORS_ORIGIN_ALLOW_ALL = True # If this is used then `CORS_ORIGIN_WHITELIST` will not have any effect CORS_ALLOW_ALL_ORIGINS = True # If this is used then `CORS_ORIGIN_WHITELIST` will not have any effect
CORS_ALLOW_CREDENTIALS = True CORS_ALLOW_CREDENTIALS = True
CORS_ORIGIN_ALLOW = True CORS_ORIGIN_ALLOW = True
@ -174,7 +176,6 @@ SWAGGER_SETTINGS = {
'SWAGGER_PATH': 'django_static/swagger/swagger.yaml', 'SWAGGER_PATH': 'django_static/swagger/swagger.yaml',
} }
SRID = 4326 SRID = 4326
# celery config # celery config
@ -182,9 +183,17 @@ SRID = 4326
CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL') CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL')
CELERY_NAMESPACE = 'CELERY' CELERY_NAMESPACE = 'CELERY'
PROJECT_NAME = 'postamates' PROJECT_NAME = 'postamates'
CACHE_TIMEOUT = 0 DOMAIN = os.getenv('DOMAIN', 'localhost')
if DEBUG:
CACHE_TIMEOUT = 60
else:
CACHE_TIMEOUT = 60 * 60 * 24 * 7
DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS = 500 DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS = 500
AGE_DAY_LIMIT = 270 AGE_DAY_LIMIT = 270
AGE_DAY_BORDER = 30 AGE_DAY_BORDER = 30
EXCEL_EXPORT_FILENAME = 'placement_points.xlsx' EXCEL_EXPORT_FILENAME = 'placement_points.xlsx'
JSON_EXPORT_FILENAME = 'placement_points.json' JSON_EXPORT_FILENAME = 'placement_points.json'
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M')
STATUS_TASK_NAME='status_task'

@ -90,3 +90,5 @@ virtualenv==20.20.0
wcwidth==0.2.6 wcwidth==0.2.6
xlrd==1.2.0 xlrd==1.2.0
XlsxWriter==3.0.8 XlsxWriter==3.0.8
django-filter==23.2
shap==0.41.0

@ -1,14 +1,22 @@
import os
from django.contrib import admin from django.contrib import admin
from django.contrib.admin import AdminSite from django.contrib.admin import AdminSite
from django.contrib.admin.sites import NotRegistered from django.contrib.admin.sites import NotRegistered
from django.contrib.auth.admin import UserAdmin from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User from django.contrib.auth.models import User
from service.layer_service import LayerService
from service.models import AO from service.models import AO
from service.models import PlacementPoint from service.models import PlacementPoint
from service.models import PointDist
from service.models import Rayon from service.models import Rayon
from service.models import Rivals from service.models import PrePlacementPoint, Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, \
OtherObjectsGroup, \
OtherObjectsCategory, PrePlacementPointPVZDistance, TempFiles, RaschetGroups, House, RaschetObjects
from service.models import PlacementPointPVZDistance, TaskStatus
from postamates.settings import DEBUG
from django.core.cache import cache
from service.utils import run_psql_command
class MyAdminSite(AdminSite): class MyAdminSite(AdminSite):
@ -19,18 +27,140 @@ class MyAdminSite(AdminSite):
return super(MyAdminSite, self).index(request, extra_context) return super(MyAdminSite, self).index(request, extra_context)
my_admin_site = MyAdminSite(name='myadmin') my_admin_site = MyAdminSite(name='POSTNET')
if DEBUG:
my_admin_site.register(AO)
my_admin_site.register(Rayon)
my_admin_site.register(TempFiles)
my_admin_site.register(PlacementPointPVZDistance)
class Post_and_PVZAdmin(admin.ModelAdmin):
def save_model(self, request, obj, form, change):
obj.save()
LayerService().count_post_pvz_for_placementpoint(obj)
RaschetGroups.objects.create(obj_id=obj.group.id)
def delete_model(self, request, obj):
RaschetObjects.objects.filter(obj_id=obj.id).delete()
super().delete_model(request, obj)
my_admin_site.register(Post_and_pvz, Post_and_PVZAdmin)
my_admin_site.register(OtherObjects)
my_admin_site.register(House)
my_admin_site.register(PrePlacementPoint)
my_admin_site.register(PrePlacementPointPVZDistance)
class TaskStatusAdmin(admin.ModelAdmin):
list_display = ('task_name', 'status')
class CategoryAdmin(admin.ModelAdmin):
list_display = ('name', 'visible')
def save_model(self, request, obj, form, change):
obj.save()
cache.clear()
def delete_model(self, request, obj):
obj.delete()
cache.clear()
class PostPvzCategoryAdmin(CategoryAdmin):
def get_readonly_fields(self, request, obj=None):
if obj.id in (1, 2):
return ['id', 'name']
else:
return super().get_readonly_fields(request, obj)
def save_model(self, request, obj, form, change):
obj.save()
run_psql_command()
if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
LayerService.update_categories(obj)
cache.clear()
def delete_model(self, request, obj):
if obj.id in (1, 2):
pass
else:
super().delete_model(request, obj)
class GroupAdmin(admin.ModelAdmin):
list_display = ('name', 'category', 'visible')
def save_model(self, request, obj, form, change):
obj.save()
cache.clear()
def delete_model(self, request, obj):
obj.delete()
cache.clear()
class PostPvzGroupAdmin(GroupAdmin):
def get_readonly_fields(self, request, obj=None):
if obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25):
return ['id', 'name']
else:
return super().get_readonly_fields(request, obj)
def save_model(self, request, obj, form, change):
obj.save()
run_psql_command()
if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
LayerService.update_groups(obj)
cache.clear()
def delete_model(self, request, obj):
if obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25):
pass
else:
RaschetGroups.objects.filter(obj_id=obj.id).delete()
super().delete_model(request, obj)
class OtherObjectsGroupAdmin(GroupAdmin):
def get_readonly_fields(self, request, obj=None):
if obj.id in (4, 5) or obj.id in list(range(22, 43)):
return ['id', 'name']
else:
return super().get_readonly_fields(request, obj)
def delete_model(self, request, obj):
if obj.id in (4, 5) or obj.id in list(range(22, 43)):
pass
else:
super().delete_model(request, obj)
class OtherObjectsCategoryAdmin(CategoryAdmin):
def get_readonly_fields(self, request, obj=None):
if obj.id in (4, 5) or obj.id in list(range(22, 43)):
return ['id', 'name']
else:
return super().get_readonly_fields(request, obj)
my_admin_site.register(AO) def delete_model(self, request, obj):
my_admin_site.register(Rayon) if obj.id in (4, 5) or obj.id in list(range(22, 43)):
my_admin_site.register(Rivals) pass
my_admin_site.register(PointDist) else:
super().delete_model(request, obj)
class PlacementPointAdmin(admin.ModelAdmin): class PlacementPointAdmin(admin.ModelAdmin):
pass pass
my_admin_site.register(TaskStatus, TaskStatusAdmin)
my_admin_site.register(Post_and_pvzGroup, PostPvzGroupAdmin)
my_admin_site.register(OtherObjectsGroup, OtherObjectsGroupAdmin)
my_admin_site.register(Post_and_pvzCategory, PostPvzCategoryAdmin)
my_admin_site.register(OtherObjectsCategory, OtherObjectsCategoryAdmin)
my_admin_site.register(PlacementPoint, PlacementPointAdmin) my_admin_site.register(PlacementPoint, PlacementPointAdmin)

@ -4,3 +4,6 @@ from django.apps import AppConfig
class ServiceConfig(AppConfig): class ServiceConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField' default_auto_field = 'django.db.models.BigAutoField'
name = 'service' name = 'service'
def ready(self):
from service import signals

@ -6,3 +6,8 @@ class PointStatus(Enum):
Installation = 'Согласование-Установка' Installation = 'Согласование-Установка'
Working = 'Работает' Working = 'Работает'
Cancelled = 'Отменено' Cancelled = 'Отменено'
class MatchingStatus(Enum):
Error = 'Ошибка'
New = 'Новая'
Matched = 'Совпадение'

@ -0,0 +1,44 @@
from django.contrib.gis.measure import Distance
from service import models
from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS
class LayerService:
def count_post_pvz_for_placementpoint(self, obj):
points = models.PlacementPoint.objects.filter(geometry__distance_lt=(obj.wkt, Distance(
m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).all()
for point in points:
LayerService.count_post_pvz(point)
@staticmethod
def update_categories(instance):
groups = models.Post_and_pvzGroup.objects.filter(category=instance)
groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
for gr in groups:
models.RaschetGroups.objects.get_or_create(obj_id=gr.id)
objects = models.Post_and_pvz.objects.filter(group=gr)
objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
for obj in objects.all():
models.RaschetObjects.objects.get_or_create(obj_id=obj.id)
@staticmethod
def update_groups(instance):
models.RaschetGroups.objects.get_or_create(obj_id=instance.id)
objects = models.Post_and_pvz.objects.filter(group=instance)
objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
for obj in objects.all():
models.RaschetObjects.objects.get_or_create(obj_id=obj.id)
@staticmethod
def get_post_and_pvz_categroies():
return models.Post_and_pvzCategory.objects.all(), models.Post_and_pvzGroup.objects.all()
@staticmethod
def count_post_pvz(point):
point.rival_post_cnt = models.Post_and_pvz.objects.filter(
category__name="Постамат", include_in_ml=True,
wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.rival_pvz_cnt = models.Post_and_pvz.objects.filter(
category__name="ПВЗ", include_in_ml=True,
wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.save()

@ -0,0 +1,63 @@
from django.core.management.base import BaseCommand
from service.utils import run_sql_command, log_to_telegram
CMD_PIVOT_DIST = """CREATE OR REPLACE VIEW compact_placementpoint AS
SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpoint;
CREATE OR REPLACE procedure pivot_dist()
--RETURNS SET OF record
AS $BODY$
DECLARE columnNames TEXT;
BEGIN
DROP MATERIALIZED VIEW IF EXISTS points_with_dist;
SELECT 'placement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_placementpointpvzdistance order by 1) as asd
INTO columnNames;
EXECUTE format('CREATE MATERIALIZED VIEW points_with_dist AS SELECT *
FROM CROSSTAB(
$$
SELECT placement_point_id, pvz_postamates_group_id, dist
FROM service_placementpointpvzdistance
ORDER BY 1, 2
$$
) AS ct(%s)
LEFT JOIN compact_placementpoint ON placement_point_id=id'
,columnNames);
END;
$BODY$
LANGUAGE plpgsql;
"""
CMD_PIVOT_DIST_PRE = """CREATE OR REPLACE VIEW compact_preplacementpoint AS
SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpoint;
CREATE OR REPLACE procedure prepivot_dist()
AS $BODY$
DECLARE columnNames TEXT;
BEGIN
DROP MATERIALIZED VIEW IF EXISTS prepoints_with_dist;
SELECT 'preplacement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_preplacementpointpvzdistance order by 1) as asd
INTO columnNames;
EXECUTE format('CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT *
FROM CROSSTAB(
$$
SELECT placement_point_id, pvz_postamates_group_id, dist
FROM service_preplacementpointpvzdistance
ORDER BY 1, 2
$$
) AS ct(%s)
LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id'
,columnNames);
END;
$BODY$
LANGUAGE plpgsql;
"""
class Command(BaseCommand):
help = 'Create procedures'
def handle(self, *args, **kwargs):
try:
log_to_telegram('Creating procedures')
run_sql_command(CMD_PIVOT_DIST)
log_to_telegram('pivot_dist created')
run_sql_command(CMD_PIVOT_DIST_PRE)
log_to_telegram('prepivot_dist created')
except Exception as e:
log_to_telegram('Error creating views: ' + str(e))

@ -0,0 +1,18 @@
from django.core.management.base import BaseCommand
from service.utils import run_sql_command, log_to_telegram
CMD_PIVOT_DIST = """CALL public.pivot_dist();"""
CMD_PIVOT_DIST_PRE = """CALL public.prepivot_dist();"""
class Command(BaseCommand):
help = 'Create views'
def handle(self, *args, **kwargs):
try:
log_to_telegram('Creating views')
run_sql_command(CMD_PIVOT_DIST)
log_to_telegram('pivot_dist created')
run_sql_command(CMD_PIVOT_DIST_PRE)
log_to_telegram('prepivot_dist created')
except Exception as e:
log_to_telegram('Error creating views: ' + str(e))

@ -0,0 +1,18 @@
from django.core.management.base import BaseCommand
from service.utils import run_sql_command, log_to_telegram
CMD_PIVOT_DIST = """DROP MATERIALIZED VIEW IF EXISTS public.points_with_dist;"""
CMD_PIVOT_DIST_PRE = """DROP MATERIALIZED VIEW IF EXISTS public.prepoints_with_dist;"""
class Command(BaseCommand):
help = 'delete views'
def handle(self, *args, **kwargs):
try:
log_to_telegram('Deleting views')
run_sql_command(CMD_PIVOT_DIST)
log_to_telegram('pivot_dist deleted')
run_sql_command(CMD_PIVOT_DIST_PRE)
log_to_telegram('prepivot_dist deleted')
except Exception as e:
log_to_telegram('Error deleting views: ' + str(e))

@ -0,0 +1,20 @@
# Generated by Django 3.2 on 2023-07-15 13:13
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0021_auto_20230402_2129'),
]
operations = [
migrations.CreateModel(
name='LastMLCall',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('dt', models.DateTimeField(auto_now_add=True)),
],
),
]

@ -0,0 +1,166 @@
# Generated by Django 3.2 on 2023-08-01 13:54
import django.contrib.gis.db.models.fields
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('service', '0022_lastmlcall'),
]
operations = [
migrations.CreateModel(
name='OtherObjects',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('wkt', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326)),
('param1', models.FloatField(blank=True, null=True)),
('param2', models.TextField(blank=True, null=True)),
('visible', models.BooleanField(default=True)),
],
options={
'verbose_name': 'Прочий объект',
'verbose_name_plural': 'Прочие объекты',
},
),
migrations.CreateModel(
name='OtherObjectsCategory',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField(verbose_name='Название слоя')),
('image', models.ImageField(blank=True, default=None, null=True, upload_to='other_objects_category_images/', verbose_name='Картинка')),
('visible', models.BooleanField(default=True)),
],
options={
'verbose_name': 'категория прочих объектов',
'verbose_name_plural': 'Категории прочих объектов',
},
),
migrations.CreateModel(
name='OtherObjectsGroup',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField(verbose_name='Название группы')),
('image', models.ImageField(blank=True, null=True, upload_to='other_objects_group_images/', verbose_name='Картинка')),
('visible', models.BooleanField(default=True)),
('category', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='otherobjectsgroup', to='service.otherobjectscategory')),
],
options={
'verbose_name': 'группа прочих объектов',
'verbose_name_plural': 'Группы прочих объектов',
},
),
migrations.CreateModel(
name='PlacementPointPVZDistance',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('dist', models.FloatField(blank=True, default=None, null=True)),
],
),
migrations.CreateModel(
name='Post_and_pvz',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('wkt', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326)),
('visible', models.BooleanField(default=True)),
('inlude_in_ml', models.BooleanField(default=True)),
],
options={
'verbose_name': 'Постамат или ПВЗ',
'verbose_name_plural': 'Постаматы и ПВЗ',
},
),
migrations.CreateModel(
name='Post_and_pvzCategory',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField(verbose_name='Название слоя')),
('image', models.ImageField(blank=True, default=None, null=True, upload_to='post_and_pvz_category_images/', verbose_name='Картинка')),
('visible', models.BooleanField(default=True)),
('inlude_in_ml', models.BooleanField(default=True)),
],
options={
'verbose_name': 'категория постаматов и ПВЗ',
'verbose_name_plural': 'Категории постаматов и ПВЗ',
},
),
migrations.CreateModel(
name='Post_and_pvzGroup',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField(verbose_name='Название группы')),
('image', models.ImageField(blank=True, null=True, upload_to='post_and_pvz_group_images/', verbose_name='Картинка')),
('visible', models.BooleanField(default=True)),
('inlude_in_ml', models.BooleanField(default=True)),
('category', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='post_and_pvz_group', to='service.post_and_pvzcategory')),
],
options={
'verbose_name': 'группа постаматов и ПВЗ',
'verbose_name_plural': 'Группы постаматов и ПВЗ',
},
),
migrations.CreateModel(
name='TaskStatus',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('task_name', models.TextField(unique=True, verbose_name='Название задачи')),
('status', models.TextField(blank=True, null=True, verbose_name='Статус выполнения')),
],
options={
'verbose_name': 'Статус фоновых задач',
'verbose_name_plural': 'Статус фоновых задач',
},
),
migrations.DeleteModel(
name='Rivals',
),
migrations.AlterModelOptions(
name='ao',
options={'verbose_name': 'АО', 'verbose_name_plural': 'АО'},
),
migrations.AlterModelOptions(
name='placementpoint',
options={'verbose_name': 'Точка', 'verbose_name_plural': 'Точки'},
),
migrations.AlterModelOptions(
name='pointdist',
options={'verbose_name_plural': 'Расстояния между точками'},
),
migrations.AlterModelOptions(
name='rayon',
options={'verbose_name': 'Район', 'verbose_name_plural': 'Районы'},
),
migrations.AddField(
model_name='post_and_pvz',
name='category',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='post_and_pvz', to='service.post_and_pvzcategory'),
),
migrations.AddField(
model_name='post_and_pvz',
name='group',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='post_and_pvz', to='service.post_and_pvzgroup'),
),
migrations.AddField(
model_name='placementpointpvzdistance',
name='placement_point',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.placementpoint'),
),
migrations.AddField(
model_name='placementpointpvzdistance',
name='pvz_postamates_group',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup'),
),
migrations.AddField(
model_name='otherobjects',
name='category',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='other_objects', to='service.otherobjectscategory'),
),
migrations.AddField(
model_name='otherobjects',
name='group',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='other_objects', to='service.otherobjectsgroup'),
),
]

@ -0,0 +1,20 @@
# Generated by Django 3.2 on 2023-08-03 15:26
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0023_auto_20230801_1654'),
]
operations = [
migrations.CreateModel(
name='TempFiles',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('data', models.TextField()),
],
),
]

@ -0,0 +1,24 @@
# Generated by Django 3.2 on 2023-08-04 10:49
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('service', '0024_tempfiles'),
]
operations = [
migrations.AlterField(
model_name='otherobjectsgroup',
name='category',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.otherobjectscategory'),
),
migrations.AlterField(
model_name='post_and_pvzgroup',
name='category',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.post_and_pvzcategory'),
),
]

@ -0,0 +1,31 @@
# Generated by Django 3.2 on 2023-08-23 17:16
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('service', '0025_auto_20230804_1349'),
]
operations = [
migrations.RenameField(
model_name='post_and_pvz',
old_name='inlude_in_ml',
new_name='include_in_ml',
),
migrations.RenameField(
model_name='post_and_pvzcategory',
old_name='inlude_in_ml',
new_name='include_in_ml',
),
migrations.RenameField(
model_name='post_and_pvzgroup',
old_name='inlude_in_ml',
new_name='include_in_ml',
),
migrations.DeleteModel(
name='PointDist',
),
]

@ -0,0 +1,21 @@
# Generated by Django 3.2 on 2023-08-24 17:29
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0026_auto_20230823_2016'),
]
operations = [
migrations.CreateModel(
name='PostMLChecker',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('current', models.IntegerField(default=0)),
('target', models.IntegerField()),
],
),
]

@ -0,0 +1,16 @@
# Generated by Django 3.2 on 2023-08-31 12:07
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('service', '0027_postmlchecker'),
]
operations = [
migrations.DeleteModel(
name='PostMLChecker',
),
]

@ -0,0 +1,18 @@
# Generated by Django 3.2 on 2023-08-31 18:12
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0028_delete_postmlchecker'),
]
operations = [
migrations.AlterField(
model_name='placementpoint',
name='postamat_id',
field=models.IntegerField(blank=True, null=True, unique=True, verbose_name='ID постамата'),
),
]

@ -0,0 +1,35 @@
# Generated by Django 3.2 on 2023-09-03 17:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0029_alter_placementpoint_postamat_id'),
]
operations = [
migrations.AlterModelOptions(
name='otherobjects',
options={'ordering': ('id',), 'verbose_name': 'Прочий объект', 'verbose_name_plural': 'Прочие объекты'},
),
migrations.AlterModelOptions(
name='placementpoint',
options={'ordering': ('id',), 'verbose_name': 'Точка', 'verbose_name_plural': 'Точки'},
),
migrations.AlterModelOptions(
name='post_and_pvz',
options={'ordering': ('id',), 'verbose_name': 'Постамат или ПВЗ', 'verbose_name_plural': 'Постаматы и ПВЗ'},
),
migrations.AddField(
model_name='otherobjects',
name='param3',
field=models.FloatField(blank=True, null=True),
),
migrations.AddField(
model_name='otherobjects',
name='param4',
field=models.FloatField(blank=True, null=True),
),
]

@ -0,0 +1,94 @@
# Generated by Django 3.2 on 2023-09-05 17:56
import django.contrib.gis.db.models.fields
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('service', '0030_auto_20230903_2006'),
]
operations = [
migrations.CreateModel(
name='PrePlacementPoint',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('address', models.TextField(blank=True, null=True, verbose_name='Адрес')),
('name', models.TextField(blank=True, null=True, verbose_name='Название')),
('postamat_id', models.IntegerField(blank=True, null=True, unique=True, verbose_name='ID постамата')),
('category', models.TextField(blank=True, null=True, verbose_name='Категория')),
('status', models.TextField(blank=True, choices=[('Pending', 'К рассмотрению'), ('Installation', 'Согласование-Установка'), ('Working', 'Работает'), ('Cancelled', 'Отменено')], null=True, verbose_name='Статус')),
('start_date', models.DateTimeField(blank=True, null=True)),
('age_day', models.IntegerField(blank=True, null=True, verbose_name='Возраст')),
('prediction_first', models.IntegerField(blank=True, null=True, verbose_name='Прогноз начальный')),
('prediction_current', models.IntegerField(blank=True, null=True, verbose_name='Прогноз текущий')),
('plan_first', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель начальный')),
('plan_current', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель текущий')),
('fact', models.IntegerField(blank=True, null=True, verbose_name='Фактический показатель')),
('fact_raw', models.IntegerField(blank=True, null=True)),
('delta_first', models.IntegerField(blank=True, null=True, verbose_name='Разница начальная')),
('delta_current', models.IntegerField(blank=True, null=True, verbose_name='Разница текущая')),
('sample_trn', models.BooleanField(blank=True, null=True)),
('flat_cnt', models.IntegerField(blank=True, null=True, verbose_name='Количество квартир')),
('year_bld', models.IntegerField(blank=True, null=True, verbose_name='Год постройки')),
('levels', models.IntegerField(blank=True, null=True)),
('enrg_cls', models.TextField(blank=True, null=True)),
('mat_nes', models.TextField(blank=True, null=True)),
('doors', models.IntegerField(blank=True, null=True)),
('flats_cnt', models.IntegerField(blank=True, null=True)),
('popul_home', models.IntegerField(blank=True, null=True)),
('popul_job', models.IntegerField(blank=True, null=True)),
('other_post_cnt', models.IntegerField(blank=True, null=True)),
('target_post_cnt', models.IntegerField(blank=True, null=True)),
('yndxfood_cnt', models.IntegerField(blank=True, null=True)),
('yndxfood_sum', models.IntegerField(blank=True, null=True)),
('yndxfood_cnt_cst', models.IntegerField(blank=True, null=True)),
('geometry', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326, verbose_name='Координаты')),
('is_vis', models.BooleanField(blank=True, null=True)),
('subject_rf', models.TextField(blank=True, null=True)),
('city', models.TextField(blank=True, null=True)),
('street', models.TextField(blank=True, null=True)),
('house_number', models.TextField(blank=True, null=True)),
('entrance', models.TextField(blank=True, null=True)),
('post_code', models.TextField(blank=True, null=True)),
('metro_dist', models.FloatField(blank=True, null=True)),
('target_dist', models.FloatField(blank=True, null=True)),
('property_price_bargains', models.FloatField(blank=True, null=True)),
('property_price_offers', models.FloatField(blank=True, null=True)),
('property_mean_floor', models.FloatField(blank=True, null=True)),
('property_era', models.TextField(blank=True, null=True)),
('business_activity', models.IntegerField(blank=True, null=True)),
('bc_cnt', models.IntegerField(blank=True, null=True)),
('tc_cnt', models.IntegerField(blank=True, null=True)),
('rival_pvz_cnt', models.IntegerField(blank=True, null=True)),
('rival_post_cnt', models.IntegerField(blank=True, null=True)),
('flats_cnt_2', models.IntegerField(blank=True, null=True)),
('school_cnt', models.IntegerField(blank=True, null=True)),
('kindergar_cnt', models.IntegerField(blank=True, null=True)),
('public_stop_cnt', models.IntegerField(blank=True, null=True)),
('sport_center_cnt', models.IntegerField(blank=True, null=True)),
('pharmacy_cnt', models.IntegerField(blank=True, null=True)),
('supermarket_cnt', models.IntegerField(blank=True, null=True)),
('supermarket_premium_cnt', models.IntegerField(blank=True, null=True)),
('clinic_cnt', models.IntegerField(blank=True, null=True)),
('bank_cnt', models.IntegerField(blank=True, null=True)),
('reca_cnt', models.IntegerField(blank=True, null=True)),
('lab_cnt', models.IntegerField(blank=True, null=True)),
('culture_cnt', models.IntegerField(blank=True, null=True)),
('attraction_cnt', models.IntegerField(blank=True, null=True)),
('mfc_cnt', models.IntegerField(blank=True, null=True)),
('target_cnt_ao_mean', models.FloatField(blank=True, null=True)),
('target_cnt_nearby_mean', models.FloatField(blank=True, null=True)),
('target_age_nearby_mean', models.FloatField(blank=True, null=True)),
('matching_status', models.TextField(blank=True, choices=[('Error', 'Ошибка'), ('New', 'Новая'), ('Matched', 'Совпадение')], null=True)),
('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.rayon')),
('district', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.ao')),
],
options={
'abstract': False,
},
),
]

@ -0,0 +1,27 @@
# Generated by Django 3.2 on 2023-09-12 13:16
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('service', '0031_preplacementpoint'),
]
operations = [
migrations.AlterModelOptions(
name='preplacementpoint',
options={'ordering': ('id',)},
),
migrations.CreateModel(
name='PrePlacementPointPVZDistance',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('dist', models.FloatField(blank=True, default=None, null=True)),
('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.preplacementpoint')),
('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')),
],
),
]

@ -0,0 +1,20 @@
# Generated by Django 3.2 on 2023-09-16 09:23
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0032_auto_20230912_1616'),
]
operations = [
migrations.CreateModel(
name='RaschetObjects',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('obj_id', models.IntegerField()),
],
),
]

@ -0,0 +1,20 @@
# Generated by Django 3.2 on 2023-09-16 11:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0033_raschetobjects'),
]
operations = [
migrations.CreateModel(
name='RaschetGroups',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('obj_id', models.IntegerField()),
],
),
]

@ -0,0 +1,333 @@
# Generated by Django 3.2 on 2023-09-28 12:19
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0034_raschetgroups'),
]
operations = [
migrations.AddField(
model_name='placementpoint',
name='attraction_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='bank_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='bc_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='business_activity_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='clinic_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='culture_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='flats_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='kindergar_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='lab_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='metro_dist_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='mfc_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='pharmacy_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='popul_home_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='popul_job_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='property_era_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='property_mean_floor_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='property_price_bargains_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='property_price_offers_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='public_stop_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='reca_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='rival_post_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='rival_pvz_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='school_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='sport_center_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='supermarket_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='supermarket_premium_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='target_cnt_ao_mean_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='target_dist_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='target_post_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='tc_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='yndxfood_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='placementpoint',
name='yndxfood_sum_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='attraction_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='bank_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='bc_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='business_activity_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='clinic_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='culture_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='flats_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='kindergar_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='lab_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='metro_dist_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='mfc_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='pharmacy_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='popul_home_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='popul_job_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='property_era_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='property_mean_floor_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='property_price_bargains_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='property_price_offers_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='public_stop_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='reca_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='rival_post_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='rival_pvz_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='school_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='sport_center_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='supermarket_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='supermarket_premium_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='target_cnt_ao_mean_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='target_dist_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='target_post_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='tc_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='yndxfood_cnt_shap',
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name='preplacementpoint',
name='yndxfood_sum_shap',
field=models.IntegerField(blank=True, null=True),
),
]

@ -0,0 +1,32 @@
# Generated by Django 3.2 on 2023-10-02 09:21
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0035_auto_20230928_1519'),
]
operations = [
migrations.CreateModel(
name='House',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('year_bld', models.IntegerField(blank=True, null=True)),
('mat_nes', models.TextField(blank=True, null=True)),
('flat_cnt', models.IntegerField(blank=True, null=True)),
('levels', models.TextField(blank=True, null=True)),
('doors', models.IntegerField(blank=True, null=True)),
('enrg_cls', models.TextField(blank=True, null=True)),
('street', models.TextField(blank=True, null=True)),
('house_number', models.TextField(blank=True, null=True)),
],
options={
'verbose_name': 'Дом',
'verbose_name_plural': 'Дома',
'ordering': ('id',),
},
),
]

@ -0,0 +1,23 @@
# Generated by Django 3.2 on 2023-10-02 10:49
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0036_house'),
]
operations = [
migrations.AlterField(
model_name='placementpoint',
name='levels',
field=models.TextField(blank=True, null=True),
),
migrations.AlterField(
model_name='preplacementpoint',
name='levels',
field=models.TextField(blank=True, null=True),
),
]

@ -0,0 +1,135 @@
# Generated by Django 3.2 on 2023-10-08 08:41
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0037_auto_20231002_1349'),
]
operations = [
migrations.RemoveField(
model_name='preplacementpointpvzdistance',
name='placement_point',
),
migrations.RemoveField(
model_name='preplacementpointpvzdistance',
name='pvz_postamates_group',
),
migrations.RemoveField(
model_name='otherobjectscategory',
name='image',
),
migrations.RemoveField(
model_name='post_and_pvzcategory',
name='image',
),
migrations.AddField(
model_name='placementpoint',
name='d10',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата СДЭК'),
),
migrations.AddField(
model_name='placementpoint',
name='d12',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Ozon'),
),
migrations.AddField(
model_name='placementpoint',
name='d17',
field=models.FloatField(null=True, verbose_name='Расстояние до ПВЗ Wildberries'),
),
migrations.AddField(
model_name='placementpoint',
name='d19',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Yandex'),
),
migrations.AddField(
model_name='placementpoint',
name='d20',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Yandex'),
),
migrations.AddField(
model_name='placementpoint',
name='d24',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Почта России'),
),
migrations.AddField(
model_name='placementpoint',
name='d25',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Почта России'),
),
migrations.AddField(
model_name='placementpoint',
name='d7',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Ozon'),
),
migrations.AddField(
model_name='placementpoint',
name='d8',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ СДЭК'),
),
migrations.AddField(
model_name='placementpoint',
name='d9',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Халва'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d10',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата СДЭК'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d12',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Ozon'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d17',
field=models.FloatField(null=True, verbose_name='Расстояние до ПВЗ Wildberries'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d19',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Yandex'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d20',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Yandex'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d24',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Почта России'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d25',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Почта России'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d7',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ Ozon'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d8',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до ПВЗ СДЭК'),
),
migrations.AddField(
model_name='preplacementpoint',
name='d9',
field=models.FloatField(blank=True, null=True, verbose_name='Расстояние до Постамата Халва'),
),
migrations.DeleteModel(
name='PlacementPointPVZDistance',
),
migrations.DeleteModel(
name='PrePlacementPointPVZDistance',
),
]

@ -0,0 +1,122 @@
# Generated by Django 3.2 on 2023-10-11 18:20
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('service', '0038_auto_20231008_1141'),
]
operations = [
migrations.RemoveField(
model_name='placementpoint',
name='d10',
),
migrations.RemoveField(
model_name='placementpoint',
name='d12',
),
migrations.RemoveField(
model_name='placementpoint',
name='d17',
),
migrations.RemoveField(
model_name='placementpoint',
name='d19',
),
migrations.RemoveField(
model_name='placementpoint',
name='d20',
),
migrations.RemoveField(
model_name='placementpoint',
name='d24',
),
migrations.RemoveField(
model_name='placementpoint',
name='d25',
),
migrations.RemoveField(
model_name='placementpoint',
name='d7',
),
migrations.RemoveField(
model_name='placementpoint',
name='d8',
),
migrations.RemoveField(
model_name='placementpoint',
name='d9',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d10',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d12',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d17',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d19',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d20',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d24',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d25',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d7',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d8',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d9',
),
migrations.AddField(
model_name='otherobjectscategory',
name='image',
field=models.ImageField(blank=True, default=None, null=True, upload_to='other_objects_category_images/', verbose_name='Картинка'),
),
migrations.AddField(
model_name='post_and_pvzcategory',
name='image',
field=models.ImageField(blank=True, default=None, null=True, upload_to='post_and_pvz_category_images/', verbose_name='Картинка'),
),
migrations.CreateModel(
name='PrePlacementPointPVZDistance',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('dist', models.FloatField(blank=True, default=None, null=True)),
('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.preplacementpoint')),
('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')),
],
),
migrations.CreateModel(
name='PlacementPointPVZDistance',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('dist', models.FloatField(blank=True, default=None, null=True)),
('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.placementpoint')),
('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')),
],
),
]

@ -1,19 +1,20 @@
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.contrib.gis.db import models as gis_models from django.contrib.gis.db import models as gis_models
from django.db import models from django.db import models
from postamates.settings import SRID from postamates.settings import SRID
from service.enums import PointStatus from service.enums import PointStatus, MatchingStatus
from service.signals import *
User._meta.get_field('email')._unique = True User._meta.get_field('email')._unique = True
class PlacementPoint(models.Model): class AbstractPlacementPoint(models.Model):
class Meta:
abstract = True
STATUS_CHOICES = [(tag.name, tag.value) for tag in PointStatus] STATUS_CHOICES = [(tag.name, tag.value) for tag in PointStatus]
address = models.TextField(null=True, blank=True, verbose_name='Адрес') address = models.TextField(null=True, blank=True, verbose_name='Адрес')
name = models.TextField(null=True, blank=True, verbose_name='Название') name = models.TextField(null=True, blank=True, verbose_name='Название')
postamat_id = models.TextField(unique=True, null=True, blank=True, verbose_name='ID постамата') postamat_id = models.IntegerField(unique=True, null=True, blank=True, verbose_name='ID постамата')
category = models.TextField(null=True, blank=True, verbose_name='Категория') category = models.TextField(null=True, blank=True, verbose_name='Категория')
status = models.TextField(choices=STATUS_CHOICES, null=True, blank=True, verbose_name='Статус') status = models.TextField(choices=STATUS_CHOICES, null=True, blank=True, verbose_name='Статус')
start_date = models.DateTimeField(null=True, blank=True) start_date = models.DateTimeField(null=True, blank=True)
@ -29,7 +30,7 @@ class PlacementPoint(models.Model):
sample_trn = models.BooleanField(null=True, blank=True) sample_trn = models.BooleanField(null=True, blank=True)
flat_cnt = models.IntegerField(null=True, blank=True, verbose_name='Количество квартир') flat_cnt = models.IntegerField(null=True, blank=True, verbose_name='Количество квартир')
year_bld = models.IntegerField(null=True, blank=True, verbose_name='Год постройки') year_bld = models.IntegerField(null=True, blank=True, verbose_name='Год постройки')
levels = models.IntegerField(null=True, blank=True) levels = models.TextField(null=True, blank=True)
enrg_cls = models.TextField(null=True, blank=True) enrg_cls = models.TextField(null=True, blank=True)
mat_nes = models.TextField(null=True, blank=True) mat_nes = models.TextField(null=True, blank=True)
doors = models.IntegerField(null=True, blank=True) doors = models.IntegerField(null=True, blank=True)
@ -80,27 +81,216 @@ class PlacementPoint(models.Model):
target_cnt_ao_mean = models.FloatField(null=True, blank=True) target_cnt_ao_mean = models.FloatField(null=True, blank=True)
target_cnt_nearby_mean = models.FloatField(null=True, blank=True) target_cnt_nearby_mean = models.FloatField(null=True, blank=True)
target_age_nearby_mean = models.FloatField(null=True, blank=True) target_age_nearby_mean = models.FloatField(null=True, blank=True)
target_dist_shap = models.IntegerField(null=True, blank=True)
target_post_cnt_shap = models.IntegerField(null=True, blank=True)
target_cnt_ao_mean_shap = models.IntegerField(null=True, blank=True)
rival_pvz_cnt_shap = models.IntegerField(null=True, blank=True)
rival_post_cnt_shap = models.IntegerField(null=True, blank=True)
metro_dist_shap = models.IntegerField(null=True, blank=True)
property_price_bargains_shap = models.IntegerField(null=True, blank=True)
property_price_offers_shap = models.IntegerField(null=True, blank=True)
property_mean_floor_shap = models.IntegerField(null=True, blank=True)
property_era_shap = models.IntegerField(null=True, blank=True)
flats_cnt_shap = models.IntegerField(null=True, blank=True)
popul_home_shap = models.IntegerField(null=True, blank=True)
popul_job_shap = models.IntegerField(null=True, blank=True)
yndxfood_sum_shap = models.IntegerField(null=True, blank=True)
yndxfood_cnt_shap = models.IntegerField(null=True, blank=True)
school_cnt_shap = models.IntegerField(null=True, blank=True)
kindergar_cnt_shap = models.IntegerField(null=True, blank=True)
public_stop_cnt_shap = models.IntegerField(null=True, blank=True)
sport_center_cnt_shap = models.IntegerField(null=True, blank=True)
pharmacy_cnt_shap = models.IntegerField(null=True, blank=True)
supermarket_cnt_shap = models.IntegerField(null=True, blank=True)
supermarket_premium_cnt_shap = models.IntegerField(null=True, blank=True)
clinic_cnt_shap = models.IntegerField(null=True, blank=True)
bank_cnt_shap = models.IntegerField(null=True, blank=True)
reca_cnt_shap = models.IntegerField(null=True, blank=True)
lab_cnt_shap = models.IntegerField(null=True, blank=True)
culture_cnt_shap = models.IntegerField(null=True, blank=True)
attraction_cnt_shap = models.IntegerField(null=True, blank=True)
mfc_cnt_shap = models.IntegerField(null=True, blank=True)
bc_cnt_shap = models.IntegerField(null=True, blank=True)
tc_cnt_shap = models.IntegerField(null=True, blank=True)
business_activity_shap = models.IntegerField(null=True, blank=True)
class PlacementPoint(AbstractPlacementPoint):
class Meta:
verbose_name = 'Точка'
verbose_name_plural = 'Точки'
ordering = ('id',)
class PrePlacementPoint(AbstractPlacementPoint):
class Meta:
ordering = ('id',)
MATCHING_CHOICES = [(tag.name, tag.value) for tag in MatchingStatus]
matching_status = models.TextField(null=True, blank=True, choices=MATCHING_CHOICES)
class AO(models.Model): class AO(models.Model):
class Meta:
verbose_name = 'АО'
verbose_name_plural = 'АО'
name = models.TextField(null=True, blank=True, verbose_name='Округ') name = models.TextField(null=True, blank=True, verbose_name='Округ')
polygon = gis_models.MultiPolygonField(null=True, srid=SRID) polygon = gis_models.MultiPolygonField(null=True, srid=SRID)
class Rayon(models.Model): class Rayon(models.Model):
class Meta:
verbose_name = 'Район'
verbose_name_plural = 'Районы'
name = models.TextField(null=True, blank=True, verbose_name='Район') name = models.TextField(null=True, blank=True, verbose_name='Район')
AO = models.ForeignKey('AO', related_name='rayons', on_delete=models.CASCADE) AO = models.ForeignKey('AO', related_name='rayons', on_delete=models.CASCADE)
polygon = gis_models.MultiPolygonField(null=True, srid=SRID) polygon = gis_models.MultiPolygonField(null=True, srid=SRID)
class Rivals(models.Model): class Post_and_pvz(models.Model):
class Meta:
verbose_name = 'Постамат или ПВЗ'
verbose_name_plural = 'Постаматы и ПВЗ'
ordering = ('id',)
wkt = gis_models.PointField(srid=SRID, null=True)
category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='post_and_pvz',
on_delete=models.CASCADE)
group = models.ForeignKey('Post_and_pvzGroup', default=None, related_name='post_and_pvz', on_delete=models.CASCADE)
visible = models.BooleanField(default=True)
include_in_ml = models.BooleanField(default=True)
class OtherObjects(models.Model):
class Meta:
verbose_name = 'Прочий объект'
verbose_name_plural = 'Прочие объекты'
ordering = ('id',)
wkt = gis_models.PointField(srid=SRID, null=True) wkt = gis_models.PointField(srid=SRID, null=True)
info = models.TextField(null=True, blank=True) category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='other_objects',
type = models.TextField(null=True, blank=True) on_delete=models.CASCADE)
source = models.TextField(null=True, blank=True) group = models.ForeignKey('OtherObjectsGroup', default=None, related_name='other_objects', on_delete=models.CASCADE)
param1 = models.FloatField(blank=True, null=True)
param2 = models.TextField(blank=True, null=True)
param3 = models.FloatField(blank=True, null=True)
param4 = models.FloatField(blank=True, null=True)
visible = models.BooleanField(default=True)
class Post_and_pvzCategory(models.Model):
class Meta:
verbose_name = 'категория постаматов и ПВЗ'
verbose_name_plural = 'Категории постаматов и ПВЗ'
def __str__(self):
return self.name
name = models.TextField(null=False, blank=False, verbose_name='Название слоя')
image = models.ImageField(blank=True, null=True, default=None, upload_to='post_and_pvz_category_images/',
verbose_name='Картинка')
visible = models.BooleanField(default=True)
include_in_ml = models.BooleanField(default=True)
class Post_and_pvzGroup(models.Model):
class Meta:
verbose_name = 'группа постаматов и ПВЗ'
verbose_name_plural = 'Группы постаматов и ПВЗ'
def __str__(self):
return self.category.name + ' ' + self.name
name = models.TextField(null=False, blank=False, verbose_name='Название группы')
category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='groups',
on_delete=models.CASCADE)
image = models.ImageField(blank=True, null=True, upload_to='post_and_pvz_group_images/', verbose_name='Картинка')
visible = models.BooleanField(default=True)
include_in_ml = models.BooleanField(default=True)
class OtherObjectsCategory(models.Model):
class Meta:
verbose_name = 'категория прочих объектов'
verbose_name_plural = 'Категории прочих объектов'
def __str__(self):
return self.name
name = models.TextField(null=False, blank=False, verbose_name='Название слоя')
image = models.ImageField(blank=True, null=True, default=None, upload_to='other_objects_category_images/',
verbose_name='Картинка')
visible = models.BooleanField(default=True)
class OtherObjectsGroup(models.Model):
class Meta:
verbose_name = 'группа прочих объектов'
verbose_name_plural = 'Группы прочих объектов'
def __str__(self):
return self.category.name + ' ' + self.name
name = models.TextField(null=False, blank=False, verbose_name='Название группы')
category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='groups',
on_delete=models.CASCADE)
image = models.ImageField(blank=True, null=True, upload_to='other_objects_group_images/', verbose_name='Картинка')
visible = models.BooleanField(default=True)
class PlacementPointPVZDistance(models.Model):
placement_point = models.ForeignKey('PlacementPoint', default=None, on_delete=models.CASCADE)
pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
dist = models.FloatField(blank=True, null=True, default=None)
class PrePlacementPointPVZDistance(models.Model):
placement_point = models.ForeignKey('PrePlacementPoint', default=None, on_delete=models.CASCADE)
pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
dist = models.FloatField(blank=True, null=True, default=None)
class TaskStatus(models.Model):
class Meta:
verbose_name = 'Статус фоновых задач'
verbose_name_plural = 'Статус фоновых задач'
def __str__(self):
return self.task_name
task_name = models.TextField(blank=False, unique=True, verbose_name='Название задачи')
status = models.TextField(blank=True, null=True, verbose_name='Статус выполнения')
class LastMLCall(models.Model):
dt = models.DateTimeField(auto_now_add=True)
class TempFiles(models.Model):
data = models.TextField(blank=False, null=False)
class RaschetGroups(models.Model):
obj_id = models.IntegerField(null=False, blank=False)
class RaschetObjects(models.Model):
obj_id = models.IntegerField(null=False, blank=False)
class House(models.Model):
class Meta:
verbose_name = 'Дом'
verbose_name_plural = 'Дома'
ordering = ('id',)
class PointDist(models.Model): year_bld = models.IntegerField(blank=True,null=True)
id1 = models.ForeignKey('PlacementPoint', on_delete=models.CASCADE, null=False, related_name='placement_point_id1') mat_nes = models.TextField(blank=True,null=True)
id2 = models.ForeignKey('PlacementPoint', on_delete=models.CASCADE, null=False, related_name='placement_point_id2') flat_cnt = models.IntegerField(blank=True, null=True)
distance = models.FloatField(null=False) levels = models.TextField(blank=True,null=True)
doors = models.IntegerField(blank=True, null=True)
enrg_cls = models.TextField(blank=True,null=True)
street = models.TextField(blank=True,null=True)
house_number = models.TextField(blank=True,null=True)

@ -1,6 +1,7 @@
from rest_framework import serializers from rest_framework import serializers
from service import models from service import models
from service.service import PointService
class PlacementPointSerializer(serializers.ModelSerializer): class PlacementPointSerializer(serializers.ModelSerializer):
@ -8,6 +9,46 @@ class PlacementPointSerializer(serializers.ModelSerializer):
model = models.PlacementPoint model = models.PlacementPoint
fields = '__all__' fields = '__all__'
def to_representation(self, instance):
representation = super().to_representation(instance)
min_distances = PointService.get_min_distances_to_group(instance.id)
representation['min_distance_to_group'] = min_distances
return representation
class PrePlacementPointSerializer(PlacementPointSerializer):
class Meta:
model = models.PrePlacementPoint
fields = '__all__'
class PostAndPVZGroupSerializer(serializers.ModelSerializer):
class Meta:
model = models.Post_and_pvzGroup
fields = '__all__'
class PostAndPVZCategorySerializer(serializers.ModelSerializer):
groups = PostAndPVZGroupSerializer(read_only=True, many=True)
class Meta:
model = models.Post_and_pvzCategory
fields = '__all__'
class OtherObjectsGroupSerializer(serializers.ModelSerializer):
class Meta:
model = models.OtherObjectsGroup
fields = '__all__'
class OtherObjectsCategorySerializer(serializers.ModelSerializer):
groups = OtherObjectsGroupSerializer(many=True, read_only=True)
class Meta:
model = models.OtherObjectsCategory
fields = '__all__'
class RayonSerializer(serializers.ModelSerializer): class RayonSerializer(serializers.ModelSerializer):
class Meta: class Meta:

@ -5,13 +5,20 @@ import pandas as pd
from django.contrib.gis.measure import Distance from django.contrib.gis.measure import Distance
from django.db.models import F from django.db.models import F
from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT
from service import models from service import models
from service.enums import PointStatus from service.enums import PointStatus
from service.tasks import raschet from service.utils import create_columns_dist, run_psql_command
import base64
import requests
from postamates.settings import GEOCODER_API_KEY
from service.enums import MatchingStatus
from django.contrib.gis.db.models.functions import Distance as Dist
from django.db.models import Avg, Sum, Count
class PointService: class PointService:
def update_fact(self, postamat_id: str, fact: int): def update_fact(self, postamat_id: str, fact: int):
qs = self.get_point_by_postamat_id(postamat_id) qs = self.get_point_by_postamat_id(postamat_id)
qs.update(**{'fact': fact}) qs.update(**{'fact': fact})
@ -20,9 +27,249 @@ class PointService:
qs = self.get_point_by_id(point_id) qs = self.get_point_by_id(point_id)
qs.update(**{'postamat_id': postamat_id}) qs.update(**{'postamat_id': postamat_id})
def start_mathing(self, obj_id: int):
file = models.TempFiles.objects.get(id=obj_id)
excel_file = base64.b64decode(file.data)
df = pd.read_excel(excel_file)
total = df.shape[0]
matched = 0
problem = 0
for _i, row in df.iterrows():
addr = row['Адрес']
cat = row['Категория объекта']
req_url = f"https://geocode.search.hereapi.com/v1/geocode?q={addr}&apiKey={GEOCODER_API_KEY}"
response = requests.get(req_url).json().get('items')
if not response:
models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
status=PointStatus.Pending.name)
problem += 1
continue
coords = response[0].get('position')
if not coords:
models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
status=PointStatus.Pending.name)
problem += 1
continue
wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")"
response = response[0]['address']
obj = models.PlacementPoint.objects.filter(street=response.get('street'),
house_number=response.get('houseNumber'),
category=cat).values()
rayon = models.Rayon.objects.filter(polygon__intersects=wkt).first()
if obj:
if cat == 'Подъезд жилого дома':
objs = obj.all()
else:
objs = [obj.first()]
for o in objs:
distances = models.PlacementPointPVZDistance.objects.filter(placement_point=o.get('id')).all()
o.pop('id')
street = o.pop('street')
house_number = o.pop('house_number')
if o.get('name'):
name = o.pop('name')
pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
house_number=house_number,name=name,
matching_status=MatchingStatus.Matched.name,
defaults=o)
else:
pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
house_number=house_number,
matching_status=MatchingStatus.Matched.name,
defaults=o)
for d in distances:
models.PrePlacementPointPVZDistance.objects.get_or_create(placement_point=pre_obj,
pvz_postamates_group=d.pvz_postamates_group,
dist=d.dist)
matched += 1
elif not rayon:
models.PrePlacementPoint.objects.get_or_create(street=response.get('street'),
house_number=response.get('houseNumber'),
address=addr,
subject_rf=response.get('state'),
city=response.get('city'),
category=cat, geometry=wkt, sample_trn=False,
is_vis=True,
matching_status=MatchingStatus.Error.name,
status=PointStatus.Pending.name)
problem += 1
elif cat == 'Подъезд жилого дома' and models.House.objects.filter(street=response.get('street'),
house_number=response.get(
'houseNumber')).first():
house = models.House.objects.filter(street=response.get('street'),
house_number=response.get('houseNumber')).values().first()
house.pop('id')
if house.get('doors') and house.get('doors') > 1:
house['flat_cnt'] = int(house['flat_cnt'] / house['doors'])
for _num in range(house['doors']):
models.PrePlacementPoint.objects.get_or_create(address=addr,
name=f'Подъезд {_num + 1}',
matching_status=MatchingStatus.New.name,
sample_trn=False,
is_vis=True, category=cat, geometry=wkt,
status=PointStatus.Pending.name, area=rayon,
district=rayon.AO, defaults=house)
else:
models.PrePlacementPoint.objects.get_or_create(address=addr,
matching_status=MatchingStatus.New.name,
sample_trn=False,
is_vis=True, category=cat, geometry=wkt,
status=PointStatus.Pending.name, area=rayon,
district=rayon.AO, defaults=house)
else:
models.PrePlacementPoint.objects.get_or_create(address=addr, street=response.get('street'),
house_number=response.get('houseNumber'),
subject_rf=response.get('state'),
city=response.get('city'),
category=cat, geometry=wkt, sample_trn=False,
is_vis=True,
matching_status=MatchingStatus.New.name,
status=PointStatus.Pending.name, area=rayon,
district=rayon.AO)
return total, matched, problem
def make_enrichment(self):
points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all()
groups = models.Post_and_pvzGroup.objects.all()
for point in points:
origin = point.geometry
qs = models.PlacementPoint.objects.filter(status=PointStatus.Working.name).annotate(
dist=Dist('geometry', origin)).order_by('dist')
if qs:
point.target_dist = qs[0].dist.m
point.target_post_cnt = qs.filter(
dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)
).count()
point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean
point.rival_post_cnt = models.Post_and_pvz.objects.filter(
category__name="Постаматы прочих сетей", include_in_ml=True,
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.rival_pvz_cnt = models.Post_and_pvz.objects.filter(
category__name="ПВЗ", include_in_ml=True,
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate(
dist=Dist('wkt', origin)).order_by('dist')[0].dist.m
point.property_price_bargains = models.OtherObjects.objects.filter(
group__name="bargains",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[
'param1__avg']
offers_estate = models.OtherObjects.objects.filter(
group__name="offers_estate",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__avg=Avg('param1'), param3__avg=Avg('param3'))
point.property_price_offers = offers_estate['param1__avg']
point.property_mean_floor = offers_estate['param3__avg']
point.property_era = models.OtherObjects.objects.filter(
group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[
'param2']
point.flats_cnt = models.OtherObjects.objects.filter(
group__name="flats_cnt",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'))['param1__sum']
popul_home_job = models.OtherObjects.objects.filter(
group__name="popul_home_job",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'), param3__sum=Sum('param3'))
point.popul_home = popul_home_job['param1__sum']
point.popul_job = popul_home_job['param3__sum']
yndx_food_cnt_amt = models.OtherObjects.objects.filter(
group__name="yndx_food_cnt_amt",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'), param3__sum=Sum('param3'))
point.yndxfood_sum = yndx_food_cnt_amt['param1__sum']
point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum']
point.school_cnt = models.OtherObjects.objects.filter(
group__name="schools",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.kindergar_cnt = models.OtherObjects.objects.filter(
group__name="kindergar",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.public_stop_cnt = models.OtherObjects.objects.filter(
group__name="stops",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.sport_center_cnt = models.OtherObjects.objects.filter(
group__name="sport_centers",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.pharmacy_cnt = models.OtherObjects.objects.filter(
group__name="pharmacies",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.supermarket_cnt = models.OtherObjects.objects.filter(
group__name="supermarkets",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.supermarket_premium_cnt = models.OtherObjects.objects.filter(
group__name="supermarkets_premium",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.clinic_cnt = models.OtherObjects.objects.filter(
group__name="clinics",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.bank_cnt = models.OtherObjects.objects.filter(
group__name="banks",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.reca_cnt = models.OtherObjects.objects.filter(
group__name="recas",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.lab_cnt = models.OtherObjects.objects.filter(
group__name="labs",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.culture_cnt = models.OtherObjects.objects.filter(
group__name="cultures",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.attraction_cnt = models.OtherObjects.objects.filter(
group__name="attractions",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.mfc_cnt = models.OtherObjects.objects.filter(
group__name="public_services",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.bc_cnt = models.OtherObjects.objects.filter(
group__name="BC",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.tc_cnt = models.OtherObjects.objects.filter(
group__name="TC",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.business_activity = models.OtherObjects.objects.filter(
group__name="business_activity",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'))['param1__sum']
point.age_day = AGE_DAY_LIMIT
placement_point = models.PlacementPoint.objects.annotate(
dist=Dist('geometry', origin)).order_by('dist')
if placement_point:
placement_point = placement_point[0]
point.target_cnt_ao_mean = placement_point.target_cnt_ao_mean
point.save()
for group in groups:
self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance)
run_psql_command()
@staticmethod
def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance):
post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate(
distance=Dist("wkt", point.geometry)).order_by('distance').first()
d = instance_type.objects.filter(placement_point=point,
pvz_postamates_group=group).first()
if post_object:
if d:
if d.dist > post_object.distance.m:
d.dist = post_object.distance.m
d.save()
else:
instance_type.objects.create(placement_point=point,
pvz_postamates_group=group,
dist=post_object.distance.m)
@staticmethod
def delete_preplacement_points(ids: list):
models.PrePlacementPoint.objects.filter(id__in=ids).all().delete()
@staticmethod
def get_min_distances_to_group(postamat_id: str):
return {d['pvz_postamates_group']: d['dist'] for d in list(
models.PlacementPointPVZDistance.objects.filter(placement_point=postamat_id).values(
'pvz_postamates_group', 'dist'))}
@staticmethod @staticmethod
def update_points_in_radius(qs: models.PlacementPoint, new_status: str): def update_points_in_radius(qs: models.PlacementPoint, new_status: str):
triggers = False
for point in qs: for point in qs:
if new_status == PointStatus.Installation.name: if new_status == PointStatus.Installation.name:
if point.status == PointStatus.Pending.name: if point.status == PointStatus.Pending.name:
@ -30,18 +277,12 @@ class PointService:
geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)), geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)),
) )
pnts.update(prediction_first=F('prediction_current'), target_post_cnt=F('target_post_cnt') + 1) pnts.update(prediction_first=F('prediction_current'), target_post_cnt=F('target_post_cnt') + 1)
triggers = True
elif new_status == PointStatus.Cancelled.name or new_status == PointStatus.Pending.name: elif new_status == PointStatus.Cancelled.name or new_status == PointStatus.Pending.name:
if point.status == PointStatus.Installation.name: if point.status == PointStatus.Installation.name:
pnts = models.PlacementPoint.objects.filter( pnts = models.PlacementPoint.objects.filter(
geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)), geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)),
) )
pnts.update(target_post_cnt=F('target_post_cnt') - 1 if F('target_post_cnt') != 0 else 0) pnts.update(target_post_cnt=F('target_post_cnt') - 1 if F('target_post_cnt') != 0 else 0)
triggers = True
elif new_status == PointStatus.Working.name and point.status == PointStatus.Pending.name:
triggers = True
if triggers:
raschet.delay()
@staticmethod @staticmethod
def update_status(qs: models.PlacementPoint, new_status: str) -> models.PlacementPoint: def update_status(qs: models.PlacementPoint, new_status: str) -> models.PlacementPoint:
@ -60,13 +301,26 @@ class PointService:
return models.PlacementPoint.objects.filter(postamat_id=postamat_id) return models.PlacementPoint.objects.filter(postamat_id=postamat_id)
@staticmethod @staticmethod
def to_excel(qs: models.PlacementPoint): def to_excel(serializer):
data = pd.DataFrame(list(qs.values())) data = pd.DataFrame(serializer.data)
if not data.empty: if not data.empty:
if data['start_date'].any(): if data['start_date'].any():
data['start_date'] = data.get('start_date').dt.tz_localize(None) data['start_date'] = data['start_date'].apply(lambda x : pd.to_datetime(x).tz_localize(None) if x else None)
data['sample_trn'] = data['sample_trn'].astype(int) if data['sample_trn'].any():
data['sample_trn'] = data['sample_trn'].astype(int)
data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True)
data['min_distance_to_group'] = data['min_distance_to_group'].apply(lambda x: list(x.items()))
new_columns = data.apply(create_columns_dist, axis=1)
for ind in new_columns.columns:
expanded = new_columns[ind].apply(pd.Series)
group = models.Post_and_pvzGroup.objects.get(id=int(expanded.loc[0, 0]))
expanded[[f"group_{ind + 1}_name", f"group_{ind + 1}_category"]] = group.name, group.category.name
expanded = expanded.rename(columns={1: f"dist_to_group_{ind + 1}"})
expanded = expanded.drop(0, axis=1)
new_columns = pd.concat([new_columns, expanded], axis=1)
new_columns = new_columns.drop(ind, axis=1)
data.drop('min_distance_to_group', axis=1, inplace=True)
data = pd.concat([data, new_columns], axis=1)
with BytesIO() as b: with BytesIO() as b:
with pd.ExcelWriter(b) as writer: with pd.ExcelWriter(b) as writer:
data.to_excel( data.to_excel(
@ -76,13 +330,25 @@ class PointService:
return b.getvalue() return b.getvalue()
@staticmethod @staticmethod
def to_json(qs: models.PlacementPoint): def to_json(serializer):
data = pd.DataFrame(list(qs.values())) data = pd.DataFrame(serializer.data)
data['start_date'] = pd.to_datetime(data['start_date'], errors='coerce') data['start_date'] = pd.to_datetime(data['start_date'], errors='coerce')
data['start_date'] = data['start_date'].dt.tz_localize(None) data['start_date'] = data['start_date'].dt.tz_localize(None)
data['sample_trn'] = data['sample_trn'].astype(int) data['sample_trn'] = data['sample_trn'].astype(int)
data['geometry'] = data['geometry'].apply(lambda x: {'latitude': x[1], 'longtitude': x[0]}) data['geometry'] = data['geometry'].apply(lambda x: {'latitude': x[1], 'longtitude': x[0]})
data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True)
data['min_distance_to_group'] = data['min_distance_to_group'].apply(lambda x: list(x.items()))
new_columns = data.apply(create_columns_dist, axis=1)
for ind in new_columns.columns:
expanded = new_columns[ind].apply(pd.Series)
group = models.Post_and_pvzGroup.objects.get(id=int(expanded.loc[0, 0]))
expanded[[f"group_{ind + 1}_name", f"group_{ind + 1}_category"]] = group.name, group.category.name
expanded = expanded.rename(columns={1: f"dist_to_group_{ind + 1}"})
expanded = expanded.drop(0, axis=1)
new_columns = pd.concat([new_columns, expanded], axis=1)
new_columns = new_columns.drop(ind, axis=1)
data.drop('min_distance_to_group', axis=1, inplace=True)
data = pd.concat([data, new_columns], axis=1)
return data.to_json(orient='records') return data.to_json(orient='records')
@staticmethod @staticmethod

@ -1,5 +1,9 @@
from django.contrib.auth.models import Group from django.contrib.auth.models import Group
from rest_registration.signals import user_registered from rest_registration.signals import user_registered
from service.models import OtherObjects, OtherObjectsGroup, \
OtherObjectsCategory
from django.db.models.signals import post_save
from django.dispatch import receiver
def user_created(sender, user, request, **kwargs): def user_created(sender, user, request, **kwargs):
@ -9,3 +13,16 @@ def user_created(sender, user, request, **kwargs):
user_registered.connect(user_created) user_registered.connect(user_created)
@receiver(post_save, sender=OtherObjectsGroup)
def other_group_handler(sender, instance, **kwargs):
OtherObjects.objects.filter(group=instance).update(visible=instance.visible)
@receiver(post_save, sender=OtherObjectsCategory)
def other_category_handler(sender, instance, **kwargs):
objects = OtherObjectsGroup.objects.filter(category=instance)
for obj in objects:
obj.visible = instance.visible
obj.save()

@ -4,6 +4,7 @@ import catboost
import geopandas as gpd import geopandas as gpd
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import shap
import psycopg2 import psycopg2
import sqlalchemy import sqlalchemy
from celery import shared_task from celery import shared_task
@ -14,21 +15,56 @@ from shapely import wkb
from sklearn import metrics from sklearn import metrics
from sklearn import model_selection as ms from sklearn import model_selection as ms
from sqlalchemy import text from sqlalchemy import text
from django.contrib.gis.db.models.functions import Distance
import requests
from postamates.settings import AGE_DAY_LIMIT from postamates.settings import AGE_DAY_LIMIT
from postamates.settings import DB_URL from postamates.settings import DB_URL, STATUS_TASK_NAME
from service.models import PlacementPoint from service.models import PlacementPoint, LastMLCall
from service import models
from service.utils import log_to_telegram
def log_to_telegram(msg): import base64
requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage', json={"chat_id": "-555238820", "text": msg}) from io import StringIO
from django.core.cache import cache
from service.layer_service import LayerService
from service.service import PointService
from service.utils import run_psql_command
@shared_task() @shared_task()
def raschet(): def raschet(table_name='service_placementpoint', need_time=True):
log_to_telegram('start raschet') print('start raschet')
status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME)
raschet_objs = models.RaschetObjects.objects.all()
if raschet_objs:
status.status = 'Начало расчета кол-ва ПВЗ вокруг точек'
status.save()
total = raschet_objs.count()
for _i, r_o in enumerate(raschet_objs):
obj = models.Post_and_pvz.objects.get(id=r_o.obj_id)
LayerService().count_post_pvz_for_placementpoint(obj)
status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%"
status.save()
status.status = 'Расчет кол-ва ПВЗ вокруг точек завершен'
status.save()
group_objects = models.RaschetGroups.objects.all()
group_total = group_objects.count()
if group_objects:
status.status = 'Начало расчета расстояний'
status.save()
qs = models.PlacementPoint.objects.all()
for _k, g_o in enumerate(group_objects):
g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id)
for q in qs:
PointService.calculate_dist_for_group(point=q, group=g)
status.status = "Подсчет расстояний: " + str(int(_k / group_total * 100)) + "%"
status.save()
status.status = "Подсчет расстояний завершен"
status.save()
models.RaschetObjects.objects.all().delete()
models.RaschetGroups.objects.all().delete()
# Запуск ML
status.status = 'Запуск ML'
status.save()
log_to_telegram(f'{table_name} start raschet')
try: try:
log_to_telegram('try connect to db') log_to_telegram('try connect to db')
conn = sqlalchemy.create_engine( conn = sqlalchemy.create_engine(
@ -44,23 +80,16 @@ def raschet():
pts['geometry'] = pts['geometry'].apply(wkb.loads, hex=True) pts['geometry'] = pts['geometry'].apply(wkb.loads, hex=True)
pts = gpd.GeoDataFrame(pts, geometry='geometry', crs='epsg:4326') pts = gpd.GeoDataFrame(pts, geometry='geometry', crs='epsg:4326')
pts = pts.to_crs('epsg:32637') pts = pts.to_crs('epsg:32637')
pts = pts.rename(
columns={
'target_cnt_nearby_mean': 'target_dist1',
'target_age_nearby_mean': 'target_dist2',
'yndxfood_cnt_cst': 'target_dist3',
},
)
feats = [ feats = [
'id', 'metro_dist', 'target_dist', 'property_price_bargains', 'property_price_offers', 'id', 'metro_dist', 'target_dist', 'property_price_bargains', 'property_price_offers',
'property_mean_floor', 'property_mean_floor',
'property_era', 'flats_cnt_2', 'flats_cnt', 'popul_home', 'popul_job', 'other_post_cnt', 'yndxfood_sum', 'property_era', 'flats_cnt', 'popul_home', 'popul_job', 'yndxfood_sum',
'yndxfood_cnt', 'school_cnt', 'kindergar_cnt', 'target_post_cnt', 'public_stop_cnt', 'sport_center_cnt', 'yndxfood_cnt', 'school_cnt', 'kindergar_cnt', 'target_post_cnt', 'public_stop_cnt', 'sport_center_cnt',
'pharmacy_cnt', 'supermarket_cnt', 'supermarket_premium_cnt', 'clinic_cnt', 'bank_cnt', 'reca_cnt', 'pharmacy_cnt', 'supermarket_cnt', 'supermarket_premium_cnt', 'clinic_cnt', 'bank_cnt', 'reca_cnt',
'lab_cnt', 'culture_cnt', 'attraction_cnt', 'mfc_cnt', 'bc_cnt', 'tc_cnt', 'rival_pvz_cnt', 'lab_cnt', 'culture_cnt', 'attraction_cnt', 'mfc_cnt', 'bc_cnt', 'tc_cnt', 'rival_pvz_cnt',
'rival_post_cnt', 'rival_post_cnt',
'business_activity', 'age_day', 'target_cnt_ao_mean', 'target_dist1', 'target_dist2', 'target_dist3', 'business_activity', 'age_day', 'target_cnt_ao_mean'
] ]
# Записи для обучения # Записи для обучения
@ -81,24 +110,6 @@ def raschet():
) )
pts_trn.loc[pts_trn.target_dist > 700, 'target_dist'] = 700 pts_trn.loc[pts_trn.target_dist > 700, 'target_dist'] = 700
pts_trn['target_dist1'] = pts_trn.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[2])),
axis=1,
)
pts_trn.loc[pts_trn.target_dist1 > 700, 'target_dist1'] = 700
pts_trn['target_dist2'] = pts_trn.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[3])),
axis=1,
)
pts_trn.loc[pts_trn.target_dist2 > 700, 'target_dist2'] = 700
pts_trn['target_dist3'] = pts_trn.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[4])),
axis=1,
)
pts_trn.loc[pts_trn.target_dist3 > 700, 'target_dist3'] = 700
pts_trn['buf'] = pts_trn.buffer(500) pts_trn['buf'] = pts_trn.buffer(500)
pts_trn = gpd.GeoDataFrame(pts_trn, geometry='buf', crs='epsg:32637') pts_trn = gpd.GeoDataFrame(pts_trn, geometry='buf', crs='epsg:32637')
target_post = gpd.sjoin(pts_trn, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) target_post = gpd.sjoin(pts_trn, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'})
@ -110,213 +121,393 @@ def raschet():
X_trn = pts_trn[feats].drop(columns=['id']) X_trn = pts_trn[feats].drop(columns=['id'])
Y_trn = pts_trn[['fact']] Y_trn = pts_trn[['fact']]
# Записи для инференса
pts_inf = pts.loc[(pts.status == 'Pending') |
(pts.status == 'Installation') |
(pts.status == 'Cancelled') |
((pts.status == 'Working') & (pts.sample_trn == False))].reset_index(drop=True)
pts_inf = gpd.GeoDataFrame(pts_inf, geometry='geometry', crs='epsg:32637')
pts_inf['buf'] = pts_inf.buffer(500)
pts_inf = gpd.GeoDataFrame(pts_inf, geometry='buf', crs='epsg:32637')
pts_target = pts.loc[(pts.status == 'Working') |
(pts.status == 'Installation') |
(pts.sample_trn == True)].reset_index(drop=True)
pts_target = pts_target[['geometry']]
pts_target['cnt'] = 1
pts_target = gpd.GeoDataFrame(pts_target, geometry='geometry', crs='epsg:32637')
target_feature_coords = []
for i in range(0, len(pts_target)):
target_feature_coords.append((pts_target.geometry.x[i], pts_target.geometry.y[i]))
target_feature_coords = np.array(target_feature_coords)
pts_inf['target_dist'] = pts_inf.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])),
axis=1,
)
pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700
pts_inf['target_dist1'] = pts_inf.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[1])),
axis=1,
)
pts_inf.loc[pts_inf.target_dist1 > 700, 'target_dist1'] = 700
pts_inf['target_dist2'] = pts_inf.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[2])),
axis=1,
)
pts_inf.loc[pts_inf.target_dist2 > 700, 'target_dist2'] = 700
pts_inf['target_dist3'] = pts_inf.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[3])),
axis=1,
)
pts_inf.loc[pts_inf.target_dist3 > 700, 'target_dist3'] = 700
pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True) status.status = 'Записи для инференса'
target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) status.save()
target_post = target_post.rename(columns={'cnt': 'target_post_cnt'}) # Записи для инференса
pts_inf = pts_inf.drop(columns=['target_post_cnt']) if table_name == 'service_placementpoint':
pts_inf = pts_inf.join(target_post.set_index('id'), on='id') pts_inf = pts.loc[(pts.status == 'Pending') |
pts_inf['age_day_init'] = pts_inf['age_day'] (pts.status == 'Installation') |
pts_inf['age_day'] = 240 (pts.status == 'Cancelled') |
X_inf = pts_inf[feats] ((pts.status == 'Working') & (pts.sample_trn == False))].reset_index(drop=True)
elif table_name == 'service_preplacementpoint':
seeds = [3, 99, 87, 21, 15] pts_inf = pd.DataFrame(conn.connect().execute(text(f"select * from {table_name}")))
pts_inf = pts_inf.loc[pts_inf.matching_status == 'New'].reset_index(drop=True)
# Обучение, инференс pts_inf['geometry'] = pts_inf['geometry'].apply(wkb.loads, hex=True)
r2_scores = [] pts_inf = gpd.GeoDataFrame(pts_inf, geometry='geometry', crs='epsg:4326')
mapes = [] if len(pts_inf) > 0:
y_infers = [] pts_inf = pts_inf.to_crs('epsg:32637')
for i in seeds: pts_inf['buf'] = pts_inf.buffer(500)
x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i) pts_inf = gpd.GeoDataFrame(pts_inf, geometry='buf', crs='epsg:32637')
model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i) pts_target = pts.loc[(pts.status == 'Working') |
model.fit(x_trn, y_trn, verbose=False) (pts.status == 'Installation')].reset_index(drop=True)
r2_score = metrics.r2_score(y_test, model.predict(x_test)) pts_target = pts_target[['geometry']]
mape = metrics.mean_absolute_percentage_error(y_test, model.predict(x_test)) pts_target['cnt'] = 1
if ((r2_score > 0.45) & (mape < 0.25)): pts_target = gpd.GeoDataFrame(pts_target, geometry='geometry', crs='epsg:32637')
r2_scores.append(r2_score)
mapes.append(mape) target_feature_coords = []
y_infers.append(model.predict(X_inf.drop(columns=['id']))) for i in range(0, len(pts_target)):
target_feature_coords.append((pts_target.geometry.x[i], pts_target.geometry.y[i]))
current_pred = sum(y_infers) / 5 target_feature_coords = np.array(target_feature_coords)
# Обновление полей по результатам работы модели pts_inf['target_dist'] = pts_inf.apply(
update_fields = pts_inf[ lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])),
[ axis=1,
'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first', )
'prediction_first', pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700
]
] pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True)
update_fields = update_fields.join( target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'})
pd.concat( target_post = target_post.rename(columns={'cnt': 'target_post_cnt'})
pts_inf = pts_inf.drop(columns=['target_post_cnt'])
pts_inf = pts_inf.join(target_post.set_index('id'), on='id')
pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0)
pts_inf['age_day_init'] = pts_inf['age_day']
pts_inf['age_day'] = 240
X_inf = pts_inf[feats]
seeds = [3, 99, 87, 21, 15]
# Обучение, инференс
r2_scores = []
mapes = []
y_infers = []
status.status = 'Обучение inference 0%'
status.save()
for i in seeds:
status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%'
status.save()
x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i)
model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i)
model.fit(x_trn, y_trn, verbose=False)
r2_score = metrics.r2_score(y_test, model.predict(x_test))
mape = metrics.mean_absolute_percentage_error(y_test, model.predict(x_test))
if ((r2_score > 0.45) & (mape < 0.25)):
r2_scores.append(r2_score)
mapes.append(mape)
y_infers.append(model.predict(X_inf.drop(columns=['id'])))
status.status = 'Обучение inference 100%'
current_pred = sum(y_infers) / 5
# расчет шапов
explainer = shap.TreeExplainer(model)
shap_values = explainer(X_inf.drop(columns=['id']))
shap_fields = pd.DataFrame(shap_values.values)
shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap'
shap_fields = shap_fields.drop(columns = ['age_day_shap'])
shap_fields['sum'] = abs(shap_fields).sum(axis=1)
shap_fields = round(shap_fields.iloc[:,:32].div(shap_fields['sum'], axis=0)*100, 2)
# Обновление полей по результатам работы модели
update_fields = pts_inf[
[ [
X_inf[['id']], 'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first',
pd.DataFrame({'prediction_current': current_pred}), 'prediction_first', 'target_post_cnt', 'target_dist'
], ]
]
update_fields = update_fields.join(
pd.concat(
[
X_inf[['id']],
pd.DataFrame({'prediction_current': current_pred}),
],
axis=1,
).set_index('id'),
on='id',
)
update_fields['prediction_current'] = update_fields['prediction_current'].astype(int)
days_x = np.array([0, 30, 60, 90, 120, 150, 180, 210, 240, 270])
perc_y = np.array([0, 0.15, 0.20, 0.30, 0.60, 0.70, 0.70, 0.75, 0.75, 0.80])
spl = interpolate.splrep(days_x, perc_y)
update_fields['plan_first'] = update_fields.apply(
lambda x: (x.prediction_first * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0),
axis=1, axis=1,
).set_index('id'), )
on='id', update_fields['plan_current'] = update_fields.apply(
) lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0),
update_fields['prediction_current'] = update_fields['prediction_current'].astype(int) axis=1,
)
days_x = np.array([0, 30, 60, 90, 120, 150, 180, 210, 240, 270]) update_fields['delta_first'] = update_fields.apply(
perc_y = np.array([0, 0.15, 0.20, 0.30, 0.60, 0.70, 0.70, 0.75, 0.75, 0.80]) lambda x: ((x.fact - x.plan_first) / x.plan_first * 100 if x.status == 'Working' else 0),
spl = interpolate.splrep(days_x, perc_y) axis=1,
)
update_fields['plan_first'] = update_fields.apply( update_fields['delta_current'] = update_fields.apply(
lambda x: (x.prediction_first * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), lambda x: ((x.fact - x.plan_current) / x.plan_current * 100 if x.status == 'Working' else 0),
axis=1, axis=1,
) )
update_fields['plan_current'] = update_fields.apply(
lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0),
axis=1,
)
update_fields['delta_first'] = update_fields.apply(
lambda x: ((x.fact - x.plan_first) / x.plan_first * 100 if x.status == 'Working' else 0),
axis=1,
)
update_fields['delta_current'] = update_fields.apply(
lambda x: ((x.fact - x.plan_current) / x.plan_current * 100 if x.status == 'Working' else 0),
axis=1,
)
update_fields_working = update_fields.loc[update_fields.status == 'Working'].reset_index(drop=True) update_fields_working = update_fields.loc[update_fields.status == 'Working'].reset_index(drop=True)
update_fields_working = update_fields_working.fillna(0) update_fields_working = update_fields_working.fillna(0)
connection.close()
except Exception as e: except Exception as e:
log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}') log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}')
log_to_telegram('Начинается обновление полей в базе') log_to_telegram('Начинается обновление полей в базе')
# Загрузка в базу обновленных значений if len(pts_inf) > 0:
try: status.status = 'Перерасчет ML: 50%'
log_to_telegram('Подключение к базе данных 2') status.save()
conn2 = psycopg2.connect( # Загрузка в базу обновленных значений
database=os.getenv('POSTGRES_DB', 'postgres'), user=os.getenv('POSTGRES_USER', 'postgres'), try:
password=os.getenv('POSTGRES_PASSWORD', 'postgres'), log_to_telegram('Подключение к базе данных 2')
host=os.getenv('POSTGRES_HOST', 'postgres'), port=os.getenv('POSTGRES_PORT', 'postgres'), conn2 = psycopg2.connect(
options='-c search_path=public', database=os.getenv('POSTGRES_DB', 'postgres'), user=os.getenv('POSTGRES_USER', 'postgres'),
) password=os.getenv('POSTGRES_PASSWORD', 'postgres'),
cursor = conn2.cursor() host=os.getenv('POSTGRES_HOST', 'postgres'), port=os.getenv('POSTGRES_PORT', 'postgres'),
except: options='-c search_path=public',
log_to_telegram('Не удалось подключиться к базе данных') )
cursor = conn2.cursor()
# prediction_current except:
update_records1 = [] conn2 = None
for i in range(0, len(update_fields)): log_to_telegram('Не удалось подключиться к базе данных')
update_records1.append((int(update_fields.prediction_current[i]), int(update_fields.id[i])))
sql_update_query = """Update service_placementpoint set prediction_current = %s where id = %s""" if conn2 is not None:
try: # апдейт шапов
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) update_fields_shap = pd.concat([shap_fields, update_fields[['id']]], axis=1)
conn2.commit() update_records0 = []
except Exception: for i in range(0, len(update_fields_shap)):
cursor.execute('ROLLBACK') update_records1 = []
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1) for n in list(update_fields_shap):
conn2.commit() update_records1.append(int(update_fields_shap[n][i]))
update_records0.append(tuple(update_records1))
# plan_first shap_fields_name = str(list(shap_fields))[1:-1].replace("'", "").replace(',', '=%s,')
update_records2 = [] sql_update_query = f"""Update {table_name} set {shap_fields_name} = %s where id = %s"""
for i in range(0, len(update_fields_working)): try:
update_records2.append((int(update_fields_working.plan_first[i]), int(update_fields_working.id[i]))) psycopg2.extras.execute_batch(cursor, sql_update_query, update_records0)
sql_update_query = """Update service_placementpoint set plan_first = %s where id = %s""" conn2.commit()
try: except:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2) cursor.execute("ROLLBACK")
conn2.commit() psycopg2.extras.execute_batch(cursor, sql_update_query, update_records0)
except Exception: conn2.commit()
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2) # target_post_cnt
conn2.commit() update_records1 = []
for i in range(0, len(update_fields)):
# plan_current update_records1.append((int(update_fields.target_post_cnt[i]), int(update_fields.id[i])))
update_records3 = [] sql_update_query = f"""Update {table_name} set target_post_cnt = %s where id = %s"""
for i in range(0, len(update_fields_working)): try:
update_records3.append((int(update_fields_working.plan_current[i]), int(update_fields_working.id[i]))) psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1)
sql_update_query = """Update service_placementpoint set plan_current = %s where id = %s""" conn2.commit()
try: except:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3) cursor.execute("ROLLBACK")
conn2.commit() psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1)
except Exception: conn2.commit()
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3) # target_dist
conn2.commit() update_records1 = []
for i in range(0, len(update_fields)):
# delta_first update_records1.append((int(update_fields.target_dist[i]), int(update_fields.id[i])))
update_records4 = [] sql_update_query = f"""Update {table_name} set target_dist = %s where id = %s"""
for i in range(0, len(update_fields_working)): try:
update_records4.append((int(update_fields_working.delta_first[i]), int(update_fields_working.id[i]))) psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1)
sql_update_query = """Update service_placementpoint set delta_first = %s where id = %s""" conn2.commit()
try: except:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4) cursor.execute("ROLLBACK")
conn2.commit() psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1)
except Exception: conn2.commit()
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4) # prediction_current
conn2.commit() update_records1 = []
for i in range(0, len(update_fields)):
# delta_current update_records1.append((int(update_fields.prediction_current[i]), int(update_fields.id[i])))
update_records5 = [] sql_update_query = f"""Update {table_name} set prediction_current = %s where id = %s"""
for i in range(0, len(update_fields_working)): try:
update_records5.append((int(update_fields_working.delta_current[i]), int(update_fields_working.id[i]))) psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1)
sql_update_query = """Update service_placementpoint set delta_current = %s where id = %s""" conn2.commit()
try: except Exception:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5) cursor.execute('ROLLBACK')
conn2.commit() psycopg2.extras.execute_batch(cursor, sql_update_query, update_records1)
except Exception: conn2.commit()
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5) # plan_first
conn2.commit() update_records2 = []
for i in range(0, len(update_fields_working)):
update_records2.append((int(update_fields_working.plan_first[i]), int(update_fields_working.id[i])))
sql_update_query = f"""Update {table_name} set plan_first = %s where id = %s"""
try:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2)
conn2.commit()
except Exception:
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records2)
conn2.commit()
# plan_current
update_records3 = []
for i in range(0, len(update_fields_working)):
update_records3.append((int(update_fields_working.plan_current[i]), int(update_fields_working.id[i])))
sql_update_query = f"""Update {table_name} set plan_current = %s where id = %s"""
try:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3)
conn2.commit()
except Exception:
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records3)
conn2.commit()
# delta_first
update_records4 = []
for i in range(0, len(update_fields_working)):
update_records4.append((int(update_fields_working.delta_first[i]), int(update_fields_working.id[i])))
sql_update_query = f"""Update {table_name} set delta_first = %s where id = %s"""
try:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4)
conn2.commit()
except Exception:
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records4)
conn2.commit()
# delta_current
update_records5 = []
for i in range(0, len(update_fields_working)):
update_records5.append((int(update_fields_working.delta_current[i]), int(update_fields_working.id[i])))
sql_update_query = f"""Update {table_name} set delta_current = %s where id = %s"""
try:
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5)
conn2.commit()
except Exception:
cursor.execute('ROLLBACK')
psycopg2.extras.execute_batch(cursor, sql_update_query, update_records5)
conn2.commit()
conn2.close()
cache.clear()
else:
log_to_telegram('len(pts_inf) <= 0')
run_psql_command()
log_to_telegram('end raschet') log_to_telegram('end raschet')
status.status = 'Перерасчет ML завершен'
status.save()
if need_time:
LastMLCall.objects.all().delete()
LastMLCall.objects.create()
@shared_task
def load_post_and_pvz(obj_id: int):
file = models.TempFiles.objects.get(id=obj_id)
status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка ПВЗ и Постаматов')
excel_file = base64.b64decode(file.data)
df = pd.read_excel(excel_file)
df = df.replace(np.nan, None)
df = df.replace('NaT', None)
df.columns = df.columns.str.lower()
data_len = df.shape[0]
for _ind, row in enumerate(df.to_dict('records')):
status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%"
status.save()
category = row.get('category')
group = row.get('group')
if category:
cat, _ = models.Post_and_pvzCategory.objects.get_or_create(name=category)
if group:
gr, _ = models.Post_and_pvzGroup.objects.get_or_create(name=group, category=cat)
row['category'] = cat
row['group'] = gr
lon = str(row.pop('lon'))
lat = str(row.pop("lat"))
row['wkt'] = "POINT(" + lon + " " + lat + ")"
models.Post_and_pvz.objects.get_or_create(**row)
status.status = "Загрузка данных завершена"
status.save()
groups = df[['group', 'category']].drop_duplicates().to_dict(orient='records')
points = models.PlacementPoint.objects.all()
num_points = points.count()
total = len(groups) * num_points
for _i, gr in enumerate(groups):
group = models.Post_and_pvzGroup.objects.get(name=gr['group'], category__name=gr['category'])
for _j, point in enumerate(points):
status.status = "Подсчет расстояний: " + str(int((num_points * _i + _j) / total * 100)) + "%"
status.save()
post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate(
distance=Distance("wkt", point.geometry)).order_by('distance').first()
d = models.PlacementPointPVZDistance.objects.filter(placement_point=point,
pvz_postamates_group=group).first()
if d:
if d.dist > post_object.distance.m:
d.dist = post_object.distance.m
d.save()
else:
models.PlacementPointPVZDistance.objects.create(placement_point=point, pvz_postamates_group=group,
dist=post_object.distance.m)
status.status = "Подсчет расстояний завершен"
status.save()
point_qs = models.PlacementPoint.objects.all()
data_len = models.PlacementPoint.objects.count()
for _ind, point in enumerate(point_qs):
status.status = "Пересчет параметров точек: " + str(int(_ind / data_len * 100)) + "%"
status.save()
LayerService.count_post_pvz(point)
status.status = "Завершено"
cache.clear()
status.save()
@shared_task() @shared_task()
def add_age_day(): def add_age_day():
qs = PlacementPoint.objects qs = PlacementPoint.objects.filter(status='Working')
c1 = qs.filter(sample_trn=True).count() # c1 = qs.filter(sample_trn=True).count()
qs.update(age_day=F('age_day') + 1) qs.update(age_day=F('age_day') + 1)
qs2 = qs.filter(age_day__gt=AGE_DAY_LIMIT) qs2 = qs.filter(age_day__gt=AGE_DAY_LIMIT)
qs2.update(sample_trn=True) qs2.update(sample_trn=True)
c2 = PlacementPoint.objects.filter(sample_trn=True).count() # c2 = PlacementPoint.objects.filter(sample_trn=True).count()
if c2 - c1 != 0: # if c2 - c1 != 0:
raschet.delay() # raschet.delay()
@shared_task()
def load_other_objects(obj_id: int):
file = models.TempFiles.objects.get(id=obj_id)
status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Прочих объектов')
excel_file = base64.b64decode(file.data)
df = pd.read_excel(excel_file)
df = df.replace(np.nan, None)
df = df.replace('NaT', None)
df.columns = df.columns.str.lower()
data_len = df.shape[0]
for _ind, row in enumerate(df.to_dict('records')):
status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%"
status.save()
category = row.get('category')
group = row.get('group')
if category:
cat, _ = models.OtherObjectsCategory.objects.get_or_create(name=category)
if group:
gr, _ = models.OtherObjectsGroup.objects.get_or_create(name=group, category=cat)
row['category'] = cat
row['group'] = gr
lon = str(row.pop('lon'))
lat = str(row.pop("lat"))
row['wkt'] = "POINT(" + lon + " " + lat + ")"
models.OtherObjects.objects.get_or_create(**row)
status.status = "Загрузка данных завершена"
cache.clear()
status.save()
@shared_task()
def load_data(obj_id: int):
status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Точек')
file = models.TempFiles.objects.get(id=obj_id)
csv_file = base64.b64decode(file.data)
models.PlacementPoint.objects.all().delete()
s = str(csv_file, 'utf-8')
data = StringIO(s)
df = pd.read_csv(data, delimiter=';')
df = df.replace(np.nan, None)
df = df.replace('NaT', None)
data_len = df.shape[0]
for _ind, row in enumerate(df.to_dict('records')):
status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%"
status.save()
data = {
k: row[k] for k in row.keys() if
k not in ['id', 'location_id', 'area', 'district', 'age_month']
}
models.PlacementPoint.objects.create(**data)
status.status = "Загрузка данных завершена"
status.save()
models.TempFiles.objects.all().delete()

@ -1,42 +1,25 @@
from django.conf.urls import url from django.conf.urls import url
from django.urls import include
from django.urls import path
from django.urls import re_path
from drf_yasg import openapi
from drf_yasg.views import get_schema_view
from rest_framework import permissions
from rest_framework import routers from rest_framework import routers
from postamates.settings import CACHE_TIMEOUT
from service import views from service import views
router = routers.DefaultRouter() router = routers.DefaultRouter()
router.register('', views.PlacementPointViewSet) router.register('placement_points', views.PlacementPointViewSet)
info = openapi.Info( router.register('pre_placement_points', views.PrePlacementPointViewSet)
title='Snippets API', router.register('ao_rayons', views.AOViewSet)
default_version='v1', router.register('postamate_and_pvz_groups', views.PostAndPVZCategoryViewSet)
description='Test description', router.register('other_object_groups', views.OtherObjectsCategoryViewSet)
terms_of_service='https://www.google.com/policies/terms/',
contact=openapi.Contact(email='contact@snippets.local'), urlpatterns = router.urls
license=openapi.License(name='BSD License'),
)
schema_view = get_schema_view(
info,
url='https://postamates.spatiality.website/',
public=True,
permission_classes=[permissions.AllowAny],
)
urlpatterns = [ urlpatterns += [
path('placement_points/', include([*router.urls]), name='placement_points'),
path('ao_rayons', views.AOViewSet.as_view({'get': 'list'}), name='ao_and_rayons'),
url(r'load_csv/', views.refresh_placement_points.as_view(), name='upload_placement_points'), url(r'load_csv/', views.refresh_placement_points.as_view(), name='upload_placement_points'),
url(r'upload_ao_and_rayons/', views.load_ao_and_rayons.as_view(), name='upload_ao_and_rayons'), url(r'upload_post_and_pvz/', views.upload_post_and_pvz, name='upload_post_and_pvz'),
url(r'upload_rivals/', views.upload_rivals, name='upload_rivals'), url(r'upload_other_objects/', views.upload_other_objects, name='upload_other_objects'),
url(r'upload_dist/', views.upload_dist, name='upload_dist'), url(r'upload_houses/', views.upload_houses, name='upload_houses'),
url(r'me/', views.get_current_user, name='me'), url(r'me/', views.get_current_user, name='me'),
re_path(r'^swagger(?P<format>\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'), url('download_pvz_template/', views.download_pvz_template, name='download_pvz_template'),
re_path(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=CACHE_TIMEOUT), name='schema-swagger-ui'), url('download_other_template/', views.download_other_template, name='download_other_template'),
url('avg_bi_values/', views.AvgBiValuesViewSet.as_view(), name='avg_bi_values'),
] ]
USE_X_FORWARDED_HOST = True USE_X_FORWARDED_HOST = True
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

@ -1,24 +1,54 @@
import os
import geojson import geojson
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from django.contrib.gis.geos import GEOSGeometry from django.contrib.gis.geos import GEOSGeometry
from geojson import MultiPolygon from geojson import MultiPolygon
from tqdm import tqdm
from service import models from service import models
import requests
from tqdm import tqdm
from django.core.cache import cache
from django.conf import settings
from rest_framework.response import Response
from rest_framework.viewsets import ReadOnlyModelViewSet
from django.db.models import Avg, Min, Max
def load_data(filepath: str): import psycopg2
models.PlacementPoint.objects.all().delete() from postamates.settings import DB_URL
df = pd.read_csv(filepath)
df = df.replace(np.nan, None)
df = df.replace('NaT', None) def run_sql_command(command):
for row in tqdm(df.to_dict('records'), desc='Loading data...'): connection = psycopg2.connect(
data = { DB_URL
k: row[k] for k in row.keys() if )
k not in ['id', 'location_id', 'area', 'district', 'age_month'] try:
} cursor = connection.cursor()
models.PlacementPoint.objects.create(**data) cursor.execute(command)
connection.commit()
except psycopg2.Error as e:
print("Error executing command:", e)
finally:
cursor.close()
connection.close()
def run_psql_command():
connection = psycopg2.connect(
DB_URL
)
try:
cursor = connection.cursor()
command = "REFRESH MATERIALIZED VIEW public.points_with_dist;REFRESH MATERIALIZED VIEW public.prepoints_with_dist;"
cursor.execute(command)
connection.commit()
except psycopg2.Error as e:
print("Error executing command:", e)
finally:
cursor.close()
connection.close()
def load_ao_and_rayons( def load_ao_and_rayons(
@ -43,20 +73,90 @@ def load_ao_and_rayons(
models.Rayon.objects.create(**{'name': name, 'polygon': GEOSGeometry(str(MultiPolygon(coords))), 'AO': ao}) models.Rayon.objects.create(**{'name': name, 'polygon': GEOSGeometry(str(MultiPolygon(coords))), 'AO': ao})
def load_rivals(filepath: str):
models.Rivals.objects.all().delete()
df = pd.read_csv(filepath)
df = df.replace(np.nan, None)
df = df.replace('NaT', None)
df.columns = df.columns.str.lower()
for row in tqdm(df.to_dict('records'), desc='Loading data...'):
models.Rivals.objects.create(**row)
def load_dist(filepath: str): def load_dist(filepath: str):
models.PointDist.objects.all().delete() models.PointDist.objects.all().delete()
df = pd.read_csv(filepath) df = pd.read_csv(filepath)
for row in tqdm(df.to_dict('records'), desc='Loading data...'): for row in df.to_dict('records'):
row['id1'] = models.PlacementPoint.objects.get(pk=row.get('id1')) row['id1'] = models.PlacementPoint.objects.get(pk=row.get('id1'))
row['id2'] = models.PlacementPoint.objects.get(pk=row.get('id2')) row['id2'] = models.PlacementPoint.objects.get(pk=row.get('id2'))
models.PointDist.objects.create(**row) models.PointDist.objects.create(**row)
def log_to_telegram(msg):
requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage',
json={"chat_id": "-555238820", "text": str(settings.DOMAIN) + '\n' + msg})
def cached_func(key, func, timeout=settings.CACHE_TIMEOUT, *args, **kwargs):
d = cache.get(key)
if d is None:
d = func(*args, **kwargs)
cache.set(key, d, timeout)
return d
def get_middle_bi_values():
fields_to_aggregate = [
'target_dist_shap',
'target_post_cnt_shap',
'target_cnt_ao_mean_shap',
'rival_pvz_cnt_shap',
'rival_post_cnt_shap',
'metro_dist_shap',
'property_price_bargains_shap',
'property_price_offers_shap',
'property_mean_floor_shap',
'property_era_shap',
'flats_cnt_shap',
'popul_home_shap',
'popul_job_shap',
'yndxfood_sum_shap',
'yndxfood_cnt_shap',
'school_cnt_shap',
'kindergar_cnt_shap',
'public_stop_cnt_shap',
'sport_center_cnt_shap',
'pharmacy_cnt_shap',
'supermarket_cnt_shap',
'supermarket_premium_cnt_shap',
'clinic_cnt_shap',
'bank_cnt_shap',
'reca_cnt_shap',
'lab_cnt_shap',
'culture_cnt_shap',
'attraction_cnt_shap',
'mfc_cnt_shap',
'bc_cnt_shap',
'tc_cnt_shap',
'business_activity_shap'
]
aggregations = {}
for field_name in fields_to_aggregate:
aggregations[f'avg_{field_name}'] = Avg(field_name)
aggregations[f'min_{field_name}'] = Min(field_name)
aggregations[f'max_{field_name}'] = Max(field_name)
result = models.PlacementPoint.objects.aggregate(**aggregations)
return result
class CustomReadOnlyModelViewSet(ReadOnlyModelViewSet):
def list(self, request, *args, **kwargs):
def f():
return ReadOnlyModelViewSet.list(self, request, *args, **kwargs).data
d = cached_func(self.__class__.__name__, f)
return Response(d)
def create_columns_dist(row):
return pd.Series(row['min_distance_to_group'])
def load_houses(filepath: str):
models.House.objects.all().delete()
df = pd.read_csv(filepath)
df = df.replace(np.nan, None)
df = df.replace('NaT', None)
for row in df.to_dict('records'):
models.House.objects.create(**row)

@ -14,32 +14,109 @@ from rest_framework.viewsets import ReadOnlyModelViewSet
from postamates.settings import AGE_DAY_BORDER from postamates.settings import AGE_DAY_BORDER
from postamates.settings import EXCEL_EXPORT_FILENAME from postamates.settings import EXCEL_EXPORT_FILENAME
from postamates.settings import JSON_EXPORT_FILENAME from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME
from service import models from service import models
from service import pagination from service import pagination
from service import serializers from service import serializers
from service import utils from service import utils
from service.enums import PointStatus from service.enums import PointStatus, MatchingStatus
from service.permissions import UserPermission from service.permissions import UserPermission
from service.service import PointService from service.service import PointService
from service.tasks import raschet from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data
from service.utils import load_data
from rest_framework.permissions import AllowAny from rest_framework.permissions import AllowAny
from django.shortcuts import redirect
from django.contrib import messages
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters
from service.utils import CustomReadOnlyModelViewSet
from django.db.models import Min, Max
import os
from django.forms.models import model_to_dict
import base64
class AOViewSet(ReadOnlyModelViewSet): class AOViewSet(CustomReadOnlyModelViewSet):
serializer_class = serializers.AOSerializer serializer_class = serializers.AOSerializer
queryset = models.AO.objects queryset = models.AO.objects
permission_classes = [AllowAny] permission_classes = [AllowAny]
class PostAndPVZCategoryViewSet(CustomReadOnlyModelViewSet):
serializer_class = serializers.PostAndPVZCategorySerializer
queryset = models.Post_and_pvzCategory.objects
class OtherObjectsCategoryViewSet(CustomReadOnlyModelViewSet):
serializer_class = serializers.OtherObjectsCategorySerializer
queryset = models.OtherObjectsCategory.objects
class PlacementPointViewSet(ReadOnlyModelViewSet): class PlacementPointViewSet(ReadOnlyModelViewSet):
serializer_class = serializers.PlacementPointSerializer serializer_class = serializers.PlacementPointSerializer
queryset = models.PlacementPoint.objects queryset = models.PlacementPoint.objects
pagination_class = pagination.MyPagination pagination_class = pagination.MyPagination
permission_classes = [UserPermission] permission_classes = [UserPermission]
filter_backends = [DjangoFilterBackend, filters.OrderingFilter, filters.SearchFilter]
@property
def filterset_fields(self):
model_cls = self.queryset.model
fieldset = {
field.name: [
"exact",
"gt",
"gte",
"lt",
"lte",
"in",
"iexact",
"startswith",
"istartswith",
"endswith",
"iendswith",
"regex",
"iregex",
"isnull",
"contains",
"icontains",
]
for field in [
field
for field in model_cls._meta.get_fields()
if field.get_internal_type()
not in [
"JSONField",
"ForeignKey",
"ManyToManyField",
"OneToOneField",
"PointField"
]
]
}
# filters for model relations
for field in [
field
for field in model_cls._meta.get_fields()
if field.get_internal_type()
in [
"ForeignKey",
"ManyToManyField",
"OneToOneField",
]
]:
fieldset[field.name] = [
"exact",
"gt",
"gte",
"lt",
"lte",
]
return fieldset
def get_queryset(self): def get_queryset(self):
basename = self.request.parser_context['view'].basename
qs = self.queryset.all().order_by('id') qs = self.queryset.all().order_by('id')
location_ids = self.request.GET.get('location_ids[]') location_ids = self.request.GET.get('location_ids[]')
prediction_first = self.request.GET.get('prediction_first[]') prediction_first = self.request.GET.get('prediction_first[]')
@ -57,6 +134,8 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
delta_current = self.request.GET.get('delta_current[]') delta_current = self.request.GET.get('delta_current[]')
rayons = self.request.GET.get('area[]') rayons = self.request.GET.get('area[]')
aos = self.request.GET.get('district[]') aos = self.request.GET.get('district[]')
group_dists_lt = self.request.GET.getlist('dist_to_group__lt')
group_dists_gt = self.request.GET.getlist('dist_to_group__gt')
if location_ids: if location_ids:
location_ids = list(location_ids.split(',')) location_ids = list(location_ids.split(','))
qs = qs.filter(pk__in=location_ids) qs = qs.filter(pk__in=location_ids)
@ -104,33 +183,77 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
qs = qs.filter(~Q(pk__in=excluded)) qs = qs.filter(~Q(pk__in=excluded))
if included: if included:
inclded = list(included.split(',')) inclded = list(included.split(','))
qs2 = models.PlacementPoint.objects.filter(pk__in=inclded).all() qs2 = self.queryset.filter(pk__in=inclded).all()
qs = (qs | qs2).distinct() qs = (qs | qs2).distinct()
if group_dists_lt:
g_d = [list(g.split(',')) for g in group_dists_lt]
for group in g_d:
if basename == 'preplacementpoint':
filtered_points = list(
models.PrePlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__lt=int(group[1])).values_list(
'placement_point__id', flat=True))
else:
filtered_points = list(
models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__lt=int(group[1])).values_list(
'placement_point__id', flat=True))
qs = qs.filter(id__in=filtered_points)
if group_dists_gt:
g_d = [list(g.split(',')) for g in group_dists_gt]
for group in g_d:
if basename == 'preplacementpoint':
filtered_points = list(
models.PrePlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__gt=int(group[1])).values_list(
'placement_point__id', flat=True))
else:
filtered_points = list(
models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__gt=int(group[1])).values_list(
'placement_point__id', flat=True))
qs = qs.filter(id__in=filtered_points)
return qs return qs
@action(methods=['get'], detail=False)
def get_filterset_fields(self, request, *args, **kwargs):
return Response(self.filterset_fields, status=HTTPStatus.OK)
@action(detail=False, methods=['get']) @action(detail=False, methods=['get'])
def filters(self, request): def filters(self, request):
qs = self.get_queryset() def get_filter_data():
keys = ( qs = self.get_queryset()
'age_day', 'prediction_first', 'prediction_current', keys = (
'plan_first', 'plan_current', 'fact', 'delta_first', 'age_day', 'prediction_first', 'prediction_current',
'delta_current', 'flat_cnt', 'year_bld', 'levels', 'plan_first', 'plan_current', 'fact', 'delta_first',
'doors', 'flats_cnt', 'popul_home', 'popul_job', 'delta_current', 'flat_cnt', 'year_bld', 'levels',
'other_post_cnt', 'target_post_cnt', 'yndxfood_cnt', 'doors', 'flats_cnt', 'popul_home', 'popul_job',
'yndxfood_sum', 'yndxfood_cnt_cst', 'other_post_cnt', 'target_post_cnt', 'yndxfood_cnt',
) 'yndxfood_sum', 'yndxfood_cnt_cst', 'rival_post_cnt', 'rival_pvz_cnt', 'tc_cnt', 'culture_cnt',
temp_data = { 'mfc_cnt', 'public_stop_cnt'
key: [ , 'supermarket_cnt', 'target_dist', 'metro_dist'
x for x in list(set(qs.values_list(key, flat=True))) if )
x is not None temp_data = {
] key: [
for key in keys x for x in list(set(qs.values_list(key, flat=True))) if
} x is not None
data = { ]
key: [ for key in keys
min(temp_data[key]), max(temp_data[key]), }
] if temp_data[key] else [0, 100] for key in keys data = {
} key: [
min(temp_data[key]), max(temp_data[key]),
] if temp_data[key] else [0, 100] for key in keys
}
data['dist_to_groups'] = [{'group_id': d['pvz_postamates_group'], 'dist': [d['min_dist'], d['max_dist']]}
for d in list(
models.PlacementPointPVZDistance.objects.values('pvz_postamates_group').annotate(
min_dist=Min('dist'), max_dist=Max('dist')))]
return data
data = get_filter_data()
return Response(data, status=HTTPStatus.OK) return Response(data, status=HTTPStatus.OK)
@action(detail=False, methods=['get']) @action(detail=False, methods=['get'])
@ -194,9 +317,10 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
@action(detail=False, methods=['get']) @action(detail=False, methods=['get'])
def to_excel(self, request): def to_excel(self, request):
qs = self.get_queryset() qs = self.get_queryset()
serializer = self.serializer_class(qs, many=True)
filename = EXCEL_EXPORT_FILENAME filename = EXCEL_EXPORT_FILENAME
res = HttpResponse( res = HttpResponse(
PointService.to_excel(qs), PointService.to_excel(serializer),
content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
) )
res['Content-Disposition'] = f'attachment; filename={filename}' res['Content-Disposition'] = f'attachment; filename={filename}'
@ -205,8 +329,9 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
@action(detail=False, methods=['get']) @action(detail=False, methods=['get'])
def to_json(self, request): def to_json(self, request):
qs = self.get_queryset() qs = self.get_queryset()
serializer = self.serializer_class(qs, many=True)
filename = JSON_EXPORT_FILENAME filename = JSON_EXPORT_FILENAME
response = HttpResponse(PointService.to_json(qs), content_type='application/json') response = HttpResponse(PointService.to_json(serializer), content_type='application/json')
response['Content-Disposition'] = f'attachment; filename={filename}' response['Content-Disposition'] = f'attachment; filename={filename}'
return response return response
@ -220,14 +345,99 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
raschet.delay() raschet.delay()
return Response('Sucess', status=HTTPStatus.OK) return Response('Sucess', status=HTTPStatus.OK)
@action(detail=False, methods=['get'])
def last_time_ml_run(self, request):
status = models.TaskStatus.objects.filter(task_name=STATUS_TASK_NAME).first()
st = None
if status:
st = status.status
return Response({'last_time': models.LastMLCall.objects.first().dt, 'task_status': st}, status=HTTPStatus.OK)
class PrePlacementPointViewSet(PlacementPointViewSet):
queryset = models.PrePlacementPoint.objects
serializer_class = serializers.PrePlacementPointSerializer
def get_queryset(self):
qs = super().get_queryset()
matching_status = self.request.GET.get('matching_status')
if matching_status:
qs = qs.filter(matching_status=matching_status)
return qs
@action(detail=False, methods=['post'])
def load_matching_file(self, request):
file = request.FILES['file'].file
file_bytes = file.read()
excel_base64 = base64.b64encode(file_bytes).decode()
obj = models.TempFiles.objects.create(data=excel_base64)
return Response(
{'id': obj.id},
status=HTTPStatus.OK,
)
@action(detail=False, methods=['post'])
def start_matching(self, request):
file_id = request.POST['id']
total, matched, problem = PointService().start_mathing(file_id)
PointService().make_enrichment()
raschet('service_preplacementpoint', need_time=False)
return Response(
{'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}},
status=HTTPStatus.OK,
)
@action(detail=False, methods=['post'])
def move_points(self, request):
qs = self.get_queryset()
qs = qs.filter(matching_status=MatchingStatus.New.name)
for q in qs:
obj = model_to_dict(q)
obj.pop('matching_status')
obj.pop('id')
ao = obj.pop('district')
rayon = obj.pop('area')
obj['district'] = models.AO.objects.get(id=ao)
obj['area'] = models.Rayon.objects.get(id=rayon)
obj['age_day'] = 1
new_obj = models.PlacementPoint.objects.create(**obj)
dists = models.PrePlacementPointPVZDistance.objects.filter(placement_point=q).all()
for d in dists:
models.PlacementPointPVZDistance.objects.get_or_create(placement_point=new_obj,
pvz_postamates_group=d.pvz_postamates_group,
dist=d.dist)
models.PrePlacementPoint.objects.all().delete()
return Response(status=HTTPStatus.OK, )
@action(detail=False, methods=['delete'])
def delete_points(self, request):
ids = request.POST.get('ids')
if ids:
ids = ids.split(',')
PointService.delete_preplacement_points(ids)
else:
models.PrePlacementPoint.objects.all().delete()
return Response(status=HTTPStatus.OK, )
@action(detail=False, methods=['get'])
def download_template(self, request):
image_buffer = open('PrePlacementPoints.xlsx', "rb").read()
response = HttpResponse(image_buffer, content_type='xlsx')
response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx')
return response
class refresh_placement_points(APIView): class refresh_placement_points(APIView):
@staticmethod @staticmethod
def post(request): def post(request):
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
file = request.FILES['file'] file = request.FILES['file'].file
load_data(file) file_bytes = file.read()
return Response(status=HTTPStatus.OK) csv_base64 = base64.b64encode(file_bytes).decode()
obj = models.TempFiles.objects.create(data=csv_base64)
load_data.delay(obj.id)
messages.success(request, 'Файл точек успешно загружен')
return redirect('/admin')
class load_ao_and_rayons(APIView): class load_ao_and_rayons(APIView):
@ -237,15 +447,40 @@ class load_ao_and_rayons(APIView):
file_ao = request.FILES['file_ao'] file_ao = request.FILES['file_ao']
file_rayon = request.FILES['file_rayon'] file_rayon = request.FILES['file_rayon']
utils.load_ao_and_rayons(file_ao, file_rayon) utils.load_ao_and_rayons(file_ao, file_rayon)
return Response(status=HTTPStatus.OK) messages.success(request, 'Файл АО и Районов успешно загружен')
return redirect('/admin')
class AvgBiValuesViewSet(APIView):
@staticmethod
def get(request):
data = utils.get_middle_bi_values()
return Response(data, status=HTTPStatus.OK)
@api_view(['POST'])
def upload_post_and_pvz(request):
warnings.filterwarnings('ignore')
file_rivals = request.FILES['file_post_and_pvz'].file
file_bytes = file_rivals.read()
excel_base64 = base64.b64encode(file_bytes).decode()
obj = models.TempFiles.objects.create(data=excel_base64)
load_post_and_pvz.delay(obj.id)
messages.success(request, 'Загрузка ПВЗ и Постаматов началась. Отслеживайте выполнение в Статусе фоновых задач')
return redirect('/admin')
@api_view(['POST']) @api_view(['POST'])
def upload_rivals(request): def upload_other_objects(request):
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
file_rivals = request.FILES['file_rivals'] file = request.FILES['file_other_objects']
utils.load_rivals(file_rivals) file_bytes = file.read()
return JsonResponse({'message': 'OK'}) excel_base64 = base64.b64encode(file_bytes).decode()
obj = models.TempFiles.objects.create(data=excel_base64)
load_other_objects.delay(obj.id)
messages.success(request, 'Загрузка Прочих объектов началась. Отслеживайте выполнение в Статусе фоновых задач')
return redirect('/admin')
@api_view(['POST']) @api_view(['POST'])
@ -253,7 +488,17 @@ def upload_dist(request):
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
file_dist = request.FILES['file_dist'] file_dist = request.FILES['file_dist']
utils.load_dist(file_dist) utils.load_dist(file_dist)
return JsonResponse({'message': 'OK'}) messages.success(request, 'Файл расстояний успешно загружен')
return redirect('/admin')
@api_view(['POST'])
def upload_houses(request):
warnings.filterwarnings('ignore')
file_dist = request.FILES['file_houses']
utils.load_houses(file_dist)
messages.success(request, 'Файл с домами успешно загружен')
return redirect('/admin')
@api_view(['GET']) @api_view(['GET'])
@ -262,3 +507,17 @@ def get_current_user(request):
return JsonResponse( return JsonResponse(
{'groups': [gr.name for gr in request.user.groups.all()]}, {'groups': [gr.name for gr in request.user.groups.all()]},
) )
def download_pvz_template(self):
image_buffer = open('Постоматы и ПВЗ.xlsx', "rb").read()
response = HttpResponse(image_buffer, content_type='xlsx')
response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('pvz_and_postomats.xlsx')
return response
def download_other_template(self):
image_buffer = open('Другие объекты.xlsx', "rb").read()
response = HttpResponse(image_buffer, content_type='xlsx')
response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('other.xlsx')
return response

@ -0,0 +1,51 @@
{% extends "admin/base_site.html" %}
{% load i18n l10n admin_urls static %}
{% block extrahead %}
{{ block.super }}
{{ media }}
<script src="{% static 'admin/js/cancel.js' %}" async></script>
{% endblock %}
{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
{% block breadcrumbs %}
<div class="breadcrumbs">
<a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
&rsaquo; {% translate 'Delete multiple objects' %}
</div>
{% endblock %}
{% block content %}
{% if perms_lacking %}
<p>{% blocktranslate %}Deleting the selected {{ objects_name }} would result in deleting related objects, but your account doesn't have permission to delete the following types of objects:{% endblocktranslate %}</p>
<ul>{{ perms_lacking|unordered_list }}</ul>
{% elif protected %}
<p>{% blocktranslate %}Deleting the selected {{ objects_name }} would require deleting the following protected related objects:{% endblocktranslate %}</p>
<ul>{{ protected|unordered_list }}</ul>
{% else %}
<p>{% blocktranslate %}Are you sure you want to delete the selected {{ objects_name }}? All of the following objects and their related items will be deleted:{% endblocktranslate %}</p>
{% include "admin/includes/object_delete_summary.html" %}
<h2>{% translate "Objects" %}</h2>
{% for deletable_object in deletable_objects %}
{% if 100 < deletable_object|length %}
<p>{{ deletable_object|length }} objects</p>
{% else %}
<ul>{{ deletable_object|unordered_list }}</ul>
{% endif %}
{% endfor %}
<form method="post">{% csrf_token %}
<div>
{% for obj in queryset %}
<input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}">
{% endfor %}
<input type="hidden" name="action" value="delete_selected">
<input type="hidden" name="post" value="yes">
<input type="submit" value="{% translate 'Yes, Im sure' %}">
<a href="#" class="button cancel-link">{% translate "No, take me back" %}</a>
</div>
</form>
{% endif %}
{% endblock %}

@ -50,39 +50,26 @@
</div> </div>
<div> <div>
{% if is_superadmin %} {% if is_superadmin %}
<h3>Обновить файл точек</h3> <h3>Загрузить файл ПВЗ и Постаматов</h3>
<form method="post" action="/api/load_csv/" enctype="multipart/form-data"> <a href="/api/download_pvz_template">Скачать шаблон</a>
<form method="post" action="/api/upload_post_and_pvz/" enctype="multipart/form-data">
{% csrf_token %} {% csrf_token %}
<input type="file" required=True name="file" accept=".csv"> <input type="file" name="file_post_and_pvz" accept=".xlsx">
<input type="submit" value="Отправить"> <input type="submit" value="Отправить">
</form> </form>
<h3>Загрузить районы и АО</h3> <h3>Загрузить файл прочих объектов</h3>
<form method="post" action="/api/upload_ao_and_rayons/" enctype="multipart/form-data"> <a href="/api/download_other_template">Скачать шаблон</a>
<form method="post" action="/api/upload_other_objects/" enctype="multipart/form-data">
{% csrf_token %} {% csrf_token %}
<h4>Файл с АО</h4> <input type="file" name="file_other_objects" accept=".xlsx">
<input type="file" required=True name="file_ao" accept=".geojson">
<h4>Файл с Районами</h4>
<input type="file" required=True name="file_rayon" accept=".geojson">
<input type="submit" value="Отправить"> <input type="submit" value="Отправить">
</form> </form>
<h3>Обновить файл полигонов</h3> <h3>Загрузить файл с домами</h3>
<form method="post" action="/api/polygons/file_import/" enctype="multipart/form-data"> <form method="post" action="/api/upload_houses/" enctype="multipart/form-data">
{% csrf_token %} {% csrf_token %}
<input type="file" name="file" accept=".zip"> <input type="file" name="file_houses" accept=".csv">
<input type="submit" value="Отправить"> <input type="submit" value="Отправить">
</form> </form>
<h3>Загрузить файл конкурентов</h3>
<form method="post" action="/api/upload_rivals/" enctype="multipart/form-data">
{% csrf_token %}
<input type="file" name="file_rivals" accept=".csv">
<input type="submit" value="Отправить">
</form>
<h3>Загрузить файл расстояний</h3>
<form method="post" action="/api/upload_dist/" enctype="multipart/form-data">
{% csrf_token %}
<input type="file" name="file_dist" accept=".csv">
<input type="submit" value="Отправить">
</form>
{% endif %} {% endif %}
</div> </div>

Binary file not shown.

Binary file not shown.
Loading…
Cancel
Save