Merge branch 'revert_to_dists' into 'dev'

revert_to_dists

See merge request spatial/postamates!167
dev
Aleksandr Popov 2 years ago
commit 6df3b49ffc

@ -19,7 +19,7 @@ spec:
containers:
- name: django
image: DEPLOY_IMAGE_TAG
command: ["sh", "-c", "python manage.py delete_views && python manage.py migrate && python manage.py create_views && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"]
command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"]
ports:
- containerPort: 8000
name: django-port
@ -96,4 +96,4 @@ spec:
port:
number: 8000
path: /accounts/
pathType: ImplementationSpecific
pathType: ImplementationSpecific

@ -9,7 +9,6 @@ https://docs.djangoproject.com/en/3.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.2/ref/settings/
"""
import json
import os
from pathlib import Path
@ -198,6 +197,3 @@ JSON_EXPORT_FILENAME = 'placement_points.json'
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M')
STATUS_TASK_NAME='status_task'
MAPPING_POST_PVZ_GROUP = json.loads(open(os.path.join(BASE_DIR, 'fixtures', 'post_pvz_groups.json')).read())
GROUP_IDS = [group['pk'] for group in MAPPING_POST_PVZ_GROUP]

@ -11,7 +11,9 @@ from service.models import AO
from service.models import PlacementPoint
from service.models import Rayon
from service.models import PrePlacementPoint, Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, \
OtherObjectsGroup, OtherObjectsCategory, TempFiles, RaschetGroups, House, TaskStatus
OtherObjectsGroup, \
OtherObjectsCategory, PrePlacementPointPVZDistance, TempFiles, RaschetGroups, House
from service.models import PlacementPointPVZDistance, TaskStatus
from postamates.settings import DEBUG
from django.core.cache import cache
from service.utils import run_psql_command
@ -30,6 +32,7 @@ if DEBUG:
my_admin_site.register(AO)
my_admin_site.register(Rayon)
my_admin_site.register(TempFiles)
my_admin_site.register(PlacementPointPVZDistance)
class Post_and_PVZAdmin(admin.ModelAdmin):
@ -43,6 +46,7 @@ my_admin_site.register(Post_and_pvz, Post_and_PVZAdmin)
my_admin_site.register(OtherObjects)
my_admin_site.register(House)
my_admin_site.register(PrePlacementPoint)
my_admin_site.register(PrePlacementPointPVZDistance)
class TaskStatusAdmin(admin.ModelAdmin):
@ -154,4 +158,4 @@ try:
except NotRegistered:
pass
my_admin_site.register(User, UserAdmin)
my_admin_site.register(User, UserAdmin)

@ -8,10 +8,10 @@ CREATE OR REPLACE procedure pivot_dist()
AS $BODY$
DECLARE columnNames TEXT;
BEGIN
DROP VIEW IF EXISTS points_with_dist;
DROP MATERIALIZED VIEW IF EXISTS points_with_dist;
SELECT 'placement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_placementpointpvzdistance order by 1) as asd
INTO columnNames;
EXECUTE format('CREATE OR REPLACE VIEW points_with_dist AS SELECT *
EXECUTE format('CREATE MATERIALIZED VIEW points_with_dist AS SELECT *
FROM CROSSTAB(
$$
SELECT placement_point_id, pvz_postamates_group_id, dist
@ -32,10 +32,10 @@ CREATE OR REPLACE procedure prepivot_dist()
AS $BODY$
DECLARE columnNames TEXT;
BEGIN
DROP VIEW IF EXISTS prepoints_with_dist;
DROP MATERIALIZED VIEW IF EXISTS prepoints_with_dist;
SELECT 'preplacement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_preplacementpointpvzdistance order by 1) as asd
INTO columnNames;
EXECUTE format('CREATE OR REPLACE VIEW prepoints_with_dist AS SELECT *
EXECUTE format('CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT *
FROM CROSSTAB(
$$
SELECT placement_point_id, pvz_postamates_group_id, dist
@ -60,4 +60,4 @@ class Command(BaseCommand):
run_sql_command(CMD_PIVOT_DIST_PRE)
log_to_telegram('prepivot_dist created')
except Exception as e:
log_to_telegram('Error creating views: ' + str(e))
log_to_telegram('Error creating views: ' + str(e))

@ -1,13 +1,9 @@
from django.core.management.base import BaseCommand
from service.utils import run_sql_command, log_to_telegram
CMD_PIVOT_DIST = """CREATE OR REPLACE VIEW compact_placementpoint AS
SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry, d10,d12,d17,d20,d24,d25,d7,d8,d9 FROM service_placementpoint;
"""
CMD_PIVOT_DIST = """CALL public.pivot_dist();"""
CMD_PIVOT_DIST_PRE = """CREATE OR REPLACE VIEW compact_preplacementpoint AS
SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry, d10,d12,d17,d20,d24,d25,d7,d8,d9 FROM service_preplacementpoint;
"""
CMD_PIVOT_DIST_PRE = """CALL public.prepivot_dist();"""
class Command(BaseCommand):
help = 'Create views'
@ -19,4 +15,4 @@ class Command(BaseCommand):
run_sql_command(CMD_PIVOT_DIST_PRE)
log_to_telegram('prepivot_dist created')
except Exception as e:
log_to_telegram('Error creating views: ' + str(e))
log_to_telegram('Error creating views: ' + str(e))

@ -1,10 +1,9 @@
from django.core.management.base import BaseCommand
from service.utils import run_sql_command, log_to_telegram
CMD_PIVOT_DIST = """DROP MATERIALIZED VIEW IF EXISTS public.points_with_dist;"""
CMD_PIVOT_DIST = """drop view compact_placementpoint;"""
CMD_PIVOT_DIST_PRE = """drop view compact_preplacementpoint;"""
CMD_PIVOT_DIST_PRE = """DROP MATERIALIZED VIEW IF EXISTS public.prepoints_with_dist;"""
class Command(BaseCommand):
help = 'delete views'

@ -0,0 +1,122 @@
# Generated by Django 3.2 on 2023-10-11 18:20
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('service', '0038_auto_20231008_1141'),
]
operations = [
migrations.RemoveField(
model_name='placementpoint',
name='d10',
),
migrations.RemoveField(
model_name='placementpoint',
name='d12',
),
migrations.RemoveField(
model_name='placementpoint',
name='d17',
),
migrations.RemoveField(
model_name='placementpoint',
name='d19',
),
migrations.RemoveField(
model_name='placementpoint',
name='d20',
),
migrations.RemoveField(
model_name='placementpoint',
name='d24',
),
migrations.RemoveField(
model_name='placementpoint',
name='d25',
),
migrations.RemoveField(
model_name='placementpoint',
name='d7',
),
migrations.RemoveField(
model_name='placementpoint',
name='d8',
),
migrations.RemoveField(
model_name='placementpoint',
name='d9',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d10',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d12',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d17',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d19',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d20',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d24',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d25',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d7',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d8',
),
migrations.RemoveField(
model_name='preplacementpoint',
name='d9',
),
migrations.AddField(
model_name='otherobjectscategory',
name='image',
field=models.ImageField(blank=True, default=None, null=True, upload_to='other_objects_category_images/', verbose_name='Картинка'),
),
migrations.AddField(
model_name='post_and_pvzcategory',
name='image',
field=models.ImageField(blank=True, default=None, null=True, upload_to='post_and_pvz_category_images/', verbose_name='Картинка'),
),
migrations.CreateModel(
name='PrePlacementPointPVZDistance',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('dist', models.FloatField(blank=True, default=None, null=True)),
('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.preplacementpoint')),
('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')),
],
),
migrations.CreateModel(
name='PlacementPointPVZDistance',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('dist', models.FloatField(blank=True, default=None, null=True)),
('placement_point', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.placementpoint')),
('pvz_postamates_group', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='service.post_and_pvzgroup')),
],
),
]

@ -115,19 +115,6 @@ class AbstractPlacementPoint(models.Model):
business_activity_shap = models.IntegerField(null=True, blank=True)
# group_distances
d7 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до ПВЗ Ozon')
d8 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до ПВЗ СДЭК')
d9 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до Постамата Халва')
d10 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до Постамата СДЭК')
d12 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до Постамата Ozon')
d17 = models.FloatField(null=True, verbose_name='Расстояние до ПВЗ Wildberries')
d19 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до ПВЗ Yandex')
d20 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до Постамата Yandex')
d24 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до ПВЗ Почта России')
d25 = models.FloatField(null=True, blank=True, verbose_name='Расстояние до Постамата Почта России')
class PlacementPoint(AbstractPlacementPoint):
class Meta:
verbose_name = 'Точка'
@ -202,6 +189,8 @@ class Post_and_pvzCategory(models.Model):
return self.name
name = models.TextField(null=False, blank=False, verbose_name='Название слоя')
image = models.ImageField(blank=True, null=True, default=None, upload_to='post_and_pvz_category_images/',
verbose_name='Картинка')
visible = models.BooleanField(default=True)
include_in_ml = models.BooleanField(default=True)
@ -231,6 +220,8 @@ class OtherObjectsCategory(models.Model):
return self.name
name = models.TextField(null=False, blank=False, verbose_name='Название слоя')
image = models.ImageField(blank=True, null=True, default=None, upload_to='other_objects_category_images/',
verbose_name='Картинка')
visible = models.BooleanField(default=True)
@ -249,18 +240,16 @@ class OtherObjectsGroup(models.Model):
visible = models.BooleanField(default=True)
# class PlacementPointPVZDistance(models.Model):
# pass
# placement_point = models.ForeignKey('PlacementPoint', default=None, on_delete=models.CASCADE)
# pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
# dist = models.FloatField(blank=True, null=True, default=None)
#
#
# class PrePlacementPointPVZDistance(models.Model):
# pass
# placement_point = models.ForeignKey('PrePlacementPoint', default=None, on_delete=models.CASCADE)
# pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
# dist = models.FloatField(blank=True, null=True, default=None)
class PlacementPointPVZDistance(models.Model):
placement_point = models.ForeignKey('PlacementPoint', default=None, on_delete=models.CASCADE)
pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
dist = models.FloatField(blank=True, null=True, default=None)
class PrePlacementPointPVZDistance(models.Model):
placement_point = models.ForeignKey('PrePlacementPoint', default=None, on_delete=models.CASCADE)
pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
dist = models.FloatField(blank=True, null=True, default=None)
class TaskStatus(models.Model):
@ -304,5 +293,4 @@ class House(models.Model):
doors = models.IntegerField(blank=True, null=True)
enrg_cls = models.TextField(blank=True,null=True)
street = models.TextField(blank=True,null=True)
house_number = models.TextField(blank=True,null=True)
house_number = models.TextField(blank=True,null=True)

@ -9,6 +9,11 @@ class PlacementPointSerializer(serializers.ModelSerializer):
model = models.PlacementPoint
fields = '__all__'
def to_representation(self, instance):
representation = super().to_representation(instance)
min_distances = PointService.get_min_distances_to_group(instance.id)
representation['min_distance_to_group'] = min_distances
return representation
class PrePlacementPointSerializer(PlacementPointSerializer):
class Meta:
@ -16,6 +21,7 @@ class PrePlacementPointSerializer(PlacementPointSerializer):
fields = '__all__'
class PostAndPVZGroupSerializer(serializers.ModelSerializer):
class Meta:
model = models.Post_and_pvzGroup

@ -62,7 +62,7 @@ class PointService:
else:
objs = [obj.first()]
for o in objs:
# distances = models.PlacementPointPVZDistance.objects.filter(placement_point=o.get('id')).all()
distances = models.PlacementPointPVZDistance.objects.filter(placement_point=o.get('id')).all()
o.pop('id')
street = o.pop('street')
house_number = o.pop('house_number')
@ -77,11 +77,10 @@ class PointService:
house_number=house_number,
matching_status=MatchingStatus.Matched.name,
defaults=o)
# по идее можно дропать
# for d in distances:
# models.PrePlacementPointPVZDistance.objects.get_or_create(placement_point=pre_obj,
# pvz_postamates_group=d.pvz_postamates_group,
# dist=d.dist)
for d in distances:
models.PrePlacementPointPVZDistance.objects.get_or_create(placement_point=pre_obj,
pvz_postamates_group=d.pvz_postamates_group,
dist=d.dist)
matched += 1
elif not rayon:
@ -240,25 +239,34 @@ class PointService:
point.target_cnt_ao_mean = placement_point.target_cnt_ao_mean
point.save()
for group in groups:
self.calculate_dist_for_group(point, group)
self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance)
run_psql_command()
@staticmethod
def calculate_dist_for_group(point, group):
def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance):
post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate(
distance=Dist("wkt", point.geometry)).order_by('distance').first()
setattr(point, f'd{group.id}', post_object.distance.m)
point.save()
d = instance_type.objects.filter(placement_point=point,
pvz_postamates_group=group).first()
if post_object:
if d:
if d.dist > post_object.distance.m:
d.dist = post_object.distance.m
d.save()
else:
instance_type.objects.create(placement_point=point,
pvz_postamates_group=group,
dist=post_object.distance.m)
@staticmethod
def delete_preplacement_points(ids: list):
models.PrePlacementPoint.objects.filter(id__in=ids).all().delete()
# @staticmethod
# def get_min_distances_to_group(postamat_id: str):
# return {d['pvz_postamates_group']: d['dist'] for d in list(
# models.PlacementPointPVZDistance.objects.filter(placement_point=postamat_id).values(
# 'pvz_postamates_group', 'dist'))}
@staticmethod
def get_min_distances_to_group(postamat_id: str):
return {d['pvz_postamates_group']: d['dist'] for d in list(
models.PlacementPointPVZDistance.objects.filter(placement_point=postamat_id).values(
'pvz_postamates_group', 'dist'))}
@staticmethod
def update_points_in_radius(qs: models.PlacementPoint, new_status: str):

@ -423,16 +423,15 @@ def load_post_and_pvz(obj_id: int):
status.save()
post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate(
distance=Distance("wkt", point.geometry)).order_by('distance').first()
setattr(point, f'd{group.id}', post_object.distance.m)
# d = models.PlacementPointPVZDistance.objects.filter(placement_point=point,
# pvz_postamates_group=group).first()
# if d:
# if d.dist > post_object.distance.m:
# d.dist = post_object.distance.m
# d.save()
# else:
# models.PlacementPointPVZDistance.objects.create(placement_point=point, pvz_postamates_group=group,
# dist=post_object.distance.m)
d = models.PlacementPointPVZDistance.objects.filter(placement_point=point,
pvz_postamates_group=group).first()
if d:
if d.dist > post_object.distance.m:
d.dist = post_object.distance.m
d.save()
else:
models.PlacementPointPVZDistance.objects.create(placement_point=point, pvz_postamates_group=group,
dist=post_object.distance.m)
status.status = "Подсчет расстояний завершен"
status.save()
point_qs = models.PlacementPoint.objects.all()

@ -41,7 +41,7 @@ def run_psql_command():
)
try:
cursor = connection.cursor()
command = "CALL public.pivot_dist();CALL public.prepivot_dist();"
command = "REFRESH MATERIALIZED VIEW public.points_with_dist;REFRESH MATERIALIZED VIEW public.prepoints_with_dist;"
cursor.execute(command)
connection.commit()
except psycopg2.Error as e:

@ -1,6 +1,5 @@
import warnings
from http import HTTPStatus
from django.conf import settings
from django.db.models import Q
from django.http import HttpResponse
@ -30,10 +29,10 @@ from django.contrib import messages
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters
from service.utils import CustomReadOnlyModelViewSet
from django.db.models import Min, Max
import os
from django.forms.models import model_to_dict
import base64
from django.db.models import Avg, Max, Min
class AOViewSet(CustomReadOnlyModelViewSet):
@ -186,17 +185,34 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
inclded = list(included.split(','))
qs2 = self.queryset.filter(pk__in=inclded).all()
qs = (qs | qs2).distinct()
# TODO: fix to new fields
if group_dists_lt:
g_d = [list(g.split(',')) for g in group_dists_lt]
for group in g_d:
kwargs = {f'd{group[0]}__lt': group[1]}
qs = qs.filter(**kwargs)
if basename == 'preplacementpoint':
filtered_points = list(
models.PrePlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__lt=int(group[1])).values_list(
'placement_point__id', flat=True))
else:
filtered_points = list(
models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__lt=int(group[1])).values_list(
'placement_point__id', flat=True))
qs = qs.filter(id__in=filtered_points)
if group_dists_gt:
g_d = [list(g.split(',')) for g in group_dists_gt]
for group in g_d:
kwargs = {f'd{group[0]}__gt': group[1]}
qs = qs.filter(**kwargs)
if basename == 'preplacementpoint':
filtered_points = list(
models.PrePlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__gt=int(group[1])).values_list(
'placement_point__id', flat=True))
else:
filtered_points = list(
models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
dist__gt=int(group[1])).values_list(
'placement_point__id', flat=True))
qs = qs.filter(id__in=filtered_points)
return qs
@action(methods=['get'], detail=False)
@ -229,19 +245,10 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
min(temp_data[key]), max(temp_data[key]),
] if temp_data[key] else [0, 100] for key in keys
}
fields_to_aggregate = [f'd{x}' for x in settings.GROUP_IDS]
aggregations = {}
for field_name in fields_to_aggregate:
aggregations[f'min_{field_name}'] = Min(field_name)
aggregations[f'max_{field_name}'] = Max(field_name)
result = models.PlacementPoint.objects.aggregate(**aggregations)
data['dist_to_groups'] = [
{'group_id': d, 'dist': [result[f'min_d{d}'], result[f'max_d{d}']]} for d in settings.GROUP_IDS
]
# data['dist_to_groups'] = [{'group_id': d['pvz_postamates_group'], 'dist': [d['min_dist'], d['max_dist']]}
# for d in list(
# models.PlacementPointPVZDistance.objects.values('pvz_postamates_group').annotate(
# min_dist=Min('dist'), max_dist=Max('dist')))]
data['dist_to_groups'] = [{'group_id': d['pvz_postamates_group'], 'dist': [d['min_dist'], d['max_dist']]}
for d in list(
models.PlacementPointPVZDistance.objects.values('pvz_postamates_group').annotate(
min_dist=Min('dist'), max_dist=Max('dist')))]
return data
@ -393,13 +400,12 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
obj['district'] = models.AO.objects.get(id=ao)
obj['area'] = models.Rayon.objects.get(id=rayon)
obj['age_day'] = 1
models.PlacementPoint.objects.create(**obj)
# new_obj = models.PlacementPoint.objects.create(**obj)
# dists = models.PrePlacementPointPVZDistance.objects.filter(placement_point=q).all()
# for d in dists:
# models.PlacementPointPVZDistance.objects.get_or_create(placement_point=new_obj,
# pvz_postamates_group=d.pvz_postamates_group,
# dist=d.dist)
new_obj = models.PlacementPoint.objects.create(**obj)
dists = models.PrePlacementPointPVZDistance.objects.filter(placement_point=q).all()
for d in dists:
models.PlacementPointPVZDistance.objects.get_or_create(placement_point=new_obj,
pvz_postamates_group=d.pvz_postamates_group,
dist=d.dist)
models.PrePlacementPoint.objects.all().delete()
return Response(status=HTTPStatus.OK, )
@ -452,6 +458,7 @@ class AvgBiValuesViewSet(APIView):
return Response(data, status=HTTPStatus.OK)
@api_view(['POST'])
def upload_post_and_pvz(request):
warnings.filterwarnings('ignore')

Loading…
Cancel
Save