Merge branch 'move_ml_and_calc' into 'dev'

move_calculations

See merge request spatial/postamates!112

postamates/settings.py

@@ -196,3 +196,4 @@ EXCEL_EXPORT_FILENAME = 'placement_points.xlsx'
 JSON_EXPORT_FILENAME = 'placement_points.json'
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None
 GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M')
+STATUS_TASK_NAME='status_task'

service/admin.py

@@ -12,12 +12,11 @@ from service.models import PlacementPoint
 from service.models import Rayon
 from service.models import PrePlacementPoint, Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, \
     OtherObjectsGroup, \
-    OtherObjectsCategory, PrePlacementPointPVZDistance, TempFiles
+    OtherObjectsCategory, PrePlacementPointPVZDistance, TempFiles, RaschetGroups
 from service.models import PlacementPointPVZDistance, TaskStatus
 from postamates.settings import DEBUG
 from django.core.cache import cache
 from service.utils import run_psql_command
-from service.tasks import start_pvz_group_count, start_pvz_category_count, raschet
 
 
 class MyAdminSite(AdminSite):
@@ -40,8 +39,7 @@ class Post_and_PVZAdmin(admin.ModelAdmin):
     def save_model(self, request, obj, form, change):
         obj.save()
         LayerService().count_post_pvz_for_placementpoint(obj)
-        if 'include_in_ml' in form.changed_data:
-            raschet.delay()
+        RaschetGroups.objects.create(obj_id=obj.group.id)
 
 
 my_admin_site.register(Post_and_pvz, Post_and_PVZAdmin)
@@ -49,6 +47,7 @@ my_admin_site.register(OtherObjects)
 my_admin_site.register(PrePlacementPoint)
 my_admin_site.register(PrePlacementPointPVZDistance)
 
+
 class TaskStatusAdmin(admin.ModelAdmin):
     list_display = ('task_name', 'status')
 
@@ -68,9 +67,8 @@ class CategoryAdmin(admin.ModelAdmin):
 class PostPvzCategoryAdmin(CategoryAdmin):
     def save_model(self, request, obj, form, change):
         obj.save()
-        if 'include_in_ml' in form.changed_data:
-            start_pvz_category_count.delay(obj.id)
-            raschet.delay()
+        if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
+            LayerService.update_categories(obj)
         cache.clear()
 
     def delete_model(self, request, obj):
@@ -97,9 +95,8 @@ class PostPvzGroupAdmin(GroupAdmin):
         if not obj.pk:
             run_psql_command()
         obj.save()
-        if 'include_in_ml' in form.changed_data:
-            start_pvz_group_count.delay(obj.id)
-            raschet.delay()
+        if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
+            LayerService.update_groups(obj)
         cache.clear()
 
 

service/layer_service.py

@@ -4,13 +4,31 @@ from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS
 
 
 class LayerService:
-
     def count_post_pvz_for_placementpoint(self, obj):
         points = models.PlacementPoint.objects.filter(geometry__distance_lt=(obj.wkt, Distance(
             m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).all()
         for point in points:
             LayerService.count_post_pvz(point)
 
+    @staticmethod
+    def update_categories(instance):
+        groups = models.Post_and_pvzGroup.objects.filter(category=instance)
+        groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
+        for gr in groups:
+            models.RaschetGroups.objects.get_or_create(obj_id=gr.id)
+            objects = models.Post_and_pvz.objects.filter(group=gr)
+            objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
+            for obj in objects.all():
+                models.RaschetObjects.objects.get_or_create(obj_id=obj.id)
+
+    @staticmethod
+    def update_groups(instance):
+        models.RaschetGroups.objects.get_or_create(obj_id=instance.id)
+        objects = models.Post_and_pvz.objects.filter(group=instance)
+        objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
+        for obj in objects.all():
+            models.RaschetObjects.objects.get_or_create(obj_id=obj.id)
+
     @staticmethod
     def get_post_and_pvz_categroies():
         return models.Post_and_pvzCategory.objects.all(), models.Post_and_pvzGroup.objects.all()

service/migrations/0033_raschetobjects.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2 on 2023-09-16 09:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('service', '0032_auto_20230912_1616'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RaschetObjects',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('obj_id', models.IntegerField()),
+            ],
+        ),
+    ]

service/migrations/0034_raschetgroups.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2 on 2023-09-16 11:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('service', '0033_raschetobjects'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='RaschetGroups',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('obj_id', models.IntegerField()),
+            ],
+        ),
+    ]

service/models.py

@@ -93,6 +93,7 @@ class PlacementPoint(AbstractPlacementPoint):
 class PrePlacementPoint(AbstractPlacementPoint):
     class Meta:
         ordering = ('id',)
+
     MATCHING_CHOICES = [(tag.name, tag.value) for tag in MatchingStatus]
     matching_status = models.TextField(null=True, blank=True, choices=MATCHING_CHOICES)
 
@@ -212,11 +213,13 @@ class PlacementPointPVZDistance(models.Model):
     pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
     dist = models.FloatField(blank=True, null=True, default=None)
 
+
 class PrePlacementPointPVZDistance(models.Model):
     placement_point = models.ForeignKey('PrePlacementPoint', default=None, on_delete=models.CASCADE)
     pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE)
     dist = models.FloatField(blank=True, null=True, default=None)
 
+
 class TaskStatus(models.Model):
     class Meta:
         verbose_name = 'Статус фоновых задач'
@@ -235,3 +238,11 @@ class LastMLCall(models.Model):
 
 class TempFiles(models.Model):
     data = models.TextField(blank=False, null=False)
+
+
+class RaschetGroups(models.Model):
+    obj_id = models.IntegerField(null=False, blank=False)
+
+
+class RaschetObjects(models.Model):
+    obj_id = models.IntegerField(null=False, blank=False)

service/service.py

@@ -8,7 +8,6 @@ from django.db.models import F
 from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT
 from service import models
 from service.enums import PointStatus
-from service.tasks import raschet
 from service.utils import create_columns_dist, run_psql_command
 import base64
 import requests
@@ -51,7 +50,8 @@ class PointService:
                 continue
             wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")"
             response = response[0]['address']
-            obj = models.PlacementPoint.objects.filter(street=response.get('street'), house_number=response.get('houseNumber'),
+            obj = models.PlacementPoint.objects.filter(street=response.get('street'),
+                                                       house_number=response.get('houseNumber'),
                                                        subject_rf=response.get('state'), city=response.get('city'),
                                                        is_vis=True,
                                                        category=cat).values().first()
@@ -63,12 +63,12 @@ class PointService:
             else:
                 models.PrePlacementPoint.objects.get_or_create(address=addr, street=response.get('street'),
                                                                house_number=response.get('houseNumber'),
-                                                               category=cat, geometry=wkt,sample_trn=False,
-                                                               matching_status=MatchingStatus.New.name, status = PointStatus.Pending)
+                                                               category=cat, geometry=wkt, sample_trn=False,
+                                                               matching_status=MatchingStatus.New.name,
+                                                               status=PointStatus.Pending)
         return total, matched, problem
 
-    @staticmethod
-    def make_enrichment():
+    def make_enrichment(self):
         points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all()
         groups = models.Post_and_pvzGroup.objects.all()
         for point in points:
@@ -174,24 +174,28 @@ class PointService:
                 dist=Dist('geometry', origin)).order_by('dist')[0]
             point.target_cnt_ao_mean = placement_point.target_cnt_ao_mean
             point.area = placement_point.area
-            point.district=placement_point.district
+            point.district = placement_point.district
             point.save()
             for group in groups:
-                post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate(
-                    distance=Dist("wkt", point.geometry)).order_by('distance').first()
-                d = models.PrePlacementPointPVZDistance.objects.filter(placement_point=point,
-                                                                       pvz_postamates_group=group).first()
-                if post_object:
-                    if d:
-                        if d.dist > post_object.distance.m:
-                            d.dist = post_object.distance.m
-                            d.save()
-                    else:
-                        models.PrePlacementPointPVZDistance.objects.create(placement_point=point,
-                                                                           pvz_postamates_group=group,
-                                                                           dist=post_object.distance.m)
+                self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance)
         run_psql_command()
 
+    @staticmethod
+    def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance):
+        post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate(
+            distance=Dist("wkt", point.geometry)).order_by('distance').first()
+        d = instance_type.objects.filter(placement_point=point,
+                                         pvz_postamates_group=group).first()
+        if post_object:
+            if d:
+                if d.dist > post_object.distance.m:
+                    d.dist = post_object.distance.m
+                    d.save()
+            else:
+                instance_type.objects.create(placement_point=point,
+                                             pvz_postamates_group=group,
+                                             dist=post_object.distance.m)
+
     @staticmethod
     def delete_preplacement_points(ids: list):
         models.PrePlacementPoint.objects.filter(id__in=ids).all().delete()
@@ -204,7 +208,6 @@ class PointService:
 
     @staticmethod
     def update_points_in_radius(qs: models.PlacementPoint, new_status: str):
-        triggers = False
         for point in qs:
             if new_status == PointStatus.Installation.name:
                 if point.status == PointStatus.Pending.name:
@@ -212,18 +215,12 @@ class PointService:
                         geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)),
                     )
                     pnts.update(prediction_first=F('prediction_current'), target_post_cnt=F('target_post_cnt') + 1)
-                    triggers = True
             elif new_status == PointStatus.Cancelled.name or new_status == PointStatus.Pending.name:
                 if point.status == PointStatus.Installation.name:
                     pnts = models.PlacementPoint.objects.filter(
                         geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)),
                     )
                     pnts.update(target_post_cnt=F('target_post_cnt') - 1 if F('target_post_cnt') != 0 else 0)
-                    triggers = True
-            elif new_status == PointStatus.Working.name and point.status == PointStatus.Pending.name:
-                triggers = True
-        if triggers:
-            raschet.delay()
 
     @staticmethod
     def update_status(qs: models.PlacementPoint, new_status: str) -> models.PlacementPoint:

service/tasks.py

@@ -16,7 +16,7 @@ from sklearn import model_selection as ms
 from sqlalchemy import text
 from django.contrib.gis.db.models.functions import Distance
 from postamates.settings import AGE_DAY_LIMIT
-from postamates.settings import DB_URL
+from postamates.settings import DB_URL, STATUS_TASK_NAME
 from service.models import PlacementPoint, LastMLCall
 from service import models
 from service.utils import log_to_telegram
@@ -24,12 +24,43 @@ import base64
 from io import StringIO
 from django.core.cache import cache
 from service.layer_service import LayerService
+from service.service import PointService
 
 
 @shared_task()
 def raschet(table_name='service_placementpoint'):
-    LastMLCall.objects.all().delete()
-    LastMLCall.objects.create()
+    status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME)
+    raschet_objs = models.RaschetObjects.objects.all()
+    if raschet_objs:
+        status.status = 'Начало расчета кол-ва ПВЗ вокруг точек'
+        status.save()
+        total = raschet_objs.count()
+        for _i, r_o in enumerate(raschet_objs):
+            obj = models.Post_and_pvz.objects.get(id=r_o.obj_id)
+            LayerService().count_post_pvz_for_placementpoint(obj)
+            status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%"
+            status.save()
+        status.status = 'Расчет кол-ва ПВЗ вокруг точек завершен'
+        status.save()
+    group_objects = models.RaschetGroups.objects.all()
+    group_total = group_objects.count()
+    if group_objects:
+        status.status = 'Начало расчета расстояний'
+        status.save()
+        qs = models.PlacementPoint.objects.all()
+        for _k, g_o in enumerate(group_objects):
+            g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id)
+            for q in qs:
+                PointService.calculate_dist_for_group(point=q, group=g)
+            status.status = "Подсчет расстояний: " + str(int(_k / group_total * 100)) + "%"
+            status.save()
+    status.status = "Подсчет расстояний завершен"
+    status.save()
+    models.RaschetObjects.objects.all().delete()
+    models.RaschetGroups.objects.all().delete()
+    # Запуск ML
+    status.status = 'Запуск ML'
+    status.save()
     log_to_telegram('start raschet')
     try:
         log_to_telegram('try connect to db')
@@ -47,11 +78,10 @@ def raschet(table_name='service_placementpoint'):
         pts = gpd.GeoDataFrame(pts, geometry='geometry', crs='epsg:4326')
         pts = pts.to_crs('epsg:32637')
 
-
         feats = [
             'id', 'metro_dist', 'target_dist', 'property_price_bargains', 'property_price_offers',
             'property_mean_floor',
-            'property_era', 'flats_cnt', 'popul_home', 'popul_job',  'yndxfood_sum',
+            'property_era', 'flats_cnt', 'popul_home', 'popul_job', 'yndxfood_sum',
             'yndxfood_cnt', 'school_cnt', 'kindergar_cnt', 'target_post_cnt', 'public_stop_cnt', 'sport_center_cnt',
             'pharmacy_cnt', 'supermarket_cnt', 'supermarket_premium_cnt', 'clinic_cnt', 'bank_cnt', 'reca_cnt',
             'lab_cnt', 'culture_cnt', 'attraction_cnt', 'mfc_cnt', 'bc_cnt', 'tc_cnt', 'rival_pvz_cnt',
@@ -77,7 +107,6 @@ def raschet(table_name='service_placementpoint'):
         )
         pts_trn.loc[pts_trn.target_dist > 700, 'target_dist'] = 700
 
-
         pts_trn['buf'] = pts_trn.buffer(500)
         pts_trn = gpd.GeoDataFrame(pts_trn, geometry='buf', crs='epsg:32637')
         target_post = gpd.sjoin(pts_trn, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'})
@@ -197,6 +226,8 @@ def raschet(table_name='service_placementpoint'):
     except Exception as e:
         log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}')
     log_to_telegram('Начинается обновление полей в базе')
+    status.status = 'Перерасчет ML: 50%'
+    status.save()
     # Загрузка в базу обновленных значений
     try:
         log_to_telegram('Подключение к базе данных 2')
@@ -279,7 +310,8 @@ def raschet(table_name='service_placementpoint'):
         conn2.close()
         cache.clear()
         log_to_telegram('end raschet')
-
+        status.status = 'Перерасчет ML завершен'
+        status.save()
 
 
 @shared_task
@@ -340,6 +372,8 @@ def load_post_and_pvz(obj_id: int):
     status.status = "Завершено"
     cache.clear()
     status.save()
+    LastMLCall.objects.all().delete()
+    LastMLCall.objects.create()
 
 
 @shared_task()
@@ -408,39 +442,38 @@ def load_data(obj_id: int):
     status.save()
     models.TempFiles.objects.all().delete()
 
-
-@shared_task()
-def start_pvz_group_count(instance_id: int):
-    instance = models.Post_and_pvzGroup.objects.filter(id=instance_id).first()
-    objects = models.Post_and_pvz.objects.filter(group=instance)
-    objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
-    status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для групп ПВЗ и Постаматов')
-    total = objects.count()
-    for _i, obj in enumerate(objects):
-        status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%"
-        status.save()
-        LayerService().count_post_pvz_for_placementpoint(obj)
-    status.status = "Подсчет завершен"
-    status.save()
-
-
-@shared_task()
-def start_pvz_category_count(instance_id: int):
-    status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для категорий ПВЗ и Постаматов')
-    instance = models.Post_and_pvzCategory.objects.filter(id=instance_id).first()
-    groups = models.Post_and_pvzGroup.objects.filter(category=instance)
-    groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
-    total = 0
-    for gr in groups:
-        total += models.Post_and_pvz.objects.filter(group=gr).count()
-    for gr in groups:
-        _i = 0
-        objects = models.Post_and_pvz.objects.filter(group=gr)
-        objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
-        for obj in objects:
-            _i += 1
-            status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int(_i / total * 100)) + "%"
-            status.save()
-            LayerService().count_post_pvz_for_placementpoint(obj)
-    status.status = "Подсчет завершен"
-    status.save()
+# @shared_task()
+# def start_pvz_group_count(instance_id: int):
+#     instance = models.Post_and_pvzGroup.objects.filter(id=instance_id).first()
+#     objects = models.Post_and_pvz.objects.filter(group=instance)
+#     objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
+#     status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для групп ПВЗ и Постаматов')
+#     total = objects.count()
+#     for _i, obj in enumerate(objects):
+#         status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%"
+#         status.save()
+#         LayerService().count_post_pvz_for_placementpoint(obj)
+#     status.status = "Подсчет завершен"
+#     status.save()
+
+
+# @shared_task()
+# def start_pvz_category_count(instance_id: int):
+#     status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для категорий ПВЗ и Постаматов')
+#     instance = models.Post_and_pvzCategory.objects.filter(id=instance_id).first()
+#     groups = models.Post_and_pvzGroup.objects.filter(category=instance)
+#     groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
+#     total = 0
+#     for gr in groups:
+#         total += models.Post_and_pvz.objects.filter(group=gr).count()
+#     for gr in groups:
+#         _i = 0
+#         objects = models.Post_and_pvz.objects.filter(group=gr)
+#         objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
+#         for obj in objects:
+#             _i += 1
+#             status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int(_i / total * 100)) + "%"
+#             status.save()
+#             LayerService().count_post_pvz_for_placementpoint(obj)
+#     status.status = "Подсчет завершен"
+#     status.save()

service/views.py

@@ -14,7 +14,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet
 
 from postamates.settings import AGE_DAY_BORDER
 from postamates.settings import EXCEL_EXPORT_FILENAME
-from postamates.settings import JSON_EXPORT_FILENAME
+from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME
 from service import models
 from service import pagination
 from service import serializers
@@ -333,7 +333,11 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
 
     @action(detail=False, methods=['get'])
     def last_time_ml_run(self, request):
-        return Response({'last_time': models.LastMLCall.objects.first().dt}, status=HTTPStatus.OK)
+        status = models.TaskStatus.objects.filter(task_name=STATUS_TASK_NAME).first()
+        st = None
+        if status:
+            st = status.status
+        return Response({'last_time': models.LastMLCall.objects.first().dt, 'task_status': st}, status=HTTPStatus.OK)
 
 
 class PrePlacementPointViewSet(PlacementPointViewSet):
@@ -362,8 +366,8 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
     def start_matching(self, request):
         file_id = request.POST['id']
         total, matched, problem = PointService().start_mathing(file_id)
-        PointService.make_enrichment()
-        raschet('service_preplacementpoint')
+        PointService().make_enrichment()
+        # raschet('service_preplacementpoint')
         return Response(
             {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}},
             status=HTTPStatus.OK,
@@ -388,7 +392,7 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
         return Response(status=HTTPStatus.OK, )
 
     @action(detail=False, methods=['get'])
-    def download_template(self,request):
+    def download_template(self, request):
         image_buffer = open('PrePlacementPoints.xlsx', "rb").read()
         response = HttpResponse(image_buffer, content_type='xlsx')
         response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx')