Merge branch 'mathing_process' into 'dev'

load_preplacementpoints

See merge request spatial/postamates!98

docker-compose.yml

@@ -31,6 +31,8 @@ services:
              python manage.py collectstatic --noinput &&
              python manage.py loaddata fixtures/groups.json &&
              python3 manage.py loaddata fixtures/post_and_pvz.json &&
+             python3 manage.py loaddata fixtures/otherobjectscategorys.json &&
+             python3 manage.py loaddata fixtures/otherobjectsgroups.json &&
              python manage.py runserver 0.0.0.0:${DJANGO_PORT}"
     environment:
       <<: *postgres-variables

fixtures/otherobjectscategorys.json

@@ -0,0 +1,23 @@
+[{"model": "service.otherobjectscategory", "pk": 4, "fields": {"name": "business_activity", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 5, "fields": {"name": "metro_stations", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 22, "fields": {"name": "bargains", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 23, "fields": {"name": "BC", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 24, "fields": {"name": "flats_cnt", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 25, "fields": {"name": "offers_estate", "image": "", "visible": false}}, {
+  "model": "service.otherobjectscategory", "pk": 26, "fields": {"name": "schools", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 27, "fields": {"name": "kindergar", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 28, "fields": {"name": "stops", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 29, "fields": {"name": "pharmacies", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 30, "fields": {"name": "sport_centers", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 31, "fields": {"name": "supermarkets", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 32, "fields": {"name": "supermarkets_premium", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 33, "fields": {"name": "banks", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 34, "fields": {"name": "recas", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 35, "fields": {"name": "labs", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 36, "fields": {"name": "clinics", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 37, "fields": {"name": "attractions", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 38, "fields": {"name": "cultures", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 39, "fields": {"name": "public_services", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 40, "fields": {"name": "popul_home_job", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 41, "fields": {"name": "TC", "image": "", "visible": false}},
+  {"model": "service.otherobjectscategory", "pk": 42, "fields": {"name": "yndx_food_cnt_amt", "image": "", "visible": false}}]

fixtures/otherobjectsgroups.json

@@ -0,0 +1,23 @@
+[{"model": "service.otherobjectsgroup", "pk": 4, "fields": {"name": "business_activity", "category": 4, "image": "other_objects_group_images/imgonline-com-ua-Blur-v2Mhg2f07JEmbet_MINlljC.png", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 5, "fields": {"name": "metro_stations", "category": 5, "image": "other_objects_group_images/imgonline-com-ua-Blur-v2Mhg2f07JEmbet_5QjtY2T.png", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 23, "fields": {"name": "bargains", "category": 22, "image": "other_objects_group_images/imgonline-com-ua-Blur-v2Mhg2f07JEmbet_3GZcX5B.png", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 24, "fields": {"name": "BC", "category": 23, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 25, "fields": {"name": "flats_cnt", "category": 24, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 26, "fields": {"name": "offers_estate", "category": 25, "image": "other_objects_group_images/imgonline-com-ua-Blur-v2Mhg2f07JEmbet_EzmxSGd.png", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 27, "fields": {"name": "schools", "category": 26, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 28, "fields": {"name": "kindergar", "category": 27, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 29, "fields": {"name": "stops", "category": 28, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 30, "fields": {"name": "pharmacies", "category": 29, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 31, "fields": {"name": "sport_centers", "category": 30, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 32, "fields": {"name": "supermarkets", "category": 31, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 33, "fields": {"name": "supermarkets_premium", "category": 32, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 34, "fields": {"name": "banks", "category": 33, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 35, "fields": {"name": "recas", "category": 34, "image": "", "visible": false}}, {
+  "model": "service.otherobjectsgroup", "pk": 36, "fields": {"name": "labs", "category": 35, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 37, "fields": {"name": "clinics", "category": 36, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 38, "fields": {"name": "attractions", "category": 37, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 39, "fields": {"name": "cultures", "category": 38, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 40, "fields": {"name": "public_services", "category": 39, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 41, "fields": {"name": "popul_home_job", "category": 40, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 42, "fields": {"name": "TC", "category": 41, "image": "", "visible": false}},
+  {"model": "service.otherobjectsgroup", "pk": 43, "fields": {"name": "yndx_food_cnt_amt", "category": 42, "image": "", "visible": false}}]

service/admin.py

@@ -45,6 +45,7 @@ class Post_and_PVZAdmin(admin.ModelAdmin):
 
 my_admin_site.register(Post_and_pvz, Post_and_PVZAdmin)
 my_admin_site.register(OtherObjects)
+my_admin_site.register(PrePlacementPoint)
 
 
 class TaskStatusAdmin(admin.ModelAdmin):
@@ -101,15 +102,44 @@ class PostPvzGroupAdmin(GroupAdmin):
         cache.clear()
 
 
+class OtherObjectsGroupAdmin(GroupAdmin):
+    def save_model(self, request, obj, form, change):
+        if obj.id in (4, 5) or obj.id in list(range(22, 43)):
+            pass
+        else:
+            super().save_model(request, obj, form, change)
+
+    def delete_model(self, request, obj):
+        if obj.id in (4, 5) or obj.id in list(range(22, 43)):
+            pass
+        else:
+            super().delete_model(request, obj)
+
+
+class OtherObjectsCategoryAdmin(CategoryAdmin):
+
+    def save_model(self, request, obj, form, change):
+        if obj.id in (4, 5) or obj.id in list(range(22, 43)):
+            pass
+        else:
+            super().save_model(request, obj, form, change)
+
+    def delete_model(self, request, obj):
+        if obj.id in (4, 5) or obj.id in list(range(22, 43)):
+            pass
+        else:
+            super().delete_model(request, obj)
+
+
 class PlacementPointAdmin(admin.ModelAdmin):
     pass
 
 
 my_admin_site.register(TaskStatus, TaskStatusAdmin)
 my_admin_site.register(Post_and_pvzGroup, PostPvzGroupAdmin)
-my_admin_site.register(OtherObjectsGroup, GroupAdmin)
+my_admin_site.register(OtherObjectsGroup, OtherObjectsGroupAdmin)
 my_admin_site.register(Post_and_pvzCategory, PostPvzCategoryAdmin)
-my_admin_site.register(OtherObjectsCategory, CategoryAdmin)
+my_admin_site.register(OtherObjectsCategory, OtherObjectsCategoryAdmin)
 my_admin_site.register(PlacementPoint, PlacementPointAdmin)
 
 

service/service.py

@@ -5,7 +5,7 @@ import pandas as pd
 from django.contrib.gis.measure import Distance
 from django.db.models import F
 
-from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS
+from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT
 from service import models
 from service.enums import PointStatus
 from service.tasks import raschet
@@ -14,6 +14,8 @@ import base64
 import requests
 from postamates.settings import GEOCODER_API_KEY
 from service.enums import MatchingStatus
+from django.contrib.gis.db.models.functions import Distance as Dist
+from django.db.models import Avg, Sum, Count
 
 
 class PointService:
@@ -26,8 +28,7 @@ class PointService:
         qs = self.get_point_by_id(point_id)
         qs.update(**{'postamat_id': postamat_id})
 
-    @staticmethod
-    def start_mathing(obj_id: int):
+    def start_mathing(self, obj_id: int):
         file = models.TempFiles.objects.get(id=obj_id)
         excel_file = base64.b64decode(file.data)
         df = pd.read_excel(excel_file)
@@ -43,6 +44,8 @@ class PointService:
                 models.PrePlacementPoint.objects.create(address=addr, matching_status=MatchingStatus.Error.name)
                 problem += 1
                 continue
+            coords = response[0]['position']
+            wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")"
             response = response[0]['address']
             obj = models.PlacementPoint.objects.filter(street=response['street'], house_number=response['houseNumber'],
                                                        category=cat).values().first()
@@ -53,10 +56,117 @@ class PointService:
             else:
                 models.PrePlacementPoint.objects.create(address=addr, street=response['street'],
                                                         house_number=response['houseNumber'],
-                                                        category=cat, matching_status=MatchingStatus.New.name)
+                                                        category=cat, geometry=wkt,
+                                                        matching_status=MatchingStatus.New.name)
         models.TempFiles.objects.all().delete()
         return total, matched, problem
 
+    @staticmethod
+    def make_enrichment():
+        points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all()
+        for point in points:
+            origin = point.geometry
+            qs = models.PlacementPoint.objects.filter(status=PointStatus.Working.name).annotate(
+                dist=Dist('geometry', origin)).order_by('dist')
+            point.target_dist = qs[0].dist.m
+            point.target_post_cnt = qs.filter(
+                dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)
+            ).count()
+            point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean
+            point.rival_post_cnt = models.Post_and_pvz.objects.filter(
+                category__name="Постамат", include_in_ml=True,
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.rival_pvz_cnt = models.Post_and_pvz.objects.filter(
+                category__name="ПВЗ", include_in_ml=True,
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate(
+                dist=Dist('wkt', origin)).order_by('dist')[0].dist.m
+            point.property_price_bargains = models.OtherObjects.objects.filter(
+                group__name="bargains",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[
+                'param1__avg']
+            offers_estate = models.OtherObjects.objects.filter(
+                group__name="offers_estate",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
+                param1__avg=Avg('param1'), param3__avg=Avg('param3'))
+            point.property_price_offers = offers_estate['param1__avg']
+            point.property_mean_floor = offers_estate['param3__avg']
+            point.property_era = models.OtherObjects.objects.filter(
+                group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[
+                'param2']
+            point.flats_cnt = models.OtherObjects.objects.filter(
+                group__name="flats_cnt",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
+                param1__sum=Sum('param1'))['param1__sum']
+            popul_home_job = models.OtherObjects.objects.filter(
+                group__name="popul_home_job",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
+                param1__sum=Sum('param1'), param3__sum=Sum('param3'))
+            point.popul_home = popul_home_job['param1__sum']
+            point.popul_job = popul_home_job['param3__sum']
+            yndx_food_cnt_amt = models.OtherObjects.objects.filter(
+                group__name="yndx_food_cnt_amt",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
+                param1__sum=Sum('param1'), param3__sum=Sum('param3'))
+            point.yndxfood_sum = yndx_food_cnt_amt['param1__sum']
+            point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum']
+            point.school_cnt = models.OtherObjects.objects.filter(
+                group__name="schools",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.kindergar_cnt = models.OtherObjects.objects.filter(
+                group__name="kindergar",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.public_stop_cnt = models.OtherObjects.objects.filter(
+                group__name="stops",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.sport_center_cnt = models.OtherObjects.objects.filter(
+                group__name="sport_centers",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.pharmacy_cnt = models.OtherObjects.objects.filter(
+                group__name="pharmacies",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.supermarket_cnt = models.OtherObjects.objects.filter(
+                group__name="supermarkets",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.supermarket_premium_cnt = models.OtherObjects.objects.filter(
+                group__name="supermarkets_premium",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.clinic_cnt = models.OtherObjects.objects.filter(
+                group__name="clinics",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.bank_cnt = models.OtherObjects.objects.filter(
+                group__name="banks",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.reca_cnt = models.OtherObjects.objects.filter(
+                group__name="recas",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.lab_cnt = models.OtherObjects.objects.filter(
+                group__name="labs",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.culture_cnt = models.OtherObjects.objects.filter(
+                group__name="cultures",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.attraction_cnt = models.OtherObjects.objects.filter(
+                group__name="attractions",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.mfc_cnt = models.OtherObjects.objects.filter(
+                group__name="public_services",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.bc_cnt = models.OtherObjects.objects.filter(
+                group__name="BC",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.tc_cnt = models.OtherObjects.objects.filter(
+                group__name="TC",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
+            point.business_activity = models.OtherObjects.objects.filter(
+                group__name="business_activity",
+                wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
+                param1__sum=Sum('param1'))['param1__sum']
+            point.age_day = AGE_DAY_LIMIT
+            point.save()
+
+            # print(qs.dist.m)
+
     @staticmethod
     def get_min_distances_to_group(postamat_id: str):
         return {d['pvz_postamates_group']: d['dist'] for d in list(

service/views.py

@@ -19,7 +19,7 @@ from service import models
 from service import pagination
 from service import serializers
 from service import utils
-from service.enums import PointStatus
+from service.enums import PointStatus, MatchingStatus
 from service.permissions import UserPermission
 from service.service import PointService
 from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data
@@ -31,6 +31,7 @@ from rest_framework import filters
 from service.utils import CustomReadOnlyModelViewSet
 from django.db.models import Min, Max
 import os
+from django.forms.models import model_to_dict
 
 
 class AOViewSet(CustomReadOnlyModelViewSet):
@@ -180,7 +181,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
             qs = qs.filter(~Q(pk__in=excluded))
         if included:
             inclded = list(included.split(','))
-            qs2 = models.PlacementPoint.objects.filter(pk__in=inclded).all()
+            qs2 = self.queryset.filter(pk__in=inclded).all()
             qs = (qs | qs2).distinct()
         if group_dists_lt:
             g_d = [list(g.split(',')) for g in group_dists_lt]
@@ -339,6 +340,13 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
     queryset = models.PrePlacementPoint.objects
     serializer_class = serializers.PrePlacementPointSerializer
 
+    def get_queryset(self):
+        qs = super().get_queryset()
+        matching_status = self.request.GET.get('matching_status')
+        if matching_status:
+            qs = qs.filter(matching_status=matching_status)
+        return qs
+
     @action(detail=False, methods=['post'])
     def load_matching_file(self, request):
         file = request.FILES['file'].file
@@ -353,12 +361,25 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
     @action(detail=False, methods=['post'])
     def start_matching(self, request):
         file_id = request.POST['id']
-        total, matched, problem = PointService.start_mathing(file_id)
+        total, matched, problem = PointService().start_mathing(file_id)
+        PointService.make_enrichment()
         return Response(
             {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}},
             status=HTTPStatus.OK,
         )
 
+    @action(detail=False, methods=['post'])
+    def move_points(self, request):
+        qs = self.get_queryset()
+        qs = qs.filter(matching_status=MatchingStatus.New.name)
+        for q in qs:
+            obj = model_to_dict(q)
+            obj.pop('matching_status')
+            obj.pop('id')
+            models.PlacementPoint.objects.create(**obj)
+        models.PrePlacementPoint.objects.all().delete()
+        return Response(status=HTTPStatus.OK, )
+
 
 class refresh_placement_points(APIView):
     @staticmethod