Merge branch '2nd_stage' into 'dev'

start_matching_func

See merge request spatial/postamates!77

.env.like

@@ -19,3 +19,4 @@ MARTIN_PORT=3000
 
 # Host domain name (maps in docker-compose to REACT_APP_DOMAIN_URL)
 DOMAIN=postnet-dev.selftech.ru
+GEOCODER_API_KEY = api_key_here

postamates/settings.py

@@ -195,3 +195,4 @@ AGE_DAY_BORDER = 30
 EXCEL_EXPORT_FILENAME = 'placement_points.xlsx'
 JSON_EXPORT_FILENAME = 'placement_points.json'
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None
+GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M')

service/admin.py

@@ -10,7 +10,8 @@ from service.layer_service import LayerService
 from service.models import AO
 from service.models import PlacementPoint
 from service.models import Rayon
-from service.models import Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, OtherObjectsGroup, \
+from service.models import PrePlacementPoint, Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, \
+    OtherObjectsGroup, \
     OtherObjectsCategory
 from service.models import PlacementPointPVZDistance, TaskStatus
 from postamates.settings import DEBUG

service/enums.py

@@ -6,3 +6,8 @@ class PointStatus(Enum):
     Installation = 'Согласование-Установка'
     Working = 'Работает'
     Cancelled = 'Отменено'
+
+class MatchingStatus(Enum):
+    Error = 'Ошибка'
+    New = 'Новая'
+    Matched = 'Совпадение'

service/migrations/0031_preplacementpoint.py

@@ -0,0 +1,94 @@
+# Generated by Django 3.2 on 2023-09-05 17:56
+
+import django.contrib.gis.db.models.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('service', '0030_auto_20230903_2006'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PrePlacementPoint',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('address', models.TextField(blank=True, null=True, verbose_name='Адрес')),
+                ('name', models.TextField(blank=True, null=True, verbose_name='Название')),
+                ('postamat_id', models.IntegerField(blank=True, null=True, unique=True, verbose_name='ID  постамата')),
+                ('category', models.TextField(blank=True, null=True, verbose_name='Категория')),
+                ('status', models.TextField(blank=True, choices=[('Pending', 'К рассмотрению'), ('Installation', 'Согласование-Установка'), ('Working', 'Работает'), ('Cancelled', 'Отменено')], null=True, verbose_name='Статус')),
+                ('start_date', models.DateTimeField(blank=True, null=True)),
+                ('age_day', models.IntegerField(blank=True, null=True, verbose_name='Возраст')),
+                ('prediction_first', models.IntegerField(blank=True, null=True, verbose_name='Прогноз начальный')),
+                ('prediction_current', models.IntegerField(blank=True, null=True, verbose_name='Прогноз текущий')),
+                ('plan_first', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель начальный')),
+                ('plan_current', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель текущий')),
+                ('fact', models.IntegerField(blank=True, null=True, verbose_name='Фактический показатель')),
+                ('fact_raw', models.IntegerField(blank=True, null=True)),
+                ('delta_first', models.IntegerField(blank=True, null=True, verbose_name='Разница начальная')),
+                ('delta_current', models.IntegerField(blank=True, null=True, verbose_name='Разница текущая')),
+                ('sample_trn', models.BooleanField(blank=True, null=True)),
+                ('flat_cnt', models.IntegerField(blank=True, null=True, verbose_name='Количество квартир')),
+                ('year_bld', models.IntegerField(blank=True, null=True, verbose_name='Год постройки')),
+                ('levels', models.IntegerField(blank=True, null=True)),
+                ('enrg_cls', models.TextField(blank=True, null=True)),
+                ('mat_nes', models.TextField(blank=True, null=True)),
+                ('doors', models.IntegerField(blank=True, null=True)),
+                ('flats_cnt', models.IntegerField(blank=True, null=True)),
+                ('popul_home', models.IntegerField(blank=True, null=True)),
+                ('popul_job', models.IntegerField(blank=True, null=True)),
+                ('other_post_cnt', models.IntegerField(blank=True, null=True)),
+                ('target_post_cnt', models.IntegerField(blank=True, null=True)),
+                ('yndxfood_cnt', models.IntegerField(blank=True, null=True)),
+                ('yndxfood_sum', models.IntegerField(blank=True, null=True)),
+                ('yndxfood_cnt_cst', models.IntegerField(blank=True, null=True)),
+                ('geometry', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326, verbose_name='Координаты')),
+                ('is_vis', models.BooleanField(blank=True, null=True)),
+                ('subject_rf', models.TextField(blank=True, null=True)),
+                ('city', models.TextField(blank=True, null=True)),
+                ('street', models.TextField(blank=True, null=True)),
+                ('house_number', models.TextField(blank=True, null=True)),
+                ('entrance', models.TextField(blank=True, null=True)),
+                ('post_code', models.TextField(blank=True, null=True)),
+                ('metro_dist', models.FloatField(blank=True, null=True)),
+                ('target_dist', models.FloatField(blank=True, null=True)),
+                ('property_price_bargains', models.FloatField(blank=True, null=True)),
+                ('property_price_offers', models.FloatField(blank=True, null=True)),
+                ('property_mean_floor', models.FloatField(blank=True, null=True)),
+                ('property_era', models.TextField(blank=True, null=True)),
+                ('business_activity', models.IntegerField(blank=True, null=True)),
+                ('bc_cnt', models.IntegerField(blank=True, null=True)),
+                ('tc_cnt', models.IntegerField(blank=True, null=True)),
+                ('rival_pvz_cnt', models.IntegerField(blank=True, null=True)),
+                ('rival_post_cnt', models.IntegerField(blank=True, null=True)),
+                ('flats_cnt_2', models.IntegerField(blank=True, null=True)),
+                ('school_cnt', models.IntegerField(blank=True, null=True)),
+                ('kindergar_cnt', models.IntegerField(blank=True, null=True)),
+                ('public_stop_cnt', models.IntegerField(blank=True, null=True)),
+                ('sport_center_cnt', models.IntegerField(blank=True, null=True)),
+                ('pharmacy_cnt', models.IntegerField(blank=True, null=True)),
+                ('supermarket_cnt', models.IntegerField(blank=True, null=True)),
+                ('supermarket_premium_cnt', models.IntegerField(blank=True, null=True)),
+                ('clinic_cnt', models.IntegerField(blank=True, null=True)),
+                ('bank_cnt', models.IntegerField(blank=True, null=True)),
+                ('reca_cnt', models.IntegerField(blank=True, null=True)),
+                ('lab_cnt', models.IntegerField(blank=True, null=True)),
+                ('culture_cnt', models.IntegerField(blank=True, null=True)),
+                ('attraction_cnt', models.IntegerField(blank=True, null=True)),
+                ('mfc_cnt', models.IntegerField(blank=True, null=True)),
+                ('target_cnt_ao_mean', models.FloatField(blank=True, null=True)),
+                ('target_cnt_nearby_mean', models.FloatField(blank=True, null=True)),
+                ('target_age_nearby_mean', models.FloatField(blank=True, null=True)),
+                ('matching_status', models.TextField(blank=True, choices=[('Error', 'Ошибка'), ('New', 'Новая'), ('Matched', 'Совпадение')], null=True)),
+                ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.rayon')),
+                ('district', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.ao')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

service/models.py

@@ -2,16 +2,14 @@ from django.contrib.auth.models import User
 from django.contrib.gis.db import models as gis_models
 from django.db import models
 from postamates.settings import SRID
-from service.enums import PointStatus
+from service.enums import PointStatus, MatchingStatus
 
 User._meta.get_field('email')._unique = True
 
 
-class PlacementPoint(models.Model):
+class AbstractPlacementPoint(models.Model):
     class Meta:
-        verbose_name = 'Точка'
+        abstract = True
-        verbose_name_plural = 'Точки'
-        ordering = ('id',)
 
     STATUS_CHOICES = [(tag.name, tag.value) for tag in PointStatus]
     address = models.TextField(null=True, blank=True, verbose_name='Адрес')
@@ -85,6 +83,20 @@ class PlacementPoint(models.Model):
     target_age_nearby_mean = models.FloatField(null=True, blank=True)
 
 
+class PlacementPoint(AbstractPlacementPoint):
+    class Meta:
+        verbose_name = 'Точка'
+        verbose_name_plural = 'Точки'
+        ordering = ('id',)
+
+
+class PrePlacementPoint(AbstractPlacementPoint):
+    class Meta:
+        ordering = ('id',)
+    MATCHING_CHOICES = [(tag.name, tag.value) for tag in MatchingStatus]
+    matching_status = models.TextField(null=True, blank=True, choices=MATCHING_CHOICES)
+
+
 class AO(models.Model):
     class Meta:
         verbose_name = 'АО'

service/serializers.py

@@ -15,6 +15,12 @@ class PlacementPointSerializer(serializers.ModelSerializer):
         representation['min_distance_to_group'] = min_distances
         return representation
 
+class PrePlacementPointSerializer(PlacementPointSerializer):
+    class Meta:
+        model = models.PrePlacementPoint
+        fields = '__all__'
+
+
 
 class PostAndPVZGroupSerializer(serializers.ModelSerializer):
     class Meta:

service/service.py

@@ -10,6 +10,10 @@ from service import models
 from service.enums import PointStatus
 from service.tasks import raschet
 from service.utils import create_columns_dist
+import base64
+import requests
+from postamates.settings import GEOCODER_API_KEY
+from service.enums import MatchingStatus
 
 
 class PointService:
@@ -22,6 +26,37 @@ class PointService:
         qs = self.get_point_by_id(point_id)
         qs.update(**{'postamat_id': postamat_id})
 
+    @staticmethod
+    def start_mathing(obj_id: int):
+        file = models.TempFiles.objects.get(id=obj_id)
+        excel_file = base64.b64decode(file.data)
+        df = pd.read_excel(excel_file)
+        total = df.shape[0]
+        matched = 0
+        problem = 0
+        for _i, row in df.iterrows():
+            addr = row['Адрес']
+            cat = row['Категория объекта']
+            req_url = f"https://geocode.search.hereapi.com/v1/geocode?q={addr}&apiKey={GEOCODER_API_KEY}"
+            response = requests.get(req_url).json().get('items')
+            if not response:
+                models.PrePlacementPoint.objects.create(address=addr, matching_status=MatchingStatus.Error.name)
+                problem+=1
+                continue
+            response = response[0]['address']
+            obj = models.PlacementPoint.objects.filter(street=response['street'], house_number=response['houseNumber'],
+                                                       category=cat).values().first()
+            if obj:
+                obj.pop('id')
+                models.PrePlacementPoint.objects.create(**{**obj, "matching_status": MatchingStatus.Matched.name})
+                matched += 1
+            else:
+                models.PrePlacementPoint.objects.create(address=addr, street=response['street'],
+                                                        house_number=response['houseNumber'],
+                                                        category=cat, matching_status=MatchingStatus.New.name)
+        models.TempFiles.objects.all().delete()
+        return total, matched, problem
+
     @staticmethod
     def get_min_distances_to_group(postamat_id: str):
         return {d['pvz_postamates_group']: d['dist'] for d in list(
@@ -73,7 +108,8 @@ class PointService:
         if not data.empty:
             if data['start_date'].any():
                 data['start_date'] = data.get('start_date').dt.tz_localize(None)
-            data['sample_trn'] = data['sample_trn'].astype(int)
+            if data['sample_trn'].any():
+                data['sample_trn'] = data['sample_trn'].astype(int)
             data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True)
             data['min_distance_to_group'] = data['min_distance_to_group'].apply(lambda x: list(x.items()))
             new_columns = data.apply(create_columns_dist, axis=1)

service/tasks.py

@@ -245,6 +245,7 @@ def raschet():
     except:
         conn2 = None
         log_to_telegram('Не удалось подключиться к базе данных')
+
     # prediction_current
     if conn2 is not None:
         update_records1 = []
@@ -450,12 +451,13 @@ def start_pvz_group_count(instance_id: int):
     status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для групп ПВЗ и Постаматов')
     total = objects.count()
     for _i, obj in enumerate(objects):
-        status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i+1) / total * 100)) + "%"
+        status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%"
         status.save()
         LayerService().count_post_pvz_for_placementpoint(obj)
     status.status = "Подсчет завершен"
     status.save()
 
+
 @shared_task()
 def start_pvz_category_count(instance_id: int):
     status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для категорий ПВЗ и Постаматов')
@@ -464,13 +466,13 @@ def start_pvz_category_count(instance_id: int):
     groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
     total = 0
     for gr in groups:
-        total+=models.Post_and_pvz.objects.filter(group=gr).count()
+        total += models.Post_and_pvz.objects.filter(group=gr).count()
     for gr in groups:
-        _i=0
+        _i = 0
         objects = models.Post_and_pvz.objects.filter(group=gr)
         objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible)
         for obj in objects:
-            _i+=1
+            _i += 1
             status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int(_i / total * 100)) + "%"
             status.save()
             LayerService().count_post_pvz_for_placementpoint(obj)

service/urls.py

@@ -4,6 +4,7 @@ from service import views
 
 router = routers.DefaultRouter()
 router.register('placement_points', views.PlacementPointViewSet)
+router.register('pre_placement_points', views.PrePlacementPointViewSet)
 router.register('ao_rayons', views.AOViewSet)
 router.register('postamate_and_pvz_groups', views.PostAndPVZCategoryViewSet)
 router.register('other_object_groups', views.OtherObjectsCategoryViewSet)

service/views.py

@@ -302,7 +302,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
     @action(detail=False, methods=['get'])
     def to_excel(self, request):
         qs = self.get_queryset()
-        serializer = serializers.PlacementPointSerializer(qs, many=True)
+        serializer = self.serializer_class(qs, many=True)
         filename = EXCEL_EXPORT_FILENAME
         res = HttpResponse(
             PointService.to_excel(serializer),
@@ -314,7 +314,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
     @action(detail=False, methods=['get'])
     def to_json(self, request):
         qs = self.get_queryset()
-        serializer = serializers.PlacementPointSerializer(qs, many=True)
+        serializer = self.serializer_class(qs, many=True)
         filename = JSON_EXPORT_FILENAME
         response = HttpResponse(PointService.to_json(serializer), content_type='application/json')
         response['Content-Disposition'] = f'attachment; filename={filename}'
@@ -335,6 +335,31 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
         return Response({'last_time': models.LastMLCall.objects.first().dt}, status=HTTPStatus.OK)
 
 
+class PrePlacementPointViewSet(PlacementPointViewSet):
+    queryset = models.PrePlacementPoint.objects
+    serializer_class = serializers.PrePlacementPointSerializer
+
+    @action(detail=False, methods=['post'])
+    def load_matching_file(self, request):
+        file = request.FILES['file'].file
+        file_bytes = file.read()
+        excel_base64 = base64.b64encode(file_bytes).decode()
+        obj = models.TempFiles.objects.create(data=excel_base64)
+        return Response(
+            {'id': obj.id},
+            status=HTTPStatus.OK,
+        )
+
+    @action(detail=False, methods=['post'])
+    def start_matching(self, request):
+        file_id = request.POST['id']
+        total, matched, problem = PointService.start_mathing(file_id)
+        return Response(
+            {'message': {'total':total,'matched': matched, 'error': problem, 'unmatched': total - matched - problem}},
+            status=HTTPStatus.OK,
+        )
+
+
 class refresh_placement_points(APIView):
     @staticmethod
     def post(request):