From ef48df8ad30fa2893fe363bba82cc4a451a14797 Mon Sep 17 00:00:00 2001 From: AlexP077 Date: Wed, 6 Sep 2023 19:31:19 +0300 Subject: [PATCH] add_pre_placementpoint --- service/admin.py | 3 +- service/enums.py | 5 ++ service/migrations/0031_preplacementpoint.py | 94 ++++++++++++++++++++ service/models.py | 22 +++-- service/serializers.py | 6 ++ service/service.py | 38 +++++++- service/tasks.py | 34 ++----- service/urls.py | 1 + service/views.py | 25 ++++-- 9 files changed, 187 insertions(+), 41 deletions(-) create mode 100644 service/migrations/0031_preplacementpoint.py diff --git a/service/admin.py b/service/admin.py index 1d6c4d1..3d07571 100644 --- a/service/admin.py +++ b/service/admin.py @@ -10,7 +10,8 @@ from service.layer_service import LayerService from service.models import AO from service.models import PlacementPoint from service.models import Rayon -from service.models import Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, OtherObjectsGroup, \ +from service.models import PrePlacementPoint, Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, \ + OtherObjectsGroup, \ OtherObjectsCategory from service.models import PlacementPointPVZDistance, TaskStatus from postamates.settings import DEBUG diff --git a/service/enums.py b/service/enums.py index 3a8df3c..e517954 100644 --- a/service/enums.py +++ b/service/enums.py @@ -6,3 +6,8 @@ class PointStatus(Enum): Installation = 'Согласование-Установка' Working = 'Работает' Cancelled = 'Отменено' + +class MatchingStatus(Enum): + Error = 'Ошибка' + New = 'Новая' + Matched = 'Совпадение' diff --git a/service/migrations/0031_preplacementpoint.py b/service/migrations/0031_preplacementpoint.py new file mode 100644 index 0000000..431e3da --- /dev/null +++ b/service/migrations/0031_preplacementpoint.py @@ -0,0 +1,94 @@ +# Generated by Django 3.2 on 2023-09-05 17:56 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0030_auto_20230903_2006'), + ] + + operations = [ + migrations.CreateModel( + name='PrePlacementPoint', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('address', models.TextField(blank=True, null=True, verbose_name='Адрес')), + ('name', models.TextField(blank=True, null=True, verbose_name='Название')), + ('postamat_id', models.IntegerField(blank=True, null=True, unique=True, verbose_name='ID постамата')), + ('category', models.TextField(blank=True, null=True, verbose_name='Категория')), + ('status', models.TextField(blank=True, choices=[('Pending', 'К рассмотрению'), ('Installation', 'Согласование-Установка'), ('Working', 'Работает'), ('Cancelled', 'Отменено')], null=True, verbose_name='Статус')), + ('start_date', models.DateTimeField(blank=True, null=True)), + ('age_day', models.IntegerField(blank=True, null=True, verbose_name='Возраст')), + ('prediction_first', models.IntegerField(blank=True, null=True, verbose_name='Прогноз начальный')), + ('prediction_current', models.IntegerField(blank=True, null=True, verbose_name='Прогноз текущий')), + ('plan_first', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель начальный')), + ('plan_current', models.IntegerField(blank=True, null=True, verbose_name='Плановый показатель текущий')), + ('fact', models.IntegerField(blank=True, null=True, verbose_name='Фактический показатель')), + ('fact_raw', models.IntegerField(blank=True, null=True)), + ('delta_first', models.IntegerField(blank=True, null=True, verbose_name='Разница начальная')), + ('delta_current', models.IntegerField(blank=True, null=True, verbose_name='Разница текущая')), + ('sample_trn', models.BooleanField(blank=True, null=True)), + ('flat_cnt', models.IntegerField(blank=True, null=True, verbose_name='Количество квартир')), + ('year_bld', models.IntegerField(blank=True, null=True, verbose_name='Год постройки')), + ('levels', models.IntegerField(blank=True, null=True)), + ('enrg_cls', models.TextField(blank=True, null=True)), + ('mat_nes', models.TextField(blank=True, null=True)), + ('doors', models.IntegerField(blank=True, null=True)), + ('flats_cnt', models.IntegerField(blank=True, null=True)), + ('popul_home', models.IntegerField(blank=True, null=True)), + ('popul_job', models.IntegerField(blank=True, null=True)), + ('other_post_cnt', models.IntegerField(blank=True, null=True)), + ('target_post_cnt', models.IntegerField(blank=True, null=True)), + ('yndxfood_cnt', models.IntegerField(blank=True, null=True)), + ('yndxfood_sum', models.IntegerField(blank=True, null=True)), + ('yndxfood_cnt_cst', models.IntegerField(blank=True, null=True)), + ('geometry', django.contrib.gis.db.models.fields.PointField(null=True, srid=4326, verbose_name='Координаты')), + ('is_vis', models.BooleanField(blank=True, null=True)), + ('subject_rf', models.TextField(blank=True, null=True)), + ('city', models.TextField(blank=True, null=True)), + ('street', models.TextField(blank=True, null=True)), + ('house_number', models.TextField(blank=True, null=True)), + ('entrance', models.TextField(blank=True, null=True)), + ('post_code', models.TextField(blank=True, null=True)), + ('metro_dist', models.FloatField(blank=True, null=True)), + ('target_dist', models.FloatField(blank=True, null=True)), + ('property_price_bargains', models.FloatField(blank=True, null=True)), + ('property_price_offers', models.FloatField(blank=True, null=True)), + ('property_mean_floor', models.FloatField(blank=True, null=True)), + ('property_era', models.TextField(blank=True, null=True)), + ('business_activity', models.IntegerField(blank=True, null=True)), + ('bc_cnt', models.IntegerField(blank=True, null=True)), + ('tc_cnt', models.IntegerField(blank=True, null=True)), + ('rival_pvz_cnt', models.IntegerField(blank=True, null=True)), + ('rival_post_cnt', models.IntegerField(blank=True, null=True)), + ('flats_cnt_2', models.IntegerField(blank=True, null=True)), + ('school_cnt', models.IntegerField(blank=True, null=True)), + ('kindergar_cnt', models.IntegerField(blank=True, null=True)), + ('public_stop_cnt', models.IntegerField(blank=True, null=True)), + ('sport_center_cnt', models.IntegerField(blank=True, null=True)), + ('pharmacy_cnt', models.IntegerField(blank=True, null=True)), + ('supermarket_cnt', models.IntegerField(blank=True, null=True)), + ('supermarket_premium_cnt', models.IntegerField(blank=True, null=True)), + ('clinic_cnt', models.IntegerField(blank=True, null=True)), + ('bank_cnt', models.IntegerField(blank=True, null=True)), + ('reca_cnt', models.IntegerField(blank=True, null=True)), + ('lab_cnt', models.IntegerField(blank=True, null=True)), + ('culture_cnt', models.IntegerField(blank=True, null=True)), + ('attraction_cnt', models.IntegerField(blank=True, null=True)), + ('mfc_cnt', models.IntegerField(blank=True, null=True)), + ('target_cnt_ao_mean', models.FloatField(blank=True, null=True)), + ('target_cnt_nearby_mean', models.FloatField(blank=True, null=True)), + ('target_age_nearby_mean', models.FloatField(blank=True, null=True)), + ('matching_status', models.TextField(blank=True, choices=[('Error', 'Ошибка'), ('New', 'Новая'), ('Matched', 'Совпадение')], null=True)), + ('area', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.rayon')), + ('district', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='service.ao')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/service/models.py b/service/models.py index 413b763..01702b6 100644 --- a/service/models.py +++ b/service/models.py @@ -2,16 +2,14 @@ from django.contrib.auth.models import User from django.contrib.gis.db import models as gis_models from django.db import models from postamates.settings import SRID -from service.enums import PointStatus +from service.enums import PointStatus, MatchingStatus User._meta.get_field('email')._unique = True -class PlacementPoint(models.Model): +class AbstractPlacementPoint(models.Model): class Meta: - verbose_name = 'Точка' - verbose_name_plural = 'Точки' - ordering = ('id',) + abstract = True STATUS_CHOICES = [(tag.name, tag.value) for tag in PointStatus] address = models.TextField(null=True, blank=True, verbose_name='Адрес') @@ -85,6 +83,20 @@ class PlacementPoint(models.Model): target_age_nearby_mean = models.FloatField(null=True, blank=True) +class PlacementPoint(AbstractPlacementPoint): + class Meta: + verbose_name = 'Точка' + verbose_name_plural = 'Точки' + ordering = ('id',) + + +class PrePlacementPoint(AbstractPlacementPoint): + class Meta: + ordering = ('id',) + MATCHING_CHOICES = [(tag.name, tag.value) for tag in MatchingStatus] + matching_status = models.TextField(null=True, blank=True, choices=MATCHING_CHOICES) + + class AO(models.Model): class Meta: verbose_name = 'АО' diff --git a/service/serializers.py b/service/serializers.py index 39cfa98..e89d809 100644 --- a/service/serializers.py +++ b/service/serializers.py @@ -15,6 +15,12 @@ class PlacementPointSerializer(serializers.ModelSerializer): representation['min_distance_to_group'] = min_distances return representation +class PrePlacementPointSerializer(PlacementPointSerializer): + class Meta: + model = models.PrePlacementPoint + fields = '__all__' + + class PostAndPVZGroupSerializer(serializers.ModelSerializer): class Meta: diff --git a/service/service.py b/service/service.py index 990e580..80aecd2 100644 --- a/service/service.py +++ b/service/service.py @@ -10,6 +10,10 @@ from service import models from service.enums import PointStatus from service.tasks import raschet from service.utils import create_columns_dist +import base64 +import requests +from postamates.settings import GEOCODER_API_KEY +from service.enums import MatchingStatus class PointService: @@ -22,6 +26,37 @@ class PointService: qs = self.get_point_by_id(point_id) qs.update(**{'postamat_id': postamat_id}) + @staticmethod + def start_mathing(obj_id: int): + file = models.TempFiles.objects.get(id=obj_id) + excel_file = base64.b64decode(file.data) + df = pd.read_excel(excel_file) + total = df.shape[0] + matched = 0 + problem = 0 + for _i, row in df.iterrows(): + addr = row['Адрес'] + cat = row['Категория объекта'] + req_url = f"https://geocode.search.hereapi.com/v1/geocode?q={addr}&apiKey={GEOCODER_API_KEY}" + response = requests.get(req_url).json().get('items') + if not response: + models.PrePlacementPoint.objects.create(address=addr, matching_status=MatchingStatus.Error.name) + problem+=1 + continue + response = response[0]['address'] + obj = models.PlacementPoint.objects.filter(street=response['street'], house_number=response['houseNumber'], + category=cat).values().first() + if obj: + obj.pop('id') + models.PrePlacementPoint.objects.create(**{**obj, "matching_status": MatchingStatus.Matched.name}) + matched += 1 + else: + models.PrePlacementPoint.objects.create(address=addr, street=response['street'], + house_number=response['houseNumber'], + category=cat, matching_status=MatchingStatus.New.name) + models.TempFiles.objects.all().delete() + return total, matched, problem + @staticmethod def get_min_distances_to_group(postamat_id: str): return {d['pvz_postamates_group']: d['dist'] for d in list( @@ -73,7 +108,8 @@ class PointService: if not data.empty: if data['start_date'].any(): data['start_date'] = data.get('start_date').dt.tz_localize(None) - data['sample_trn'] = data['sample_trn'].astype(int) + if data['sample_trn'].any(): + data['sample_trn'] = data['sample_trn'].astype(int) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) data['min_distance_to_group'] = data['min_distance_to_group'].apply(lambda x: list(x.items())) new_columns = data.apply(create_columns_dist, axis=1) diff --git a/service/tasks.py b/service/tasks.py index 43c9616..3a76eed 100644 --- a/service/tasks.py +++ b/service/tasks.py @@ -24,8 +24,7 @@ import base64 from io import StringIO from django.core.cache import cache from service.layer_service import LayerService -import requests -from postamates.settings import GEOCODER_API_KEY + @shared_task() def raschet(): @@ -444,28 +443,6 @@ def load_data(obj_id: int): models.TempFiles.objects.all().delete() -@shared_task() -def start_matching(obj_id: int): - file = models.TempFiles.objects.get(id=obj_id) - status, _ = models.TaskStatus.objects.get_or_create(task_name='Мэтчинг точек') - excel_file = base64.b64decode(file.data) - df = pd.read_excel(excel_file) - total = df.shape[0] - matched = 0 - for _i, row in df.iterrows(): - status.status = f'Мэтчинг: {_i} из {total}' - status.save() - addr = row['Адрес'] - cat = row['Категория объекта'] - req_url = f"https://geocode.search.hereapi.com/v1/geocode?q={addr}&apiKey={GEOCODER_API_KEY}" - response = requests.get(req_url).json()['items'][0]['address'] - obj = models.PlacementPoint.objects.filter(street=response['street'], house_number=response['houseNumber'], category=cat).first() - if obj: - matched += 1 - status.status = f"Мэтчинг завершен. Смэтчилось {matched}, несмэтчилось {total-matched}" - status.save() - models.TempFiles.objects.all().delete() - @shared_task() def start_pvz_group_count(instance_id: int): instance = models.Post_and_pvzGroup.objects.filter(id=instance_id).first() @@ -474,12 +451,13 @@ def start_pvz_group_count(instance_id: int): status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для групп ПВЗ и Постаматов') total = objects.count() for _i, obj in enumerate(objects): - status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i+1) / total * 100)) + "%" + status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%" status.save() LayerService().count_post_pvz_for_placementpoint(obj) status.status = "Подсчет завершен" status.save() + @shared_task() def start_pvz_category_count(instance_id: int): status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для категорий ПВЗ и Постаматов') @@ -488,13 +466,13 @@ def start_pvz_category_count(instance_id: int): groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible) total = 0 for gr in groups: - total+=models.Post_and_pvz.objects.filter(group=gr).count() + total += models.Post_and_pvz.objects.filter(group=gr).count() for gr in groups: - _i=0 + _i = 0 objects = models.Post_and_pvz.objects.filter(group=gr) objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) for obj in objects: - _i+=1 + _i += 1 status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int(_i / total * 100)) + "%" status.save() LayerService().count_post_pvz_for_placementpoint(obj) diff --git a/service/urls.py b/service/urls.py index 5385596..69d3e6d 100644 --- a/service/urls.py +++ b/service/urls.py @@ -4,6 +4,7 @@ from service import views router = routers.DefaultRouter() router.register('placement_points', views.PlacementPointViewSet) +router.register('pre_placement_points', views.PrePlacementPointViewSet) router.register('ao_rayons', views.AOViewSet) router.register('postamate_and_pvz_groups', views.PostAndPVZCategoryViewSet) router.register('other_object_groups', views.OtherObjectsCategoryViewSet) diff --git a/service/views.py b/service/views.py index 23d1571..754f254 100644 --- a/service/views.py +++ b/service/views.py @@ -22,7 +22,7 @@ from service import utils from service.enums import PointStatus from service.permissions import UserPermission from service.service import PointService -from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data, start_matching +from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data from rest_framework.permissions import AllowAny from django.shortcuts import redirect from django.contrib import messages @@ -302,7 +302,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): @action(detail=False, methods=['get']) def to_excel(self, request): qs = self.get_queryset() - serializer = serializers.PlacementPointSerializer(qs, many=True) + serializer = self.serializer_class(qs, many=True) filename = EXCEL_EXPORT_FILENAME res = HttpResponse( PointService.to_excel(serializer), @@ -314,7 +314,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): @action(detail=False, methods=['get']) def to_json(self, request): qs = self.get_queryset() - serializer = serializers.PlacementPointSerializer(qs, many=True) + serializer = self.serializer_class(qs, many=True) filename = JSON_EXPORT_FILENAME response = HttpResponse(PointService.to_json(serializer), content_type='application/json') response['Content-Disposition'] = f'attachment; filename={filename}' @@ -334,15 +334,28 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): def last_time_ml_run(self, request): return Response({'last_time': models.LastMLCall.objects.first().dt}, status=HTTPStatus.OK) + +class PrePlacementPointViewSet(PlacementPointViewSet): + queryset = models.PrePlacementPoint.objects + serializer_class = serializers.PrePlacementPointSerializer + @action(detail=False, methods=['post']) - def start_matching(self, request): + def load_matching_file(self, request): file = request.FILES['file'].file file_bytes = file.read() excel_base64 = base64.b64encode(file_bytes).decode() obj = models.TempFiles.objects.create(data=excel_base64) - start_matching.delay(obj.id) return Response( - {'message': 'OK'}, + {'id': obj.id}, + status=HTTPStatus.OK, + ) + + @action(detail=False, methods=['post']) + def start_matching(self, request): + file_id = request.POST['id'] + total, matched, problem = PointService.start_mathing(file_id) + return Response( + {'message': {'total':total,'matched': matched, 'error': problem, 'unmatched': total - matched - problem}}, status=HTTPStatus.OK, )