diff --git a/postamates/settings.py b/postamates/settings.py index 9ee9acb..1a878c2 100644 --- a/postamates/settings.py +++ b/postamates/settings.py @@ -196,3 +196,4 @@ EXCEL_EXPORT_FILENAME = 'placement_points.xlsx' JSON_EXPORT_FILENAME = 'placement_points.json' DATA_UPLOAD_MAX_NUMBER_FIELDS = None GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M') +STATUS_TASK_NAME='status_task' diff --git a/service/admin.py b/service/admin.py index eb54fba..11aa77d 100644 --- a/service/admin.py +++ b/service/admin.py @@ -12,12 +12,11 @@ from service.models import PlacementPoint from service.models import Rayon from service.models import PrePlacementPoint, Post_and_pvz, Post_and_pvzCategory, Post_and_pvzGroup, OtherObjects, \ OtherObjectsGroup, \ - OtherObjectsCategory, PrePlacementPointPVZDistance, TempFiles + OtherObjectsCategory, PrePlacementPointPVZDistance, TempFiles, RaschetGroups from service.models import PlacementPointPVZDistance, TaskStatus from postamates.settings import DEBUG from django.core.cache import cache from service.utils import run_psql_command -from service.tasks import start_pvz_group_count, start_pvz_category_count, raschet class MyAdminSite(AdminSite): @@ -40,8 +39,7 @@ class Post_and_PVZAdmin(admin.ModelAdmin): def save_model(self, request, obj, form, change): obj.save() LayerService().count_post_pvz_for_placementpoint(obj) - if 'include_in_ml' in form.changed_data: - raschet.delay() + RaschetGroups.objects.create(obj_id=obj.group.id) my_admin_site.register(Post_and_pvz, Post_and_PVZAdmin) @@ -49,6 +47,7 @@ my_admin_site.register(OtherObjects) my_admin_site.register(PrePlacementPoint) my_admin_site.register(PrePlacementPointPVZDistance) + class TaskStatusAdmin(admin.ModelAdmin): list_display = ('task_name', 'status') @@ -68,9 +67,8 @@ class CategoryAdmin(admin.ModelAdmin): class PostPvzCategoryAdmin(CategoryAdmin): def save_model(self, request, obj, form, change): obj.save() - if 'include_in_ml' in form.changed_data: - start_pvz_category_count.delay(obj.id) - raschet.delay() + if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: + LayerService.update_categories(obj) cache.clear() def delete_model(self, request, obj): @@ -97,9 +95,8 @@ class PostPvzGroupAdmin(GroupAdmin): if not obj.pk: run_psql_command() obj.save() - if 'include_in_ml' in form.changed_data: - start_pvz_group_count.delay(obj.id) - raschet.delay() + if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: + LayerService.update_groups(obj) cache.clear() diff --git a/service/layer_service.py b/service/layer_service.py index 344c9f7..ada5077 100644 --- a/service/layer_service.py +++ b/service/layer_service.py @@ -4,13 +4,31 @@ from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS class LayerService: - def count_post_pvz_for_placementpoint(self, obj): points = models.PlacementPoint.objects.filter(geometry__distance_lt=(obj.wkt, Distance( m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).all() for point in points: LayerService.count_post_pvz(point) + @staticmethod + def update_categories(instance): + groups = models.Post_and_pvzGroup.objects.filter(category=instance) + groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible) + for gr in groups: + models.RaschetGroups.objects.get_or_create(obj_id=gr.id) + objects = models.Post_and_pvz.objects.filter(group=gr) + objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) + for obj in objects.all(): + models.RaschetObjects.objects.get_or_create(obj_id=obj.id) + + @staticmethod + def update_groups(instance): + models.RaschetGroups.objects.get_or_create(obj_id=instance.id) + objects = models.Post_and_pvz.objects.filter(group=instance) + objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) + for obj in objects.all(): + models.RaschetObjects.objects.get_or_create(obj_id=obj.id) + @staticmethod def get_post_and_pvz_categroies(): return models.Post_and_pvzCategory.objects.all(), models.Post_and_pvzGroup.objects.all() diff --git a/service/migrations/0033_raschetobjects.py b/service/migrations/0033_raschetobjects.py new file mode 100644 index 0000000..0adecf0 --- /dev/null +++ b/service/migrations/0033_raschetobjects.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2023-09-16 09:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0032_auto_20230912_1616'), + ] + + operations = [ + migrations.CreateModel( + name='RaschetObjects', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obj_id', models.IntegerField()), + ], + ), + ] diff --git a/service/migrations/0034_raschetgroups.py b/service/migrations/0034_raschetgroups.py new file mode 100644 index 0000000..e312314 --- /dev/null +++ b/service/migrations/0034_raschetgroups.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2023-09-16 11:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0033_raschetobjects'), + ] + + operations = [ + migrations.CreateModel( + name='RaschetGroups', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obj_id', models.IntegerField()), + ], + ), + ] diff --git a/service/models.py b/service/models.py index d6e272d..6f576ae 100644 --- a/service/models.py +++ b/service/models.py @@ -93,6 +93,7 @@ class PlacementPoint(AbstractPlacementPoint): class PrePlacementPoint(AbstractPlacementPoint): class Meta: ordering = ('id',) + MATCHING_CHOICES = [(tag.name, tag.value) for tag in MatchingStatus] matching_status = models.TextField(null=True, blank=True, choices=MATCHING_CHOICES) @@ -212,11 +213,13 @@ class PlacementPointPVZDistance(models.Model): pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE) dist = models.FloatField(blank=True, null=True, default=None) + class PrePlacementPointPVZDistance(models.Model): placement_point = models.ForeignKey('PrePlacementPoint', default=None, on_delete=models.CASCADE) pvz_postamates_group = models.ForeignKey('Post_and_pvzGroup', default=None, on_delete=models.CASCADE) dist = models.FloatField(blank=True, null=True, default=None) + class TaskStatus(models.Model): class Meta: verbose_name = 'Статус фоновых задач' @@ -235,3 +238,11 @@ class LastMLCall(models.Model): class TempFiles(models.Model): data = models.TextField(blank=False, null=False) + + +class RaschetGroups(models.Model): + obj_id = models.IntegerField(null=False, blank=False) + + +class RaschetObjects(models.Model): + obj_id = models.IntegerField(null=False, blank=False) diff --git a/service/service.py b/service/service.py index e3cfd61..8f44f05 100644 --- a/service/service.py +++ b/service/service.py @@ -8,7 +8,6 @@ from django.db.models import F from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT from service import models from service.enums import PointStatus -from service.tasks import raschet from service.utils import create_columns_dist, run_psql_command import base64 import requests @@ -51,7 +50,8 @@ class PointService: continue wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")" response = response[0]['address'] - obj = models.PlacementPoint.objects.filter(street=response.get('street'), house_number=response.get('houseNumber'), + obj = models.PlacementPoint.objects.filter(street=response.get('street'), + house_number=response.get('houseNumber'), subject_rf=response.get('state'), city=response.get('city'), is_vis=True, category=cat).values().first() @@ -63,12 +63,12 @@ class PointService: else: models.PrePlacementPoint.objects.get_or_create(address=addr, street=response.get('street'), house_number=response.get('houseNumber'), - category=cat, geometry=wkt,sample_trn=False, - matching_status=MatchingStatus.New.name, status = PointStatus.Pending) + category=cat, geometry=wkt, sample_trn=False, + matching_status=MatchingStatus.New.name, + status=PointStatus.Pending) return total, matched, problem - @staticmethod - def make_enrichment(): + def make_enrichment(self): points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all() groups = models.Post_and_pvzGroup.objects.all() for point in points: @@ -174,24 +174,28 @@ class PointService: dist=Dist('geometry', origin)).order_by('dist')[0] point.target_cnt_ao_mean = placement_point.target_cnt_ao_mean point.area = placement_point.area - point.district=placement_point.district + point.district = placement_point.district point.save() for group in groups: - post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( - distance=Dist("wkt", point.geometry)).order_by('distance').first() - d = models.PrePlacementPointPVZDistance.objects.filter(placement_point=point, - pvz_postamates_group=group).first() - if post_object: - if d: - if d.dist > post_object.distance.m: - d.dist = post_object.distance.m - d.save() - else: - models.PrePlacementPointPVZDistance.objects.create(placement_point=point, - pvz_postamates_group=group, - dist=post_object.distance.m) + self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance) run_psql_command() + @staticmethod + def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance): + post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( + distance=Dist("wkt", point.geometry)).order_by('distance').first() + d = instance_type.objects.filter(placement_point=point, + pvz_postamates_group=group).first() + if post_object: + if d: + if d.dist > post_object.distance.m: + d.dist = post_object.distance.m + d.save() + else: + instance_type.objects.create(placement_point=point, + pvz_postamates_group=group, + dist=post_object.distance.m) + @staticmethod def delete_preplacement_points(ids: list): models.PrePlacementPoint.objects.filter(id__in=ids).all().delete() @@ -204,7 +208,6 @@ class PointService: @staticmethod def update_points_in_radius(qs: models.PlacementPoint, new_status: str): - triggers = False for point in qs: if new_status == PointStatus.Installation.name: if point.status == PointStatus.Pending.name: @@ -212,18 +215,12 @@ class PointService: geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)), ) pnts.update(prediction_first=F('prediction_current'), target_post_cnt=F('target_post_cnt') + 1) - triggers = True elif new_status == PointStatus.Cancelled.name or new_status == PointStatus.Pending.name: if point.status == PointStatus.Installation.name: pnts = models.PlacementPoint.objects.filter( geometry__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)), ) pnts.update(target_post_cnt=F('target_post_cnt') - 1 if F('target_post_cnt') != 0 else 0) - triggers = True - elif new_status == PointStatus.Working.name and point.status == PointStatus.Pending.name: - triggers = True - if triggers: - raschet.delay() @staticmethod def update_status(qs: models.PlacementPoint, new_status: str) -> models.PlacementPoint: diff --git a/service/tasks.py b/service/tasks.py index 6a27588..9445bc8 100644 --- a/service/tasks.py +++ b/service/tasks.py @@ -16,7 +16,7 @@ from sklearn import model_selection as ms from sqlalchemy import text from django.contrib.gis.db.models.functions import Distance from postamates.settings import AGE_DAY_LIMIT -from postamates.settings import DB_URL +from postamates.settings import DB_URL, STATUS_TASK_NAME from service.models import PlacementPoint, LastMLCall from service import models from service.utils import log_to_telegram @@ -24,12 +24,43 @@ import base64 from io import StringIO from django.core.cache import cache from service.layer_service import LayerService +from service.service import PointService @shared_task() def raschet(table_name='service_placementpoint'): - LastMLCall.objects.all().delete() - LastMLCall.objects.create() + status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME) + raschet_objs = models.RaschetObjects.objects.all() + if raschet_objs: + status.status = 'Начало расчета кол-ва ПВЗ вокруг точек' + status.save() + total = raschet_objs.count() + for _i, r_o in enumerate(raschet_objs): + obj = models.Post_and_pvz.objects.get(id=r_o.obj_id) + LayerService().count_post_pvz_for_placementpoint(obj) + status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%" + status.save() + status.status = 'Расчет кол-ва ПВЗ вокруг точек завершен' + status.save() + group_objects = models.RaschetGroups.objects.all() + group_total = group_objects.count() + if group_objects: + status.status = 'Начало расчета расстояний' + status.save() + qs = models.PlacementPoint.objects.all() + for _k, g_o in enumerate(group_objects): + g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id) + for q in qs: + PointService.calculate_dist_for_group(point=q, group=g) + status.status = "Подсчет расстояний: " + str(int(_k / group_total * 100)) + "%" + status.save() + status.status = "Подсчет расстояний завершен" + status.save() + models.RaschetObjects.objects.all().delete() + models.RaschetGroups.objects.all().delete() + # Запуск ML + status.status = 'Запуск ML' + status.save() log_to_telegram('start raschet') try: log_to_telegram('try connect to db') @@ -47,11 +78,10 @@ def raschet(table_name='service_placementpoint'): pts = gpd.GeoDataFrame(pts, geometry='geometry', crs='epsg:4326') pts = pts.to_crs('epsg:32637') - feats = [ 'id', 'metro_dist', 'target_dist', 'property_price_bargains', 'property_price_offers', 'property_mean_floor', - 'property_era', 'flats_cnt', 'popul_home', 'popul_job', 'yndxfood_sum', + 'property_era', 'flats_cnt', 'popul_home', 'popul_job', 'yndxfood_sum', 'yndxfood_cnt', 'school_cnt', 'kindergar_cnt', 'target_post_cnt', 'public_stop_cnt', 'sport_center_cnt', 'pharmacy_cnt', 'supermarket_cnt', 'supermarket_premium_cnt', 'clinic_cnt', 'bank_cnt', 'reca_cnt', 'lab_cnt', 'culture_cnt', 'attraction_cnt', 'mfc_cnt', 'bc_cnt', 'tc_cnt', 'rival_pvz_cnt', @@ -77,7 +107,6 @@ def raschet(table_name='service_placementpoint'): ) pts_trn.loc[pts_trn.target_dist > 700, 'target_dist'] = 700 - pts_trn['buf'] = pts_trn.buffer(500) pts_trn = gpd.GeoDataFrame(pts_trn, geometry='buf', crs='epsg:32637') target_post = gpd.sjoin(pts_trn, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) @@ -197,6 +226,8 @@ def raschet(table_name='service_placementpoint'): except Exception as e: log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}') log_to_telegram('Начинается обновление полей в базе') + status.status = 'Перерасчет ML: 50%' + status.save() # Загрузка в базу обновленных значений try: log_to_telegram('Подключение к базе данных 2') @@ -279,7 +310,8 @@ def raschet(table_name='service_placementpoint'): conn2.close() cache.clear() log_to_telegram('end raschet') - + status.status = 'Перерасчет ML завершен' + status.save() @shared_task @@ -340,6 +372,8 @@ def load_post_and_pvz(obj_id: int): status.status = "Завершено" cache.clear() status.save() + LastMLCall.objects.all().delete() + LastMLCall.objects.create() @shared_task() @@ -408,39 +442,38 @@ def load_data(obj_id: int): status.save() models.TempFiles.objects.all().delete() - -@shared_task() -def start_pvz_group_count(instance_id: int): - instance = models.Post_and_pvzGroup.objects.filter(id=instance_id).first() - objects = models.Post_and_pvz.objects.filter(group=instance) - objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) - status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для групп ПВЗ и Постаматов') - total = objects.count() - for _i, obj in enumerate(objects): - status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%" - status.save() - LayerService().count_post_pvz_for_placementpoint(obj) - status.status = "Подсчет завершен" - status.save() - - -@shared_task() -def start_pvz_category_count(instance_id: int): - status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для категорий ПВЗ и Постаматов') - instance = models.Post_and_pvzCategory.objects.filter(id=instance_id).first() - groups = models.Post_and_pvzGroup.objects.filter(category=instance) - groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible) - total = 0 - for gr in groups: - total += models.Post_and_pvz.objects.filter(group=gr).count() - for gr in groups: - _i = 0 - objects = models.Post_and_pvz.objects.filter(group=gr) - objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) - for obj in objects: - _i += 1 - status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int(_i / total * 100)) + "%" - status.save() - LayerService().count_post_pvz_for_placementpoint(obj) - status.status = "Подсчет завершен" - status.save() +# @shared_task() +# def start_pvz_group_count(instance_id: int): +# instance = models.Post_and_pvzGroup.objects.filter(id=instance_id).first() +# objects = models.Post_and_pvz.objects.filter(group=instance) +# objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) +# status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для групп ПВЗ и Постаматов') +# total = objects.count() +# for _i, obj in enumerate(objects): +# status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%" +# status.save() +# LayerService().count_post_pvz_for_placementpoint(obj) +# status.status = "Подсчет завершен" +# status.save() + + +# @shared_task() +# def start_pvz_category_count(instance_id: int): +# status, _ = models.TaskStatus.objects.get_or_create(task_name='Смена статуса для категорий ПВЗ и Постаматов') +# instance = models.Post_and_pvzCategory.objects.filter(id=instance_id).first() +# groups = models.Post_and_pvzGroup.objects.filter(category=instance) +# groups.update(include_in_ml=instance.include_in_ml, visible=instance.visible) +# total = 0 +# for gr in groups: +# total += models.Post_and_pvz.objects.filter(group=gr).count() +# for gr in groups: +# _i = 0 +# objects = models.Post_and_pvz.objects.filter(group=gr) +# objects.update(include_in_ml=instance.include_in_ml, visible=instance.visible) +# for obj in objects: +# _i += 1 +# status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int(_i / total * 100)) + "%" +# status.save() +# LayerService().count_post_pvz_for_placementpoint(obj) +# status.status = "Подсчет завершен" +# status.save() diff --git a/service/views.py b/service/views.py index 0a5905e..32971a2 100644 --- a/service/views.py +++ b/service/views.py @@ -14,7 +14,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet from postamates.settings import AGE_DAY_BORDER from postamates.settings import EXCEL_EXPORT_FILENAME -from postamates.settings import JSON_EXPORT_FILENAME +from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME from service import models from service import pagination from service import serializers @@ -333,7 +333,11 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): @action(detail=False, methods=['get']) def last_time_ml_run(self, request): - return Response({'last_time': models.LastMLCall.objects.first().dt}, status=HTTPStatus.OK) + status = models.TaskStatus.objects.filter(task_name=STATUS_TASK_NAME).first() + st = None + if status: + st = status.status + return Response({'last_time': models.LastMLCall.objects.first().dt, 'task_status': st}, status=HTTPStatus.OK) class PrePlacementPointViewSet(PlacementPointViewSet): @@ -362,8 +366,8 @@ class PrePlacementPointViewSet(PlacementPointViewSet): def start_matching(self, request): file_id = request.POST['id'] total, matched, problem = PointService().start_mathing(file_id) - PointService.make_enrichment() - raschet('service_preplacementpoint') + PointService().make_enrichment() + # raschet('service_preplacementpoint') return Response( {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}}, status=HTTPStatus.OK, @@ -388,7 +392,7 @@ class PrePlacementPointViewSet(PlacementPointViewSet): return Response(status=HTTPStatus.OK, ) @action(detail=False, methods=['get']) - def download_template(self,request): + def download_template(self, request): image_buffer = open('PrePlacementPoints.xlsx', "rb").read() response = HttpResponse(image_buffer, content_type='xlsx') response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx')