Merge branch 'merge_to_sst_main' into 'sst_main'

Merge to sst main

See merge request spatial/postamates!180

deploy/django.yml

@@ -19,7 +19,7 @@ spec:
       containers:
       - name: django
         image: DEPLOY_IMAGE_TAG
-        command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"]
+        command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py kill_update && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"]
         ports:
           - containerPort: 8000
             name: django-port
@@ -96,4 +96,4 @@ spec:
             port:
               number: 8000
         path: /accounts/
-        pathType: ImplementationSpecific
+        pathType: ImplementationSpecific

postamates/settings.py

@@ -197,3 +197,4 @@ JSON_EXPORT_FILENAME = 'placement_points.json'
 DATA_UPLOAD_MAX_NUMBER_FIELDS = None
 GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M')
 STATUS_TASK_NAME='status_task'
+STATUS_TASK_NAME_IMPORT='import_status_task'

service/admin.py

@@ -63,6 +63,7 @@ class CategoryAdmin(admin.ModelAdmin):
     def save_model(self, request, obj, form, change):
         obj.save()
         cache.clear()
+        run_psql_command()
 
     def delete_model(self, request, obj):
         obj.delete()
@@ -71,16 +72,16 @@ class CategoryAdmin(admin.ModelAdmin):
 
 class PostPvzCategoryAdmin(CategoryAdmin):
     def get_readonly_fields(self, request, obj=None):
-        if obj.id in (1, 2):
+        if obj and obj.id in (1, 2):
             return ['id', 'name']
         else:
             return super().get_readonly_fields(request, obj)
 
     def save_model(self, request, obj, form, change):
         obj.save()
-        run_psql_command()
         if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
             LayerService.update_categories(obj)
+        run_psql_command()
         cache.clear()
 
     def delete_model(self, request, obj):
@@ -96,6 +97,7 @@ class GroupAdmin(admin.ModelAdmin):
     def save_model(self, request, obj, form, change):
         obj.save()
         cache.clear()
+        run_psql_command()
 
     def delete_model(self, request, obj):
         obj.delete()
@@ -104,16 +106,16 @@ class GroupAdmin(admin.ModelAdmin):
 
 class PostPvzGroupAdmin(GroupAdmin):
     def get_readonly_fields(self, request, obj=None):
-        if obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25):
+        if obj and obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25):
             return ['id', 'name']
         else:
             return super().get_readonly_fields(request, obj)
 
     def save_model(self, request, obj, form, change):
         obj.save()
-        run_psql_command()
         if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
             LayerService.update_groups(obj)
+        run_psql_command()
         cache.clear()
 
     def delete_model(self, request, obj):
@@ -126,7 +128,7 @@ class PostPvzGroupAdmin(GroupAdmin):
 
 class OtherObjectsGroupAdmin(GroupAdmin):
     def get_readonly_fields(self, request, obj=None):
-        if obj.id in (4, 5) or obj.id in list(range(22, 43)):
+        if obj and (obj.id in (4, 5) or obj.id in list(range(22, 43))):
             return ['id', 'name']
         else:
             return super().get_readonly_fields(request, obj)
@@ -140,7 +142,7 @@ class OtherObjectsGroupAdmin(GroupAdmin):
 
 class OtherObjectsCategoryAdmin(CategoryAdmin):
     def get_readonly_fields(self, request, obj=None):
-        if obj.id in (4, 5) or obj.id in list(range(22, 43)):
+        if obj and (obj.id in (4, 5) or obj.id in list(range(22, 43))):
             return ['id', 'name']
         else:
             return super().get_readonly_fields(request, obj)

service/layer_service.py

@@ -36,7 +36,7 @@ class LayerService:
     @staticmethod
     def count_post_pvz(point):
         point.rival_post_cnt = models.Post_and_pvz.objects.filter(
-            category__name="Постамат", include_in_ml=True,
+            category__name="Постаматы прочих сетей", include_in_ml=True,
             wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
         point.rival_pvz_cnt = models.Post_and_pvz.objects.filter(
             category__name="ПВЗ", include_in_ml=True,

service/management/commands/create_procedures.py

@@ -4,13 +4,14 @@ from service.utils import run_sql_command, log_to_telegram
 CMD_PIVOT_DIST = """CREATE OR REPLACE VIEW compact_placementpoint AS
 SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpoint;
 CREATE OR REPLACE procedure pivot_dist()
---RETURNS SET OF record
 AS $BODY$
 DECLARE columnNames TEXT;
 BEGIN
 DROP MATERIALIZED VIEW IF EXISTS points_with_dist;
 SELECT 'placement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_placementpointpvzdistance order by 1) as asd
 INTO columnNames;
+IF columnNames IS NOT NULL
+THEN
 EXECUTE format('CREATE MATERIALIZED VIEW points_with_dist AS SELECT *
 FROM CROSSTAB(
     $$
@@ -21,6 +22,9 @@ FROM CROSSTAB(
 ) AS ct(%s)
 LEFT JOIN compact_placementpoint ON placement_point_id=id'
 	,columnNames);
+ELSE
+CREATE MATERIALIZED VIEW points_with_dist AS SELECT placement_point_id, compact_placementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpointpvzdistance LEFT JOIN compact_placementpoint ON placement_point_id=compact_placementpoint.id;
+END IF;
 END;
 $BODY$
 LANGUAGE plpgsql;
@@ -35,6 +39,8 @@ BEGIN
 DROP MATERIALIZED VIEW IF EXISTS prepoints_with_dist;
 SELECT 'preplacement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_preplacementpointpvzdistance order by 1) as asd
 INTO columnNames;
+IF columnNames IS NOT NULL
+THEN
 EXECUTE format('CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT *
 FROM CROSSTAB(
     $$
@@ -45,6 +51,9 @@ FROM CROSSTAB(
 ) AS ct(%s)
 LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id'
 	,columnNames);
+ELSE
+CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT placement_point_id, compact_preplacementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpointpvzdistance LEFT JOIN compact_preplacementpoint ON placement_point_id=compact_preplacementpoint.id;
+END IF;
 END;
 $BODY$
 LANGUAGE plpgsql;

service/management/commands/kill_update.py

@@ -0,0 +1,12 @@
+from django.core.management.base import BaseCommand
+from service.utils import run_sql_command, log_to_telegram
+from service.models import TaskStatus
+from postamates.settings import STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
+
+
+class Command(BaseCommand):
+    help = 'Kill update processes'
+
+    def handle(self, *args, **kwargs):
+        TaskStatus.objects.filter(task_name=STATUS_TASK_NAME).update(status='Перерасчет ML завершен')
+        TaskStatus.objects.filter(task_name=STATUS_TASK_NAME_IMPORT).update(status='Перерасчет ML завершен')

service/migrations/0040_taskstatus_data.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2 on 2023-10-17 09:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('service', '0039_auto_20231011_2120'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='taskstatus',
+            name='data',
+            field=models.JSONField(blank=True, default=dict, null=True, verbose_name='Данные'),
+        ),
+    ]

service/models.py

@@ -262,6 +262,7 @@ class TaskStatus(models.Model):
 
     task_name = models.TextField(blank=False, unique=True, verbose_name='Название задачи')
     status = models.TextField(blank=True, null=True, verbose_name='Статус выполнения')
+    data = models.JSONField(blank=True, null=True, verbose_name='Данные', default=dict)
 
 
 class LastMLCall(models.Model):
@@ -286,11 +287,11 @@ class House(models.Model):
         verbose_name_plural = 'Дома'
         ordering = ('id',)
 
-    year_bld = models.IntegerField(blank=True,null=True)
-    mat_nes = models.TextField(blank=True,null=True)
+    year_bld = models.IntegerField(blank=True, null=True)
+    mat_nes = models.TextField(blank=True, null=True)
     flat_cnt = models.IntegerField(blank=True, null=True)
-    levels = models.TextField(blank=True,null=True)
+    levels = models.TextField(blank=True, null=True)
     doors = models.IntegerField(blank=True, null=True)
-    enrg_cls = models.TextField(blank=True,null=True)
-    street = models.TextField(blank=True,null=True)
-    house_number = models.TextField(blank=True,null=True)
+    enrg_cls = models.TextField(blank=True, null=True)
+    street = models.TextField(blank=True, null=True)
+    house_number = models.TextField(blank=True, null=True)

service/serializers.py

@@ -15,13 +15,13 @@ class PlacementPointSerializer(serializers.ModelSerializer):
         representation['min_distance_to_group'] = min_distances
         return representation
 
+
 class PrePlacementPointSerializer(PlacementPointSerializer):
     class Meta:
         model = models.PrePlacementPoint
         fields = '__all__'
 
 
-
 class PostAndPVZGroupSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.Post_and_pvzGroup

service/service.py

@@ -5,10 +5,11 @@ import pandas as pd
 from django.contrib.gis.measure import Distance
 from django.db.models import F
 
-from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT
+from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT, STATUS_TASK_NAME, \
+    STATUS_TASK_NAME_IMPORT
 from service import models
 from service.enums import PointStatus
-from service.utils import create_columns_dist, run_psql_command
+from service.utils import create_columns_dist, run_psql_command, change_status
 import base64
 import requests
 from postamates.settings import GEOCODER_API_KEY
@@ -18,7 +19,6 @@ from django.db.models import Avg, Sum, Count
 
 
 class PointService:
-
     def update_fact(self, postamat_id: str, fact: int):
         qs = self.get_point_by_postamat_id(postamat_id)
         qs.update(**{'fact': fact})
@@ -27,7 +27,8 @@ class PointService:
         qs = self.get_point_by_id(point_id)
         qs.update(**{'postamat_id': postamat_id})
 
-    def start_mathing(self, obj_id: int):
+    def start_mathing(self, obj_id: int, task_name=STATUS_TASK_NAME_IMPORT):
+        change_status('Шаг 1. Начинается мэтчинг точек', task_name)
         file = models.TempFiles.objects.get(id=obj_id)
         excel_file = base64.b64decode(file.data)
         df = pd.read_excel(excel_file)
@@ -41,13 +42,13 @@ class PointService:
             response = requests.get(req_url).json().get('items')
             if not response:
                 models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
-                                                        status=PointStatus.Pending.name)
+                                                               status=PointStatus.Pending.name)
                 problem += 1
                 continue
             coords = response[0].get('position')
             if not coords:
                 models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
-                                                        status=PointStatus.Pending.name)
+                                                               status=PointStatus.Pending.name)
                 problem += 1
                 continue
             wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")"
@@ -69,9 +70,10 @@ class PointService:
                     if o.get('name'):
                         name = o.pop('name')
                         pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
-                                                                                house_number=house_number,name=name,
-                                                                                matching_status=MatchingStatus.Matched.name,
-                                                                                defaults=o)
+                                                                                    house_number=house_number,
+                                                                                    name=name,
+                                                                                    matching_status=MatchingStatus.Matched.name,
+                                                                                    defaults=o)
                     else:
                         pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
                                                                                     house_number=house_number,
@@ -127,11 +129,18 @@ class PointService:
                                                                matching_status=MatchingStatus.New.name,
                                                                status=PointStatus.Pending.name, area=rayon,
                                                                district=rayon.AO)
-        return total, matched, problem
+            change_status(f'Шаг 1. Обработано {matched + problem} из {total}', task_name)
+
+        ts = models.TaskStatus.objects.get(task_name=task_name)
+        ts.data = {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}
+        ts.save()
+        # return total, matched, problem
 
-    def make_enrichment(self):
+    def make_enrichment(self, task_name=STATUS_TASK_NAME_IMPORT):
+        change_status('Шаг 2. Начинается обогащение точек', task_name)
         points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all()
         groups = models.Post_and_pvzGroup.objects.all()
+
         for point in points:
             origin = point.geometry
             qs = models.PlacementPoint.objects.filter(status=PointStatus.Working.name).annotate(
@@ -240,6 +249,10 @@ class PointService:
             point.save()
             for group in groups:
                 self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance)
+            change_status(
+                f'Шаг 2. Обогащено {points.filter(matching_status=MatchingStatus.Matched.name).count()} из {points.count()}',
+                task_name)
+
         run_psql_command()
 
     @staticmethod
@@ -305,7 +318,8 @@ class PointService:
         data = pd.DataFrame(serializer.data)
         if not data.empty:
             if data['start_date'].any():
-                data['start_date'] = data['start_date'].apply(lambda x :  pd.to_datetime(x).tz_localize(None) if x else None)
+                data['start_date'] = data['start_date'].apply(
+                    lambda x: pd.to_datetime(x).tz_localize(None) if x else None)
             if data['sample_trn'].any():
                 data['sample_trn'] = data['sample_trn'].astype(int)
             data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True)

service/tasks.py

@@ -17,7 +17,7 @@ from sklearn import model_selection as ms
 from sqlalchemy import text
 from django.contrib.gis.db.models.functions import Distance
 from postamates.settings import AGE_DAY_LIMIT
-from postamates.settings import DB_URL, STATUS_TASK_NAME
+from postamates.settings import DB_URL, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
 from service.models import PlacementPoint, LastMLCall
 from service import models
 from service.utils import log_to_telegram
@@ -26,44 +26,37 @@ from io import StringIO
 from django.core.cache import cache
 from service.layer_service import LayerService
 from service.service import PointService
-from service.utils import run_psql_command
+from service.utils import run_psql_command, change_status
 
 
 @shared_task()
-def raschet(table_name='service_placementpoint', need_time=True):
+def raschet(table_name='service_placementpoint', need_time=True, task_name=STATUS_TASK_NAME):
     print('start raschet')
-    status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME)
+    # status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME)
     raschet_objs = models.RaschetObjects.objects.all()
     if raschet_objs:
-        status.status = 'Начало расчета кол-ва ПВЗ вокруг точек'
-        status.save()
+        change_status('Начало расчета кол-ва ПВЗ вокруг точек', task_name=task_name)
         total = raschet_objs.count()
         for _i, r_o in enumerate(raschet_objs):
             obj = models.Post_and_pvz.objects.get(id=r_o.obj_id)
             LayerService().count_post_pvz_for_placementpoint(obj)
-            status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%"
-            status.save()
-        status.status = 'Расчет кол-ва ПВЗ вокруг точек завершен'
-        status.save()
+            change_status(f'Подсчет кол-ва ПВЗ вокруг точек: {str(int((_i + 1) / total * 100))}%', task_name=task_name)
+        change_status('Подсчет кол-ва ПВЗ вокруг точек завершен', task_name=task_name)
     group_objects = models.RaschetGroups.objects.all()
     group_total = group_objects.count()
     if group_objects:
-        status.status = 'Начало расчета расстояний'
-        status.save()
+        change_status('Начало расчета расстояний', task_name=task_name)
         qs = models.PlacementPoint.objects.all()
         for _k, g_o in enumerate(group_objects):
             g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id)
             for q in qs:
                 PointService.calculate_dist_for_group(point=q, group=g)
-            status.status = "Подсчет расстояний: " + str(int(_k / group_total * 100)) + "%"
-            status.save()
-    status.status = "Подсчет расстояний завершен"
-    status.save()
+            change_status(f'Подсчет расстояний: {str(int((_k + 1) / group_total * 100))}%', task_name=task_name)
+    change_status('Расчет завершен', task_name=task_name)
     models.RaschetObjects.objects.all().delete()
     models.RaschetGroups.objects.all().delete()
     # Запуск ML
-    status.status = 'Запуск ML'
-    status.save()
+    change_status('Запуск ML', task_name=task_name)
     log_to_telegram(f'{table_name} start raschet')
     try:
         log_to_telegram('try connect to db')
@@ -121,9 +114,9 @@ def raschet(table_name='service_placementpoint', need_time=True):
         X_trn = pts_trn[feats].drop(columns=['id'])
         Y_trn = pts_trn[['fact']]
 
-
-        status.status = 'Записи для инференса'
-        status.save()
+        # status.status = 'Записи для инференса'
+        # status.save()
+        change_status('Записи для инференса', task_name=task_name)
         # Записи для инференса
         if table_name == 'service_placementpoint':
             pts_inf = pts.loc[(pts.status == 'Pending') |
@@ -152,17 +145,23 @@ def raschet(table_name='service_placementpoint', need_time=True):
             target_feature_coords = np.array(target_feature_coords)
 
             pts_inf['target_dist'] = pts_inf.apply(
-                lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])),
+                lambda x: (
+                    (sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[1]) if (
+                                (x.status == 'Working') or (x.status == 'Installation')) else
+                    (sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])),
                 axis=1,
             )
             pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700
 
             pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True)
-            target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'})
+            target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg(
+                {'cnt': 'count'})
             target_post = target_post.rename(columns={'cnt': 'target_post_cnt'})
             pts_inf = pts_inf.drop(columns=['target_post_cnt'])
             pts_inf = pts_inf.join(target_post.set_index('id'), on='id')
             pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0)
+            pts_inf['target_post_cnt'] = pts_inf.apply(lambda x: ((x.target_post_cnt - 1) if (
+                        (x.status == 'Working') or (x.status == 'Installation')) else x.target_post_cnt), axis=1)
             pts_inf['age_day_init'] = pts_inf['age_day']
             pts_inf['age_day'] = 240
             X_inf = pts_inf[feats]
@@ -173,11 +172,14 @@ def raschet(table_name='service_placementpoint', need_time=True):
             r2_scores = []
             mapes = []
             y_infers = []
-            status.status = 'Обучение inference 0%'
-            status.save()
+            # status.status = 'Обучение inference 0%'
+            # status.save()
+            change_status('Обучение inference 0%', task_name=task_name)
             for i in seeds:
-                status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%'
-                status.save()
+                # status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%'
+                # status.save()
+                change_status(f'Обучение inference: {str(int((seeds.index(i) + 1) / len(seeds) * 100))}%',
+                              task_name=task_name)
                 x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i)
                 model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i)
                 model.fit(x_trn, y_trn, verbose=False)
@@ -187,7 +189,8 @@ def raschet(table_name='service_placementpoint', need_time=True):
                     r2_scores.append(r2_score)
                     mapes.append(mape)
                     y_infers.append(model.predict(X_inf.drop(columns=['id'])))
-            status.status = 'Обучение inference 100%'
+            change_status('Обучение inference 100%', task_name=task_name)
+            # status.status = 'Обучение inference 100%'
             current_pred = sum(y_infers) / 5
 
             # расчет шапов
@@ -195,14 +198,15 @@ def raschet(table_name='service_placementpoint', need_time=True):
             shap_values = explainer(X_inf.drop(columns=['id']))
             shap_fields = pd.DataFrame(shap_values.values)
             shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap'
-            shap_fields = shap_fields.drop(columns = ['age_day_shap'])
+            shap_fields = shap_fields.drop(columns=['age_day_shap'])
             shap_fields['sum'] = abs(shap_fields).sum(axis=1)
-            shap_fields = round(shap_fields.iloc[:,:32].div(shap_fields['sum'], axis=0)*100, 2)
+            shap_fields = round(shap_fields.iloc[:, :32].div(shap_fields['sum'], axis=0) * 100, 2)
 
             # Обновление полей по результатам работы модели
             update_fields = pts_inf[
                 [
-                    'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first',
+                    'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current',
+                    'plan_first',
                     'prediction_first', 'target_post_cnt', 'target_dist'
                 ]
             ]
@@ -227,7 +231,8 @@ def raschet(table_name='service_placementpoint', need_time=True):
                 axis=1,
             )
             update_fields['plan_current'] = update_fields.apply(
-                lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0),
+                lambda x: (
+                    x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0),
                 axis=1,
             )
             update_fields['delta_first'] = update_fields.apply(
@@ -246,8 +251,10 @@ def raschet(table_name='service_placementpoint', need_time=True):
         log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}')
     log_to_telegram('Начинается обновление полей в базе')
     if len(pts_inf) > 0:
-        status.status = 'Перерасчет ML: 50%'
-        status.save()
+
+        # status.status = 'Перерасчет ML: 50%'
+        # status.save()
+        change_status('Перерасчет ML: 50%', task_name=task_name)
         # Загрузка в базу обновленных значений
         try:
             log_to_telegram('Подключение к базе данных 2')
@@ -377,14 +384,14 @@ def raschet(table_name='service_placementpoint', need_time=True):
         log_to_telegram('len(pts_inf) <= 0')
     run_psql_command()
     log_to_telegram('end raschet')
-    status.status = 'Перерасчет ML завершен'
-    status.save()
+    # status.status = 'Перерасчет ML завершен'
+    # status.save()
+    change_status('Перерасчет ML завершен', task_name=task_name)
     if need_time:
         LastMLCall.objects.all().delete()
         LastMLCall.objects.create()
 
 
-
 @shared_task
 def load_post_and_pvz(obj_id: int):
     file = models.TempFiles.objects.get(id=obj_id)
@@ -443,7 +450,7 @@ def load_post_and_pvz(obj_id: int):
     status.status = "Завершено"
     cache.clear()
     status.save()
-
+    run_psql_command()
 
 
 @shared_task()
@@ -486,6 +493,7 @@ def load_other_objects(obj_id: int):
     status.status = "Загрузка данных завершена"
     cache.clear()
     status.save()
+    run_psql_command()
 
 
 @shared_task()
@@ -511,3 +519,10 @@ def load_data(obj_id: int):
     status.status = "Загрузка данных завершена"
     status.save()
     models.TempFiles.objects.all().delete()
+
+
+@shared_task()
+def import_task(file_id):
+    PointService().start_mathing(file_id)
+    PointService().make_enrichment()
+    raschet('service_preplacementpoint', need_time=False, task_name=STATUS_TASK_NAME_IMPORT)

service/utils.py

@@ -15,11 +15,18 @@ from rest_framework.response import Response
 from rest_framework.viewsets import ReadOnlyModelViewSet
 from django.db.models import Avg, Min, Max
 
+from postamates.settings import STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
 
 import psycopg2
 from postamates.settings import DB_URL
 
 
+def change_status(status, task_name=STATUS_TASK_NAME):
+    ts, _ = models.TaskStatus.objects.get_or_create(task_name=task_name)
+    ts.status = status
+    ts.save()
+
+
 def run_sql_command(command):
     connection = psycopg2.connect(
         DB_URL

service/views.py

@@ -14,7 +14,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet
 
 from postamates.settings import AGE_DAY_BORDER
 from postamates.settings import EXCEL_EXPORT_FILENAME
-from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME
+from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
 from service import models
 from service import pagination
 from service import serializers
@@ -22,13 +22,13 @@ from service import utils
 from service.enums import PointStatus, MatchingStatus
 from service.permissions import UserPermission
 from service.service import PointService
-from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data
+from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data, import_task
 from rest_framework.permissions import AllowAny
 from django.shortcuts import redirect
 from django.contrib import messages
 from django_filters.rest_framework import DjangoFilterBackend
 from rest_framework import filters
-from service.utils import CustomReadOnlyModelViewSet
+from service.utils import CustomReadOnlyModelViewSet, run_psql_command
 from django.db.models import Min, Max
 import os
 from django.forms.models import model_to_dict
@@ -279,6 +279,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
             return Response({'message': 'No status'}, HTTPStatus.BAD_REQUEST)
         PointService.update_points_in_radius(qs, new_status)
         PointService.update_status(qs, new_status)
+        run_psql_command()
         return Response(
             {'message': 'status updated'},
             status=HTTPStatus.OK,
@@ -379,11 +380,10 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
     @action(detail=False, methods=['post'])
     def start_matching(self, request):
         file_id = request.POST['id']
-        total, matched, problem = PointService().start_mathing(file_id)
-        PointService().make_enrichment()
-        raschet('service_preplacementpoint', need_time=False)
+        import_task.delay(file_id)
         return Response(
-            {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}},
+            {'message': 'ok'},
+            # {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}},
             status=HTTPStatus.OK,
         )
 
@@ -417,6 +417,7 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
             PointService.delete_preplacement_points(ids)
         else:
             models.PrePlacementPoint.objects.all().delete()
+        run_psql_command()
         return Response(status=HTTPStatus.OK, )
 
     @action(detail=False, methods=['get'])
@@ -426,6 +427,16 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
         response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx')
         return response
 
+    @action(detail=False, methods=['get'])
+    def import_status(self, request):
+        status = models.TaskStatus.objects.filter(task_name=STATUS_TASK_NAME_IMPORT).first()
+        st = None
+        data = {}
+        if status:
+            st = status.status
+            data = status.data
+        return Response({'task_status': st, 'data': data}, status=HTTPStatus.OK)
+
 
 class refresh_placement_points(APIView):
     @staticmethod
@@ -458,7 +469,6 @@ class AvgBiValuesViewSet(APIView):
         return Response(data, status=HTTPStatus.OK)
 
 
-
 @api_view(['POST'])
 def upload_post_and_pvz(request):
     warnings.filterwarnings('ignore')