diff --git a/deploy/django.yml b/deploy/django.yml index 1b60a89..fef0ba2 100644 --- a/deploy/django.yml +++ b/deploy/django.yml @@ -19,7 +19,7 @@ spec: containers: - name: django image: DEPLOY_IMAGE_TAG - command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"] + command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py kill_update && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"] ports: - containerPort: 8000 name: django-port @@ -96,4 +96,4 @@ spec: port: number: 8000 path: /accounts/ - pathType: ImplementationSpecific \ No newline at end of file + pathType: ImplementationSpecific diff --git a/postamates/settings.py b/postamates/settings.py index 1a878c2..fcf0e4f 100644 --- a/postamates/settings.py +++ b/postamates/settings.py @@ -197,3 +197,4 @@ JSON_EXPORT_FILENAME = 'placement_points.json' DATA_UPLOAD_MAX_NUMBER_FIELDS = None GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M') STATUS_TASK_NAME='status_task' +STATUS_TASK_NAME_IMPORT='import_status_task' diff --git a/service/admin.py b/service/admin.py index 0173cba..2166858 100644 --- a/service/admin.py +++ b/service/admin.py @@ -63,6 +63,7 @@ class CategoryAdmin(admin.ModelAdmin): def save_model(self, request, obj, form, change): obj.save() cache.clear() + run_psql_command() def delete_model(self, request, obj): obj.delete() @@ -71,16 +72,16 @@ class CategoryAdmin(admin.ModelAdmin): class PostPvzCategoryAdmin(CategoryAdmin): def get_readonly_fields(self, request, obj=None): - if obj.id in (1, 2): + if obj and obj.id in (1, 2): return ['id', 'name'] else: return super().get_readonly_fields(request, obj) def save_model(self, request, obj, form, change): obj.save() - run_psql_command() if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: LayerService.update_categories(obj) + run_psql_command() cache.clear() def delete_model(self, request, obj): @@ -96,6 +97,7 @@ class GroupAdmin(admin.ModelAdmin): def save_model(self, request, obj, form, change): obj.save() cache.clear() + run_psql_command() def delete_model(self, request, obj): obj.delete() @@ -104,16 +106,16 @@ class GroupAdmin(admin.ModelAdmin): class PostPvzGroupAdmin(GroupAdmin): def get_readonly_fields(self, request, obj=None): - if obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25): + if obj and obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25): return ['id', 'name'] else: return super().get_readonly_fields(request, obj) def save_model(self, request, obj, form, change): obj.save() - run_psql_command() if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: LayerService.update_groups(obj) + run_psql_command() cache.clear() def delete_model(self, request, obj): @@ -126,7 +128,7 @@ class PostPvzGroupAdmin(GroupAdmin): class OtherObjectsGroupAdmin(GroupAdmin): def get_readonly_fields(self, request, obj=None): - if obj.id in (4, 5) or obj.id in list(range(22, 43)): + if obj and (obj.id in (4, 5) or obj.id in list(range(22, 43))): return ['id', 'name'] else: return super().get_readonly_fields(request, obj) @@ -140,7 +142,7 @@ class OtherObjectsGroupAdmin(GroupAdmin): class OtherObjectsCategoryAdmin(CategoryAdmin): def get_readonly_fields(self, request, obj=None): - if obj.id in (4, 5) or obj.id in list(range(22, 43)): + if obj and (obj.id in (4, 5) or obj.id in list(range(22, 43))): return ['id', 'name'] else: return super().get_readonly_fields(request, obj) diff --git a/service/layer_service.py b/service/layer_service.py index ada5077..7d03b46 100644 --- a/service/layer_service.py +++ b/service/layer_service.py @@ -36,7 +36,7 @@ class LayerService: @staticmethod def count_post_pvz(point): point.rival_post_cnt = models.Post_and_pvz.objects.filter( - category__name="Постамат", include_in_ml=True, + category__name="Постаматы прочих сетей", include_in_ml=True, wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( category__name="ПВЗ", include_in_ml=True, diff --git a/service/management/commands/create_procedures.py b/service/management/commands/create_procedures.py index 30fcff4..f80b587 100644 --- a/service/management/commands/create_procedures.py +++ b/service/management/commands/create_procedures.py @@ -4,13 +4,14 @@ from service.utils import run_sql_command, log_to_telegram CMD_PIVOT_DIST = """CREATE OR REPLACE VIEW compact_placementpoint AS SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpoint; CREATE OR REPLACE procedure pivot_dist() ---RETURNS SET OF record AS $BODY$ DECLARE columnNames TEXT; BEGIN DROP MATERIALIZED VIEW IF EXISTS points_with_dist; SELECT 'placement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_placementpointpvzdistance order by 1) as asd INTO columnNames; +IF columnNames IS NOT NULL +THEN EXECUTE format('CREATE MATERIALIZED VIEW points_with_dist AS SELECT * FROM CROSSTAB( $$ @@ -21,6 +22,9 @@ FROM CROSSTAB( ) AS ct(%s) LEFT JOIN compact_placementpoint ON placement_point_id=id' ,columnNames); +ELSE +CREATE MATERIALIZED VIEW points_with_dist AS SELECT placement_point_id, compact_placementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpointpvzdistance LEFT JOIN compact_placementpoint ON placement_point_id=compact_placementpoint.id; +END IF; END; $BODY$ LANGUAGE plpgsql; @@ -35,6 +39,8 @@ BEGIN DROP MATERIALIZED VIEW IF EXISTS prepoints_with_dist; SELECT 'preplacement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_preplacementpointpvzdistance order by 1) as asd INTO columnNames; +IF columnNames IS NOT NULL +THEN EXECUTE format('CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT * FROM CROSSTAB( $$ @@ -45,6 +51,9 @@ FROM CROSSTAB( ) AS ct(%s) LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id' ,columnNames); +ELSE +CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT placement_point_id, compact_preplacementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpointpvzdistance LEFT JOIN compact_preplacementpoint ON placement_point_id=compact_preplacementpoint.id; +END IF; END; $BODY$ LANGUAGE plpgsql; diff --git a/service/management/commands/kill_update.py b/service/management/commands/kill_update.py new file mode 100644 index 0000000..b811c46 --- /dev/null +++ b/service/management/commands/kill_update.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand +from service.utils import run_sql_command, log_to_telegram +from service.models import TaskStatus +from postamates.settings import STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT + + +class Command(BaseCommand): + help = 'Kill update processes' + + def handle(self, *args, **kwargs): + TaskStatus.objects.filter(task_name=STATUS_TASK_NAME).update(status='Перерасчет ML завершен') + TaskStatus.objects.filter(task_name=STATUS_TASK_NAME_IMPORT).update(status='Перерасчет ML завершен') \ No newline at end of file diff --git a/service/migrations/0040_taskstatus_data.py b/service/migrations/0040_taskstatus_data.py new file mode 100644 index 0000000..cb93cc9 --- /dev/null +++ b/service/migrations/0040_taskstatus_data.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2 on 2023-10-17 09:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0039_auto_20231011_2120'), + ] + + operations = [ + migrations.AddField( + model_name='taskstatus', + name='data', + field=models.JSONField(blank=True, default=dict, null=True, verbose_name='Данные'), + ), + ] diff --git a/service/models.py b/service/models.py index 578d0f4..0cf3ca9 100644 --- a/service/models.py +++ b/service/models.py @@ -262,6 +262,7 @@ class TaskStatus(models.Model): task_name = models.TextField(blank=False, unique=True, verbose_name='Название задачи') status = models.TextField(blank=True, null=True, verbose_name='Статус выполнения') + data = models.JSONField(blank=True, null=True, verbose_name='Данные', default=dict) class LastMLCall(models.Model): @@ -286,11 +287,11 @@ class House(models.Model): verbose_name_plural = 'Дома' ordering = ('id',) - year_bld = models.IntegerField(blank=True,null=True) - mat_nes = models.TextField(blank=True,null=True) + year_bld = models.IntegerField(blank=True, null=True) + mat_nes = models.TextField(blank=True, null=True) flat_cnt = models.IntegerField(blank=True, null=True) - levels = models.TextField(blank=True,null=True) + levels = models.TextField(blank=True, null=True) doors = models.IntegerField(blank=True, null=True) - enrg_cls = models.TextField(blank=True,null=True) - street = models.TextField(blank=True,null=True) - house_number = models.TextField(blank=True,null=True) \ No newline at end of file + enrg_cls = models.TextField(blank=True, null=True) + street = models.TextField(blank=True, null=True) + house_number = models.TextField(blank=True, null=True) diff --git a/service/serializers.py b/service/serializers.py index e89d809..710e52c 100644 --- a/service/serializers.py +++ b/service/serializers.py @@ -15,13 +15,13 @@ class PlacementPointSerializer(serializers.ModelSerializer): representation['min_distance_to_group'] = min_distances return representation + class PrePlacementPointSerializer(PlacementPointSerializer): class Meta: model = models.PrePlacementPoint fields = '__all__' - class PostAndPVZGroupSerializer(serializers.ModelSerializer): class Meta: model = models.Post_and_pvzGroup diff --git a/service/service.py b/service/service.py index b7b5ecb..3b94c6e 100644 --- a/service/service.py +++ b/service/service.py @@ -5,10 +5,11 @@ import pandas as pd from django.contrib.gis.measure import Distance from django.db.models import F -from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT +from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT, STATUS_TASK_NAME, \ + STATUS_TASK_NAME_IMPORT from service import models from service.enums import PointStatus -from service.utils import create_columns_dist, run_psql_command +from service.utils import create_columns_dist, run_psql_command, change_status import base64 import requests from postamates.settings import GEOCODER_API_KEY @@ -18,7 +19,6 @@ from django.db.models import Avg, Sum, Count class PointService: - def update_fact(self, postamat_id: str, fact: int): qs = self.get_point_by_postamat_id(postamat_id) qs.update(**{'fact': fact}) @@ -27,7 +27,8 @@ class PointService: qs = self.get_point_by_id(point_id) qs.update(**{'postamat_id': postamat_id}) - def start_mathing(self, obj_id: int): + def start_mathing(self, obj_id: int, task_name=STATUS_TASK_NAME_IMPORT): + change_status('Шаг 1. Начинается мэтчинг точек', task_name) file = models.TempFiles.objects.get(id=obj_id) excel_file = base64.b64decode(file.data) df = pd.read_excel(excel_file) @@ -41,13 +42,13 @@ class PointService: response = requests.get(req_url).json().get('items') if not response: models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, - status=PointStatus.Pending.name) + status=PointStatus.Pending.name) problem += 1 continue coords = response[0].get('position') if not coords: models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, - status=PointStatus.Pending.name) + status=PointStatus.Pending.name) problem += 1 continue wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")" @@ -69,9 +70,10 @@ class PointService: if o.get('name'): name = o.pop('name') pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, - house_number=house_number,name=name, - matching_status=MatchingStatus.Matched.name, - defaults=o) + house_number=house_number, + name=name, + matching_status=MatchingStatus.Matched.name, + defaults=o) else: pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, house_number=house_number, @@ -127,11 +129,18 @@ class PointService: matching_status=MatchingStatus.New.name, status=PointStatus.Pending.name, area=rayon, district=rayon.AO) - return total, matched, problem + change_status(f'Шаг 1. Обработано {matched + problem} из {total}', task_name) + + ts = models.TaskStatus.objects.get(task_name=task_name) + ts.data = {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem} + ts.save() + # return total, matched, problem - def make_enrichment(self): + def make_enrichment(self, task_name=STATUS_TASK_NAME_IMPORT): + change_status('Шаг 2. Начинается обогащение точек', task_name) points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all() groups = models.Post_and_pvzGroup.objects.all() + for point in points: origin = point.geometry qs = models.PlacementPoint.objects.filter(status=PointStatus.Working.name).annotate( @@ -240,6 +249,10 @@ class PointService: point.save() for group in groups: self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance) + change_status( + f'Шаг 2. Обогащено {points.filter(matching_status=MatchingStatus.Matched.name).count()} из {points.count()}', + task_name) + run_psql_command() @staticmethod @@ -305,7 +318,8 @@ class PointService: data = pd.DataFrame(serializer.data) if not data.empty: if data['start_date'].any(): - data['start_date'] = data['start_date'].apply(lambda x : pd.to_datetime(x).tz_localize(None) if x else None) + data['start_date'] = data['start_date'].apply( + lambda x: pd.to_datetime(x).tz_localize(None) if x else None) if data['sample_trn'].any(): data['sample_trn'] = data['sample_trn'].astype(int) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) diff --git a/service/tasks.py b/service/tasks.py index 5f80fca..002b221 100644 --- a/service/tasks.py +++ b/service/tasks.py @@ -17,7 +17,7 @@ from sklearn import model_selection as ms from sqlalchemy import text from django.contrib.gis.db.models.functions import Distance from postamates.settings import AGE_DAY_LIMIT -from postamates.settings import DB_URL, STATUS_TASK_NAME +from postamates.settings import DB_URL, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT from service.models import PlacementPoint, LastMLCall from service import models from service.utils import log_to_telegram @@ -26,44 +26,37 @@ from io import StringIO from django.core.cache import cache from service.layer_service import LayerService from service.service import PointService -from service.utils import run_psql_command +from service.utils import run_psql_command, change_status @shared_task() -def raschet(table_name='service_placementpoint', need_time=True): +def raschet(table_name='service_placementpoint', need_time=True, task_name=STATUS_TASK_NAME): print('start raschet') - status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME) + # status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME) raschet_objs = models.RaschetObjects.objects.all() if raschet_objs: - status.status = 'Начало расчета кол-ва ПВЗ вокруг точек' - status.save() + change_status('Начало расчета кол-ва ПВЗ вокруг точек', task_name=task_name) total = raschet_objs.count() for _i, r_o in enumerate(raschet_objs): obj = models.Post_and_pvz.objects.get(id=r_o.obj_id) LayerService().count_post_pvz_for_placementpoint(obj) - status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%" - status.save() - status.status = 'Расчет кол-ва ПВЗ вокруг точек завершен' - status.save() + change_status(f'Подсчет кол-ва ПВЗ вокруг точек: {str(int((_i + 1) / total * 100))}%', task_name=task_name) + change_status('Подсчет кол-ва ПВЗ вокруг точек завершен', task_name=task_name) group_objects = models.RaschetGroups.objects.all() group_total = group_objects.count() if group_objects: - status.status = 'Начало расчета расстояний' - status.save() + change_status('Начало расчета расстояний', task_name=task_name) qs = models.PlacementPoint.objects.all() for _k, g_o in enumerate(group_objects): g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id) for q in qs: PointService.calculate_dist_for_group(point=q, group=g) - status.status = "Подсчет расстояний: " + str(int(_k / group_total * 100)) + "%" - status.save() - status.status = "Подсчет расстояний завершен" - status.save() + change_status(f'Подсчет расстояний: {str(int((_k + 1) / group_total * 100))}%', task_name=task_name) + change_status('Расчет завершен', task_name=task_name) models.RaschetObjects.objects.all().delete() models.RaschetGroups.objects.all().delete() # Запуск ML - status.status = 'Запуск ML' - status.save() + change_status('Запуск ML', task_name=task_name) log_to_telegram(f'{table_name} start raschet') try: log_to_telegram('try connect to db') @@ -121,9 +114,9 @@ def raschet(table_name='service_placementpoint', need_time=True): X_trn = pts_trn[feats].drop(columns=['id']) Y_trn = pts_trn[['fact']] - - status.status = 'Записи для инференса' - status.save() + # status.status = 'Записи для инференса' + # status.save() + change_status('Записи для инференса', task_name=task_name) # Записи для инференса if table_name == 'service_placementpoint': pts_inf = pts.loc[(pts.status == 'Pending') | @@ -152,17 +145,23 @@ def raschet(table_name='service_placementpoint', need_time=True): target_feature_coords = np.array(target_feature_coords) pts_inf['target_dist'] = pts_inf.apply( - lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])), + lambda x: ( + (sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[1]) if ( + (x.status == 'Working') or (x.status == 'Installation')) else + (sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])), axis=1, ) pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700 pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True) - target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) + target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg( + {'cnt': 'count'}) target_post = target_post.rename(columns={'cnt': 'target_post_cnt'}) pts_inf = pts_inf.drop(columns=['target_post_cnt']) pts_inf = pts_inf.join(target_post.set_index('id'), on='id') pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0) + pts_inf['target_post_cnt'] = pts_inf.apply(lambda x: ((x.target_post_cnt - 1) if ( + (x.status == 'Working') or (x.status == 'Installation')) else x.target_post_cnt), axis=1) pts_inf['age_day_init'] = pts_inf['age_day'] pts_inf['age_day'] = 240 X_inf = pts_inf[feats] @@ -173,11 +172,14 @@ def raschet(table_name='service_placementpoint', need_time=True): r2_scores = [] mapes = [] y_infers = [] - status.status = 'Обучение inference 0%' - status.save() + # status.status = 'Обучение inference 0%' + # status.save() + change_status('Обучение inference 0%', task_name=task_name) for i in seeds: - status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%' - status.save() + # status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%' + # status.save() + change_status(f'Обучение inference: {str(int((seeds.index(i) + 1) / len(seeds) * 100))}%', + task_name=task_name) x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i) model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i) model.fit(x_trn, y_trn, verbose=False) @@ -187,7 +189,8 @@ def raschet(table_name='service_placementpoint', need_time=True): r2_scores.append(r2_score) mapes.append(mape) y_infers.append(model.predict(X_inf.drop(columns=['id']))) - status.status = 'Обучение inference 100%' + change_status('Обучение inference 100%', task_name=task_name) + # status.status = 'Обучение inference 100%' current_pred = sum(y_infers) / 5 # расчет шапов @@ -195,14 +198,15 @@ def raschet(table_name='service_placementpoint', need_time=True): shap_values = explainer(X_inf.drop(columns=['id'])) shap_fields = pd.DataFrame(shap_values.values) shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap' - shap_fields = shap_fields.drop(columns = ['age_day_shap']) + shap_fields = shap_fields.drop(columns=['age_day_shap']) shap_fields['sum'] = abs(shap_fields).sum(axis=1) - shap_fields = round(shap_fields.iloc[:,:32].div(shap_fields['sum'], axis=0)*100, 2) + shap_fields = round(shap_fields.iloc[:, :32].div(shap_fields['sum'], axis=0) * 100, 2) # Обновление полей по результатам работы модели update_fields = pts_inf[ [ - 'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first', + 'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', + 'plan_first', 'prediction_first', 'target_post_cnt', 'target_dist' ] ] @@ -227,7 +231,8 @@ def raschet(table_name='service_placementpoint', need_time=True): axis=1, ) update_fields['plan_current'] = update_fields.apply( - lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), + lambda x: ( + x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), axis=1, ) update_fields['delta_first'] = update_fields.apply( @@ -246,8 +251,10 @@ def raschet(table_name='service_placementpoint', need_time=True): log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}') log_to_telegram('Начинается обновление полей в базе') if len(pts_inf) > 0: - status.status = 'Перерасчет ML: 50%' - status.save() + + # status.status = 'Перерасчет ML: 50%' + # status.save() + change_status('Перерасчет ML: 50%', task_name=task_name) # Загрузка в базу обновленных значений try: log_to_telegram('Подключение к базе данных 2') @@ -377,14 +384,14 @@ def raschet(table_name='service_placementpoint', need_time=True): log_to_telegram('len(pts_inf) <= 0') run_psql_command() log_to_telegram('end raschet') - status.status = 'Перерасчет ML завершен' - status.save() + # status.status = 'Перерасчет ML завершен' + # status.save() + change_status('Перерасчет ML завершен', task_name=task_name) if need_time: LastMLCall.objects.all().delete() LastMLCall.objects.create() - @shared_task def load_post_and_pvz(obj_id: int): file = models.TempFiles.objects.get(id=obj_id) @@ -443,7 +450,7 @@ def load_post_and_pvz(obj_id: int): status.status = "Завершено" cache.clear() status.save() - + run_psql_command() @shared_task() @@ -486,6 +493,7 @@ def load_other_objects(obj_id: int): status.status = "Загрузка данных завершена" cache.clear() status.save() + run_psql_command() @shared_task() @@ -511,3 +519,10 @@ def load_data(obj_id: int): status.status = "Загрузка данных завершена" status.save() models.TempFiles.objects.all().delete() + + +@shared_task() +def import_task(file_id): + PointService().start_mathing(file_id) + PointService().make_enrichment() + raschet('service_preplacementpoint', need_time=False, task_name=STATUS_TASK_NAME_IMPORT) diff --git a/service/utils.py b/service/utils.py index 3549510..429ac3d 100644 --- a/service/utils.py +++ b/service/utils.py @@ -15,11 +15,18 @@ from rest_framework.response import Response from rest_framework.viewsets import ReadOnlyModelViewSet from django.db.models import Avg, Min, Max +from postamates.settings import STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT import psycopg2 from postamates.settings import DB_URL +def change_status(status, task_name=STATUS_TASK_NAME): + ts, _ = models.TaskStatus.objects.get_or_create(task_name=task_name) + ts.status = status + ts.save() + + def run_sql_command(command): connection = psycopg2.connect( DB_URL diff --git a/service/views.py b/service/views.py index 60b97ef..0367c49 100644 --- a/service/views.py +++ b/service/views.py @@ -14,7 +14,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet from postamates.settings import AGE_DAY_BORDER from postamates.settings import EXCEL_EXPORT_FILENAME -from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME +from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT from service import models from service import pagination from service import serializers @@ -22,13 +22,13 @@ from service import utils from service.enums import PointStatus, MatchingStatus from service.permissions import UserPermission from service.service import PointService -from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data +from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data, import_task from rest_framework.permissions import AllowAny from django.shortcuts import redirect from django.contrib import messages from django_filters.rest_framework import DjangoFilterBackend from rest_framework import filters -from service.utils import CustomReadOnlyModelViewSet +from service.utils import CustomReadOnlyModelViewSet, run_psql_command from django.db.models import Min, Max import os from django.forms.models import model_to_dict @@ -279,6 +279,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): return Response({'message': 'No status'}, HTTPStatus.BAD_REQUEST) PointService.update_points_in_radius(qs, new_status) PointService.update_status(qs, new_status) + run_psql_command() return Response( {'message': 'status updated'}, status=HTTPStatus.OK, @@ -379,11 +380,10 @@ class PrePlacementPointViewSet(PlacementPointViewSet): @action(detail=False, methods=['post']) def start_matching(self, request): file_id = request.POST['id'] - total, matched, problem = PointService().start_mathing(file_id) - PointService().make_enrichment() - raschet('service_preplacementpoint', need_time=False) + import_task.delay(file_id) return Response( - {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}}, + {'message': 'ok'}, + # {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}}, status=HTTPStatus.OK, ) @@ -417,6 +417,7 @@ class PrePlacementPointViewSet(PlacementPointViewSet): PointService.delete_preplacement_points(ids) else: models.PrePlacementPoint.objects.all().delete() + run_psql_command() return Response(status=HTTPStatus.OK, ) @action(detail=False, methods=['get']) @@ -426,6 +427,16 @@ class PrePlacementPointViewSet(PlacementPointViewSet): response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx') return response + @action(detail=False, methods=['get']) + def import_status(self, request): + status = models.TaskStatus.objects.filter(task_name=STATUS_TASK_NAME_IMPORT).first() + st = None + data = {} + if status: + st = status.status + data = status.data + return Response({'task_status': st, 'data': data}, status=HTTPStatus.OK) + class refresh_placement_points(APIView): @staticmethod @@ -458,7 +469,6 @@ class AvgBiValuesViewSet(APIView): return Response(data, status=HTTPStatus.OK) - @api_view(['POST']) def upload_post_and_pvz(request): warnings.filterwarnings('ignore')