TRUNK
/
postnet
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge to sst main

Browse Source
dev
Aleksandr Popov 2 years ago committed by Timofey Malinin
parent ce3e64703b
commit d5d97a059d
13 changed files with 164 additions and 75 deletions
Show Stats Download Patch File Download Diff File

4
deploy/django.yml
Unescape View File

@ -19,7 +19,7 @@ spec:
containers: containers:
- name: django - name: django
image: DEPLOY_IMAGE_TAG image: DEPLOY_IMAGE_TAG
command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"] command: ["sh", "-c", "python manage.py delete_views && python manage.py create_procedures && python manage.py migrate && python manage.py create_views && python manage.py kill_update && python manage.py runserver 0.0.0.0:${DJANGO_PORT}"]
ports: ports:
- containerPort: 8000 - containerPort: 8000
name: django-port name: django-port
@ -96,4 +96,4 @@ spec:
port: port:
number: 8000 number: 8000
path: /accounts/ path: /accounts/
pathType: ImplementationSpecific pathType: ImplementationSpecific

1
postamates/settings.py
Unescape View File

@ -197,3 +197,4 @@ JSON_EXPORT_FILENAME = 'placement_points.json'
DATA_UPLOAD_MAX_NUMBER_FIELDS = None DATA_UPLOAD_MAX_NUMBER_FIELDS = None
GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M') GEOCODER_API_KEY = os.getenv('GEOCODER_API_KEY','TzgdKWgyI2nfaz1WHRD-aYJK4e400MiOJQP7Enf1e1M')
STATUS_TASK_NAME='status_task' STATUS_TASK_NAME='status_task'
STATUS_TASK_NAME_IMPORT='import_status_task'

14
service/admin.py
Unescape View File

@ -63,6 +63,7 @@ class CategoryAdmin(admin.ModelAdmin):
def save_model(self, request, obj, form, change): def save_model(self, request, obj, form, change):
obj.save() obj.save()
cache.clear() cache.clear()
run_psql_command()
def delete_model(self, request, obj): def delete_model(self, request, obj):
obj.delete() obj.delete()
@ -71,16 +72,16 @@ class CategoryAdmin(admin.ModelAdmin):
class PostPvzCategoryAdmin(CategoryAdmin): class PostPvzCategoryAdmin(CategoryAdmin):
def get_readonly_fields(self, request, obj=None): def get_readonly_fields(self, request, obj=None):
if obj.id in (1, 2): if obj and obj.id in (1, 2):
return ['id', 'name'] return ['id', 'name']
else: else:
return super().get_readonly_fields(request, obj) return super().get_readonly_fields(request, obj)
def save_model(self, request, obj, form, change): def save_model(self, request, obj, form, change):
obj.save() obj.save()
run_psql_command()
if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
LayerService.update_categories(obj) LayerService.update_categories(obj)
run_psql_command()
cache.clear() cache.clear()
def delete_model(self, request, obj): def delete_model(self, request, obj):
@ -96,6 +97,7 @@ class GroupAdmin(admin.ModelAdmin):
def save_model(self, request, obj, form, change): def save_model(self, request, obj, form, change):
obj.save() obj.save()
cache.clear() cache.clear()
run_psql_command()
def delete_model(self, request, obj): def delete_model(self, request, obj):
obj.delete() obj.delete()
@ -104,16 +106,16 @@ class GroupAdmin(admin.ModelAdmin):
class PostPvzGroupAdmin(GroupAdmin): class PostPvzGroupAdmin(GroupAdmin):
def get_readonly_fields(self, request, obj=None): def get_readonly_fields(self, request, obj=None):
if obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25): if obj and obj.id in (7, 8, 9, 10, 12, 17, 19, 20, 24, 25):
return ['id', 'name'] return ['id', 'name']
else: else:
return super().get_readonly_fields(request, obj) return super().get_readonly_fields(request, obj)
def save_model(self, request, obj, form, change): def save_model(self, request, obj, form, change):
obj.save() obj.save()
run_psql_command()
if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data: if 'include_in_ml' in form.changed_data or 'visible' in form.changed_data:
LayerService.update_groups(obj) LayerService.update_groups(obj)
run_psql_command()
cache.clear() cache.clear()
def delete_model(self, request, obj): def delete_model(self, request, obj):
@ -126,7 +128,7 @@ class PostPvzGroupAdmin(GroupAdmin):
class OtherObjectsGroupAdmin(GroupAdmin): class OtherObjectsGroupAdmin(GroupAdmin):
def get_readonly_fields(self, request, obj=None): def get_readonly_fields(self, request, obj=None):
if obj.id in (4, 5) or obj.id in list(range(22, 43)): if obj and (obj.id in (4, 5) or obj.id in list(range(22, 43))):
return ['id', 'name'] return ['id', 'name']
else: else:
return super().get_readonly_fields(request, obj) return super().get_readonly_fields(request, obj)
@ -140,7 +142,7 @@ class OtherObjectsGroupAdmin(GroupAdmin):
class OtherObjectsCategoryAdmin(CategoryAdmin): class OtherObjectsCategoryAdmin(CategoryAdmin):
def get_readonly_fields(self, request, obj=None): def get_readonly_fields(self, request, obj=None):
if obj.id in (4, 5) or obj.id in list(range(22, 43)): if obj and (obj.id in (4, 5) or obj.id in list(range(22, 43))):
return ['id', 'name'] return ['id', 'name']
else: else:
return super().get_readonly_fields(request, obj) return super().get_readonly_fields(request, obj)

2
service/layer_service.py
Unescape View File

@ -36,7 +36,7 @@ class LayerService:
@staticmethod @staticmethod
def count_post_pvz(point): def count_post_pvz(point):
point.rival_post_cnt = models.Post_and_pvz.objects.filter( point.rival_post_cnt = models.Post_and_pvz.objects.filter(
category__name="Постамат", include_in_ml=True, category__name="Постаматы прочих сетей", include_in_ml=True,
wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(point.geometry, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( point.rival_pvz_cnt = models.Post_and_pvz.objects.filter(
category__name="ПВЗ", include_in_ml=True, category__name="ПВЗ", include_in_ml=True,

11
service/management/commands/create_procedures.py
Unescape View File

@ -4,13 +4,14 @@ from service.utils import run_sql_command, log_to_telegram
CMD_PIVOT_DIST = """CREATE OR REPLACE VIEW compact_placementpoint AS CMD_PIVOT_DIST = """CREATE OR REPLACE VIEW compact_placementpoint AS
SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpoint; SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpoint;
CREATE OR REPLACE procedure pivot_dist() CREATE OR REPLACE procedure pivot_dist()
--RETURNS SET OF record
AS $BODY$ AS $BODY$
DECLARE columnNames TEXT; DECLARE columnNames TEXT;
BEGIN BEGIN
DROP MATERIALIZED VIEW IF EXISTS points_with_dist; DROP MATERIALIZED VIEW IF EXISTS points_with_dist;
SELECT 'placement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_placementpointpvzdistance order by 1) as asd SELECT 'placement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_placementpointpvzdistance order by 1) as asd
INTO columnNames; INTO columnNames;
IF columnNames IS NOT NULL
THEN
EXECUTE format('CREATE MATERIALIZED VIEW points_with_dist AS SELECT * EXECUTE format('CREATE MATERIALIZED VIEW points_with_dist AS SELECT *
FROM CROSSTAB( FROM CROSSTAB(
$$ $$
@ -21,6 +22,9 @@ FROM CROSSTAB(
) AS ct(%s) ) AS ct(%s)
LEFT JOIN compact_placementpoint ON placement_point_id=id' LEFT JOIN compact_placementpoint ON placement_point_id=id'
,columnNames); ,columnNames);
ELSE
CREATE MATERIALIZED VIEW points_with_dist AS SELECT placement_point_id, compact_placementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_placementpointpvzdistance LEFT JOIN compact_placementpoint ON placement_point_id=compact_placementpoint.id;
END IF;
END; END;
$BODY$ $BODY$
LANGUAGE plpgsql; LANGUAGE plpgsql;
@ -35,6 +39,8 @@ BEGIN
DROP MATERIALIZED VIEW IF EXISTS prepoints_with_dist; DROP MATERIALIZED VIEW IF EXISTS prepoints_with_dist;
SELECT 'preplacement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_preplacementpointpvzdistance order by 1) as asd SELECT 'preplacement_point_id bigint, ' || string_agg(c, ', ') FROM (SELECT distinct pvz_postamates_group_id, 'd' || pvz_postamates_group_id || ' double precision' as c from service_preplacementpointpvzdistance order by 1) as asd
INTO columnNames; INTO columnNames;
IF columnNames IS NOT NULL
THEN
EXECUTE format('CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT * EXECUTE format('CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT *
FROM CROSSTAB( FROM CROSSTAB(
$$ $$
@ -45,6 +51,9 @@ FROM CROSSTAB(
) AS ct(%s) ) AS ct(%s)
LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id' LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id'
,columnNames); ,columnNames);
ELSE
CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT placement_point_id, compact_preplacementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpointpvzdistance LEFT JOIN compact_preplacementpoint ON placement_point_id=compact_preplacementpoint.id;
END IF;
END; END;
$BODY$ $BODY$
LANGUAGE plpgsql; LANGUAGE plpgsql;

12
service/management/commands/kill_update.py
Unescape View File

@ -0,0 +1,12 @@
from django.core.management.base import BaseCommand
from service.utils import run_sql_command, log_to_telegram
from service.models import TaskStatus
from postamates.settings import STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
class Command(BaseCommand):
help = 'Kill update processes'
def handle(self, *args, **kwargs):
TaskStatus.objects.filter(task_name=STATUS_TASK_NAME).update(status='Перерасчет ML завершен')
TaskStatus.objects.filter(task_name=STATUS_TASK_NAME_IMPORT).update(status='Перерасчет ML завершен')

18
service/migrations/0040_taskstatus_data.py
Unescape View File

@ -0,0 +1,18 @@
# Generated by Django 3.2 on 2023-10-17 09:36
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0039_auto_20231011_2120'),
]
operations = [
migrations.AddField(
model_name='taskstatus',
name='data',
field=models.JSONField(blank=True, default=dict, null=True, verbose_name='Данные'),
),
]

13
service/models.py
Unescape View File

@ -262,6 +262,7 @@ class TaskStatus(models.Model):
task_name = models.TextField(blank=False, unique=True, verbose_name='Название задачи') task_name = models.TextField(blank=False, unique=True, verbose_name='Название задачи')
status = models.TextField(blank=True, null=True, verbose_name='Статус выполнения') status = models.TextField(blank=True, null=True, verbose_name='Статус выполнения')
data = models.JSONField(blank=True, null=True, verbose_name='Данные', default=dict)
class LastMLCall(models.Model): class LastMLCall(models.Model):
@ -286,11 +287,11 @@ class House(models.Model):
verbose_name_plural = 'Дома' verbose_name_plural = 'Дома'
ordering = ('id',) ordering = ('id',)
year_bld = models.IntegerField(blank=True,null=True) year_bld = models.IntegerField(blank=True, null=True)
mat_nes = models.TextField(blank=True,null=True) mat_nes = models.TextField(blank=True, null=True)
flat_cnt = models.IntegerField(blank=True, null=True) flat_cnt = models.IntegerField(blank=True, null=True)
levels = models.TextField(blank=True,null=True) levels = models.TextField(blank=True, null=True)
doors = models.IntegerField(blank=True, null=True) doors = models.IntegerField(blank=True, null=True)
enrg_cls = models.TextField(blank=True,null=True) enrg_cls = models.TextField(blank=True, null=True)
street = models.TextField(blank=True,null=True) street = models.TextField(blank=True, null=True)
house_number = models.TextField(blank=True,null=True) house_number = models.TextField(blank=True, null=True)

2
service/serializers.py
Unescape View File

@ -15,13 +15,13 @@ class PlacementPointSerializer(serializers.ModelSerializer):
representation['min_distance_to_group'] = min_distances representation['min_distance_to_group'] = min_distances
return representation return representation
class PrePlacementPointSerializer(PlacementPointSerializer): class PrePlacementPointSerializer(PlacementPointSerializer):
class Meta: class Meta:
model = models.PrePlacementPoint model = models.PrePlacementPoint
fields = '__all__' fields = '__all__'
class PostAndPVZGroupSerializer(serializers.ModelSerializer): class PostAndPVZGroupSerializer(serializers.ModelSerializer):
class Meta: class Meta:
model = models.Post_and_pvzGroup model = models.Post_and_pvzGroup

38
service/service.py
Unescape View File

@ -5,10 +5,11 @@ import pandas as pd
from django.contrib.gis.measure import Distance from django.contrib.gis.measure import Distance
from django.db.models import F from django.db.models import F
from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT, STATUS_TASK_NAME, \
STATUS_TASK_NAME_IMPORT
from service import models from service import models
from service.enums import PointStatus from service.enums import PointStatus
from service.utils import create_columns_dist, run_psql_command from service.utils import create_columns_dist, run_psql_command, change_status
import base64 import base64
import requests import requests
from postamates.settings import GEOCODER_API_KEY from postamates.settings import GEOCODER_API_KEY
@ -18,7 +19,6 @@ from django.db.models import Avg, Sum, Count
class PointService: class PointService:
def update_fact(self, postamat_id: str, fact: int): def update_fact(self, postamat_id: str, fact: int):
qs = self.get_point_by_postamat_id(postamat_id) qs = self.get_point_by_postamat_id(postamat_id)
qs.update(**{'fact': fact}) qs.update(**{'fact': fact})
@ -27,7 +27,8 @@ class PointService:
qs = self.get_point_by_id(point_id) qs = self.get_point_by_id(point_id)
qs.update(**{'postamat_id': postamat_id}) qs.update(**{'postamat_id': postamat_id})
def start_mathing(self, obj_id: int): def start_mathing(self, obj_id: int, task_name=STATUS_TASK_NAME_IMPORT):
change_status('Шаг 1. Начинается мэтчинг точек', task_name)
file = models.TempFiles.objects.get(id=obj_id) file = models.TempFiles.objects.get(id=obj_id)
excel_file = base64.b64decode(file.data) excel_file = base64.b64decode(file.data)
df = pd.read_excel(excel_file) df = pd.read_excel(excel_file)
@ -41,13 +42,13 @@ class PointService:
response = requests.get(req_url).json().get('items') response = requests.get(req_url).json().get('items')
if not response: if not response:
models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
status=PointStatus.Pending.name) status=PointStatus.Pending.name)
problem += 1 problem += 1
continue continue
coords = response[0].get('position') coords = response[0].get('position')
if not coords: if not coords:
models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
status=PointStatus.Pending.name) status=PointStatus.Pending.name)
problem += 1 problem += 1
continue continue
wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")" wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")"
@ -69,9 +70,10 @@ class PointService:
if o.get('name'): if o.get('name'):
name = o.pop('name') name = o.pop('name')
pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
house_number=house_number,name=name, house_number=house_number,
matching_status=MatchingStatus.Matched.name, name=name,
defaults=o) matching_status=MatchingStatus.Matched.name,
defaults=o)
else: else:
pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
house_number=house_number, house_number=house_number,
@ -127,11 +129,18 @@ class PointService:
matching_status=MatchingStatus.New.name, matching_status=MatchingStatus.New.name,
status=PointStatus.Pending.name, area=rayon, status=PointStatus.Pending.name, area=rayon,
district=rayon.AO) district=rayon.AO)
return total, matched, problem change_status(f'Шаг 1. Обработано {matched + problem} из {total}', task_name)
ts = models.TaskStatus.objects.get(task_name=task_name)
ts.data = {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}
ts.save()
# return total, matched, problem
def make_enrichment(self): def make_enrichment(self, task_name=STATUS_TASK_NAME_IMPORT):
change_status('Шаг 2. Начинается обогащение точек', task_name)
points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all() points = models.PrePlacementPoint.objects.filter(matching_status=MatchingStatus.New.name).all()
groups = models.Post_and_pvzGroup.objects.all() groups = models.Post_and_pvzGroup.objects.all()
for point in points: for point in points:
origin = point.geometry origin = point.geometry
qs = models.PlacementPoint.objects.filter(status=PointStatus.Working.name).annotate( qs = models.PlacementPoint.objects.filter(status=PointStatus.Working.name).annotate(
@ -240,6 +249,10 @@ class PointService:
point.save() point.save()
for group in groups: for group in groups:
self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance) self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance)
change_status(
f'Шаг 2. Обогащено {points.filter(matching_status=MatchingStatus.Matched.name).count()} из {points.count()}',
task_name)
run_psql_command() run_psql_command()
@staticmethod @staticmethod
@ -305,7 +318,8 @@ class PointService:
data = pd.DataFrame(serializer.data) data = pd.DataFrame(serializer.data)
if not data.empty: if not data.empty:
if data['start_date'].any(): if data['start_date'].any():
data['start_date'] = data['start_date'].apply(lambda x : pd.to_datetime(x).tz_localize(None) if x else None) data['start_date'] = data['start_date'].apply(
lambda x: pd.to_datetime(x).tz_localize(None) if x else None)
if data['sample_trn'].any(): if data['sample_trn'].any():
data['sample_trn'] = data['sample_trn'].astype(int) data['sample_trn'] = data['sample_trn'].astype(int)
data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True)

91
service/tasks.py
Unescape View File

@ -17,7 +17,7 @@ from sklearn import model_selection as ms
from sqlalchemy import text from sqlalchemy import text
from django.contrib.gis.db.models.functions import Distance from django.contrib.gis.db.models.functions import Distance
from postamates.settings import AGE_DAY_LIMIT from postamates.settings import AGE_DAY_LIMIT
from postamates.settings import DB_URL, STATUS_TASK_NAME from postamates.settings import DB_URL, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
from service.models import PlacementPoint, LastMLCall from service.models import PlacementPoint, LastMLCall
from service import models from service import models
from service.utils import log_to_telegram from service.utils import log_to_telegram
@ -26,44 +26,37 @@ from io import StringIO
from django.core.cache import cache from django.core.cache import cache
from service.layer_service import LayerService from service.layer_service import LayerService
from service.service import PointService from service.service import PointService
from service.utils import run_psql_command from service.utils import run_psql_command, change_status
@shared_task() @shared_task()
def raschet(table_name='service_placementpoint', need_time=True): def raschet(table_name='service_placementpoint', need_time=True, task_name=STATUS_TASK_NAME):
print('start raschet') print('start raschet')
status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME) # status, _ = models.TaskStatus.objects.get_or_create(task_name=STATUS_TASK_NAME)
raschet_objs = models.RaschetObjects.objects.all() raschet_objs = models.RaschetObjects.objects.all()
if raschet_objs: if raschet_objs:
status.status = 'Начало расчета кол-ва ПВЗ вокруг точек' change_status('Начало расчета кол-ва ПВЗ вокруг точек', task_name=task_name)
status.save()
total = raschet_objs.count() total = raschet_objs.count()
for _i, r_o in enumerate(raschet_objs): for _i, r_o in enumerate(raschet_objs):
obj = models.Post_and_pvz.objects.get(id=r_o.obj_id) obj = models.Post_and_pvz.objects.get(id=r_o.obj_id)
LayerService().count_post_pvz_for_placementpoint(obj) LayerService().count_post_pvz_for_placementpoint(obj)
status.status = "Подсчет кол-ва ПВЗ вокруг точек: " + str(int((_i + 1) / total * 100)) + "%" change_status(f'Подсчет кол-ва ПВЗ вокруг точек: {str(int((_i + 1) / total * 100))}%', task_name=task_name)
status.save() change_status('Подсчет кол-ва ПВЗ вокруг точек завершен', task_name=task_name)
status.status = 'Расчет кол-ва ПВЗ вокруг точек завершен'
status.save()
group_objects = models.RaschetGroups.objects.all() group_objects = models.RaschetGroups.objects.all()
group_total = group_objects.count() group_total = group_objects.count()
if group_objects: if group_objects:
status.status = 'Начало расчета расстояний' change_status('Начало расчета расстояний', task_name=task_name)
status.save()
qs = models.PlacementPoint.objects.all() qs = models.PlacementPoint.objects.all()
for _k, g_o in enumerate(group_objects): for _k, g_o in enumerate(group_objects):
g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id) g = models.Post_and_pvzGroup.objects.get(id=g_o.obj_id)
for q in qs: for q in qs:
PointService.calculate_dist_for_group(point=q, group=g) PointService.calculate_dist_for_group(point=q, group=g)
status.status = "Подсчет расстояний: " + str(int(_k / group_total * 100)) + "%" change_status(f'Подсчет расстояний: {str(int((_k + 1) / group_total * 100))}%', task_name=task_name)
status.save() change_status('Расчет завершен', task_name=task_name)
status.status = "Подсчет расстояний завершен"
status.save()
models.RaschetObjects.objects.all().delete() models.RaschetObjects.objects.all().delete()
models.RaschetGroups.objects.all().delete() models.RaschetGroups.objects.all().delete()
# Запуск ML # Запуск ML
status.status = 'Запуск ML' change_status('Запуск ML', task_name=task_name)
status.save()
log_to_telegram(f'{table_name} start raschet') log_to_telegram(f'{table_name} start raschet')
try: try:
log_to_telegram('try connect to db') log_to_telegram('try connect to db')
@ -121,9 +114,9 @@ def raschet(table_name='service_placementpoint', need_time=True):
X_trn = pts_trn[feats].drop(columns=['id']) X_trn = pts_trn[feats].drop(columns=['id'])
Y_trn = pts_trn[['fact']] Y_trn = pts_trn[['fact']]
# status.status = 'Записи для инференса'
status.status = 'Записи для инференса' # status.save()
status.save() change_status('Записи для инференса', task_name=task_name)
# Записи для инференса # Записи для инференса
if table_name == 'service_placementpoint': if table_name == 'service_placementpoint':
pts_inf = pts.loc[(pts.status == 'Pending') | pts_inf = pts.loc[(pts.status == 'Pending') |
@ -152,17 +145,23 @@ def raschet(table_name='service_placementpoint', need_time=True):
target_feature_coords = np.array(target_feature_coords) target_feature_coords = np.array(target_feature_coords)
pts_inf['target_dist'] = pts_inf.apply( pts_inf['target_dist'] = pts_inf.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])), lambda x: (
(sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[1]) if (
(x.status == 'Working') or (x.status == 'Installation')) else
(sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])),
axis=1, axis=1,
) )
pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700 pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700
pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True) pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True)
target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg(
{'cnt': 'count'})
target_post = target_post.rename(columns={'cnt': 'target_post_cnt'}) target_post = target_post.rename(columns={'cnt': 'target_post_cnt'})
pts_inf = pts_inf.drop(columns=['target_post_cnt']) pts_inf = pts_inf.drop(columns=['target_post_cnt'])
pts_inf = pts_inf.join(target_post.set_index('id'), on='id') pts_inf = pts_inf.join(target_post.set_index('id'), on='id')
pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0) pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0)
pts_inf['target_post_cnt'] = pts_inf.apply(lambda x: ((x.target_post_cnt - 1) if (
(x.status == 'Working') or (x.status == 'Installation')) else x.target_post_cnt), axis=1)
pts_inf['age_day_init'] = pts_inf['age_day'] pts_inf['age_day_init'] = pts_inf['age_day']
pts_inf['age_day'] = 240 pts_inf['age_day'] = 240
X_inf = pts_inf[feats] X_inf = pts_inf[feats]
@ -173,11 +172,14 @@ def raschet(table_name='service_placementpoint', need_time=True):
r2_scores = [] r2_scores = []
mapes = [] mapes = []
y_infers = [] y_infers = []
status.status = 'Обучение inference 0%' # status.status = 'Обучение inference 0%'
status.save() # status.save()
change_status('Обучение inference 0%', task_name=task_name)
for i in seeds: for i in seeds:
status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%' # status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%'
status.save() # status.save()
change_status(f'Обучение inference: {str(int((seeds.index(i) + 1) / len(seeds) * 100))}%',
task_name=task_name)
x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i) x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i)
model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i) model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i)
model.fit(x_trn, y_trn, verbose=False) model.fit(x_trn, y_trn, verbose=False)
@ -187,7 +189,8 @@ def raschet(table_name='service_placementpoint', need_time=True):
r2_scores.append(r2_score) r2_scores.append(r2_score)
mapes.append(mape) mapes.append(mape)
y_infers.append(model.predict(X_inf.drop(columns=['id']))) y_infers.append(model.predict(X_inf.drop(columns=['id'])))
status.status = 'Обучение inference 100%' change_status('Обучение inference 100%', task_name=task_name)
# status.status = 'Обучение inference 100%'
current_pred = sum(y_infers) / 5 current_pred = sum(y_infers) / 5
# расчет шапов # расчет шапов
@ -195,14 +198,15 @@ def raschet(table_name='service_placementpoint', need_time=True):
shap_values = explainer(X_inf.drop(columns=['id'])) shap_values = explainer(X_inf.drop(columns=['id']))
shap_fields = pd.DataFrame(shap_values.values) shap_fields = pd.DataFrame(shap_values.values)
shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap' shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap'
shap_fields = shap_fields.drop(columns = ['age_day_shap']) shap_fields = shap_fields.drop(columns=['age_day_shap'])
shap_fields['sum'] = abs(shap_fields).sum(axis=1) shap_fields['sum'] = abs(shap_fields).sum(axis=1)
shap_fields = round(shap_fields.iloc[:,:32].div(shap_fields['sum'], axis=0)*100, 2) shap_fields = round(shap_fields.iloc[:, :32].div(shap_fields['sum'], axis=0) * 100, 2)
# Обновление полей по результатам работы модели # Обновление полей по результатам работы модели
update_fields = pts_inf[ update_fields = pts_inf[
[ [
'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first', 'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current',
'plan_first',
'prediction_first', 'target_post_cnt', 'target_dist' 'prediction_first', 'target_post_cnt', 'target_dist'
] ]
] ]
@ -227,7 +231,8 @@ def raschet(table_name='service_placementpoint', need_time=True):
axis=1, axis=1,
) )
update_fields['plan_current'] = update_fields.apply( update_fields['plan_current'] = update_fields.apply(
lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), lambda x: (
x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0),
axis=1, axis=1,
) )
update_fields['delta_first'] = update_fields.apply( update_fields['delta_first'] = update_fields.apply(
@ -246,8 +251,10 @@ def raschet(table_name='service_placementpoint', need_time=True):
log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}') log_to_telegram(f'Ошибка при обновлении полей в базе данных: {e}')
log_to_telegram('Начинается обновление полей в базе') log_to_telegram('Начинается обновление полей в базе')
if len(pts_inf) > 0: if len(pts_inf) > 0:
status.status = 'Перерасчет ML: 50%'
status.save() # status.status = 'Перерасчет ML: 50%'
# status.save()
change_status('Перерасчет ML: 50%', task_name=task_name)
# Загрузка в базу обновленных значений # Загрузка в базу обновленных значений
try: try:
log_to_telegram('Подключение к базе данных 2') log_to_telegram('Подключение к базе данных 2')
@ -377,14 +384,14 @@ def raschet(table_name='service_placementpoint', need_time=True):
log_to_telegram('len(pts_inf) <= 0') log_to_telegram('len(pts_inf) <= 0')
run_psql_command() run_psql_command()
log_to_telegram('end raschet') log_to_telegram('end raschet')
status.status = 'Перерасчет ML завершен' # status.status = 'Перерасчет ML завершен'
status.save() # status.save()
change_status('Перерасчет ML завершен', task_name=task_name)
if need_time: if need_time:
LastMLCall.objects.all().delete() LastMLCall.objects.all().delete()
LastMLCall.objects.create() LastMLCall.objects.create()
@shared_task @shared_task
def load_post_and_pvz(obj_id: int): def load_post_and_pvz(obj_id: int):
file = models.TempFiles.objects.get(id=obj_id) file = models.TempFiles.objects.get(id=obj_id)
@ -443,7 +450,7 @@ def load_post_and_pvz(obj_id: int):
status.status = "Завершено" status.status = "Завершено"
cache.clear() cache.clear()
status.save() status.save()
run_psql_command()
@shared_task() @shared_task()
@ -486,6 +493,7 @@ def load_other_objects(obj_id: int):
status.status = "Загрузка данных завершена" status.status = "Загрузка данных завершена"
cache.clear() cache.clear()
status.save() status.save()
run_psql_command()
@shared_task() @shared_task()
@ -511,3 +519,10 @@ def load_data(obj_id: int):
status.status = "Загрузка данных завершена" status.status = "Загрузка данных завершена"
status.save() status.save()
models.TempFiles.objects.all().delete() models.TempFiles.objects.all().delete()
@shared_task()
def import_task(file_id):
PointService().start_mathing(file_id)
PointService().make_enrichment()
raschet('service_preplacementpoint', need_time=False, task_name=STATUS_TASK_NAME_IMPORT)

7
service/utils.py
Unescape View File

@ -15,11 +15,18 @@ from rest_framework.response import Response
from rest_framework.viewsets import ReadOnlyModelViewSet from rest_framework.viewsets import ReadOnlyModelViewSet
from django.db.models import Avg, Min, Max from django.db.models import Avg, Min, Max
from postamates.settings import STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
import psycopg2 import psycopg2
from postamates.settings import DB_URL from postamates.settings import DB_URL
def change_status(status, task_name=STATUS_TASK_NAME):
ts, _ = models.TaskStatus.objects.get_or_create(task_name=task_name)
ts.status = status
ts.save()
def run_sql_command(command): def run_sql_command(command):
connection = psycopg2.connect( connection = psycopg2.connect(
DB_URL DB_URL

26
service/views.py
Unescape View File

@ -14,7 +14,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet
from postamates.settings import AGE_DAY_BORDER from postamates.settings import AGE_DAY_BORDER
from postamates.settings import EXCEL_EXPORT_FILENAME from postamates.settings import EXCEL_EXPORT_FILENAME
from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME from postamates.settings import JSON_EXPORT_FILENAME, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT
from service import models from service import models
from service import pagination from service import pagination
from service import serializers from service import serializers
@ -22,13 +22,13 @@ from service import utils
from service.enums import PointStatus, MatchingStatus from service.enums import PointStatus, MatchingStatus
from service.permissions import UserPermission from service.permissions import UserPermission
from service.service import PointService from service.service import PointService
from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data, import_task
from rest_framework.permissions import AllowAny from rest_framework.permissions import AllowAny
from django.shortcuts import redirect from django.shortcuts import redirect
from django.contrib import messages from django.contrib import messages
from django_filters.rest_framework import DjangoFilterBackend from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters from rest_framework import filters
from service.utils import CustomReadOnlyModelViewSet from service.utils import CustomReadOnlyModelViewSet, run_psql_command
from django.db.models import Min, Max from django.db.models import Min, Max
import os import os
from django.forms.models import model_to_dict from django.forms.models import model_to_dict
@ -279,6 +279,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
return Response({'message': 'No status'}, HTTPStatus.BAD_REQUEST) return Response({'message': 'No status'}, HTTPStatus.BAD_REQUEST)
PointService.update_points_in_radius(qs, new_status) PointService.update_points_in_radius(qs, new_status)
PointService.update_status(qs, new_status) PointService.update_status(qs, new_status)
run_psql_command()
return Response( return Response(
{'message': 'status updated'}, {'message': 'status updated'},
status=HTTPStatus.OK, status=HTTPStatus.OK,
@ -379,11 +380,10 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
@action(detail=False, methods=['post']) @action(detail=False, methods=['post'])
def start_matching(self, request): def start_matching(self, request):
file_id = request.POST['id'] file_id = request.POST['id']
total, matched, problem = PointService().start_mathing(file_id) import_task.delay(file_id)
PointService().make_enrichment()
raschet('service_preplacementpoint', need_time=False)
return Response( return Response(
{'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}}, {'message': 'ok'},
# {'message': {'total': total, 'matched': matched, 'error': problem, 'unmatched': total - matched - problem}},
status=HTTPStatus.OK, status=HTTPStatus.OK,
) )
@ -417,6 +417,7 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
PointService.delete_preplacement_points(ids) PointService.delete_preplacement_points(ids)
else: else:
models.PrePlacementPoint.objects.all().delete() models.PrePlacementPoint.objects.all().delete()
run_psql_command()
return Response(status=HTTPStatus.OK, ) return Response(status=HTTPStatus.OK, )
@action(detail=False, methods=['get']) @action(detail=False, methods=['get'])
@ -426,6 +427,16 @@ class PrePlacementPointViewSet(PlacementPointViewSet):
response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx') response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename('preplacementpoints.xlsx')
return response return response
@action(detail=False, methods=['get'])
def import_status(self, request):
status = models.TaskStatus.objects.filter(task_name=STATUS_TASK_NAME_IMPORT).first()
st = None
data = {}
if status:
st = status.status
data = status.data
return Response({'task_status': st, 'data': data}, status=HTTPStatus.OK)
class refresh_placement_points(APIView): class refresh_placement_points(APIView):
@staticmethod @staticmethod
@ -458,7 +469,6 @@ class AvgBiValuesViewSet(APIView):
return Response(data, status=HTTPStatus.OK) return Response(data, status=HTTPStatus.OK)
@api_view(['POST']) @api_view(['POST'])
def upload_post_and_pvz(request): def upload_post_and_pvz(request):
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')

Write Preview
Loading…
Cancel
Save