diff --git a/service/management/commands/create_procedures.py b/service/management/commands/create_procedures.py index f80b587..6ee2b42 100644 --- a/service/management/commands/create_procedures.py +++ b/service/management/commands/create_procedures.py @@ -31,7 +31,7 @@ LANGUAGE plpgsql; """ CMD_PIVOT_DIST_PRE = """CREATE OR REPLACE VIEW compact_preplacementpoint AS -SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpoint; +SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, delta_first, delta_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpoint; CREATE OR REPLACE procedure prepivot_dist() AS $BODY$ DECLARE columnNames TEXT; @@ -52,7 +52,7 @@ FROM CROSSTAB( LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id' ,columnNames); ELSE -CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT placement_point_id, compact_preplacementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpointpvzdistance LEFT JOIN compact_preplacementpoint ON placement_point_id=compact_preplacementpoint.id; +CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT placement_point_id, compact_preplacementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, delta_first, delta_current,doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpointpvzdistance LEFT JOIN compact_preplacementpoint ON placement_point_id=compact_preplacementpoint.id; END IF; END; $BODY$ diff --git a/service/service.py b/service/service.py index 627aa5d..c70ad86 100644 --- a/service/service.py +++ b/service/service.py @@ -5,7 +5,8 @@ import pandas as pd from django.contrib.gis.measure import Distance from django.db.models import F -from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT +from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT, STATUS_TASK_NAME, \ + STATUS_TASK_NAME_IMPORT from service import models from service.enums import PointStatus from service.utils import create_columns_dist, run_psql_command, change_status @@ -42,13 +43,13 @@ class PointService: response = requests.get(req_url).json().get('items') if not response: models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, - status=PointStatus.Pending.name) + status=PointStatus.Pending.name) problem += 1 continue coords = response[0].get('position') if not coords: models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, - status=PointStatus.Pending.name) + status=PointStatus.Pending.name) problem += 1 continue wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")" @@ -70,9 +71,10 @@ class PointService: if o.get('name'): name = o.pop('name') pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, - house_number=house_number,name=name, - matching_status=MatchingStatus.Matched.name, - defaults=o) + house_number=house_number, + name=name, + matching_status=MatchingStatus.Matched.name, + defaults=o) else: pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, house_number=house_number, @@ -150,96 +152,96 @@ class PointService: dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS) ).count() point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean - point.rival_post_cnt = models.Post_and_pvz.objects.filter( - category__name="Постаматы прочих сетей", include_in_ml=True, - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( - category__name="ПВЗ", include_in_ml=True, - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate( - dist=Dist('wkt', origin)).order_by('dist')[0].dist.m - point.property_price_bargains = models.OtherObjects.objects.filter( - group__name="bargains", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[ - 'param1__avg'] - offers_estate = models.OtherObjects.objects.filter( - group__name="offers_estate", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( - param1__avg=Avg('param1'), param3__avg=Avg('param3')) - point.property_price_offers = offers_estate['param1__avg'] - point.property_mean_floor = offers_estate['param3__avg'] - point.property_era = models.OtherObjects.objects.filter( - group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[ - 'param2'] - point.flats_cnt = models.OtherObjects.objects.filter( - group__name="flats_cnt", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( - param1__sum=Sum('param1'))['param1__sum'] - popul_home_job = models.OtherObjects.objects.filter( - group__name="popul_home_job", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( - param1__sum=Sum('param1'), param3__sum=Sum('param3')) - point.popul_home = popul_home_job['param1__sum'] - point.popul_job = popul_home_job['param3__sum'] - yndx_food_cnt_amt = models.OtherObjects.objects.filter( - group__name="yndx_food_cnt_amt", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( - param1__sum=Sum('param1'), param3__sum=Sum('param3')) - point.yndxfood_sum = yndx_food_cnt_amt['param1__sum'] - point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum'] - point.school_cnt = models.OtherObjects.objects.filter( - group__name="schools", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.kindergar_cnt = models.OtherObjects.objects.filter( - group__name="kindergar", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.public_stop_cnt = models.OtherObjects.objects.filter( - group__name="stops", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.sport_center_cnt = models.OtherObjects.objects.filter( - group__name="sport_centers", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.pharmacy_cnt = models.OtherObjects.objects.filter( - group__name="pharmacies", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.supermarket_cnt = models.OtherObjects.objects.filter( - group__name="supermarkets", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.supermarket_premium_cnt = models.OtherObjects.objects.filter( - group__name="supermarkets_premium", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.clinic_cnt = models.OtherObjects.objects.filter( - group__name="clinics", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.bank_cnt = models.OtherObjects.objects.filter( - group__name="banks", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.reca_cnt = models.OtherObjects.objects.filter( - group__name="recas", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.lab_cnt = models.OtherObjects.objects.filter( - group__name="labs", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.culture_cnt = models.OtherObjects.objects.filter( - group__name="cultures", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.attraction_cnt = models.OtherObjects.objects.filter( - group__name="attractions", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.mfc_cnt = models.OtherObjects.objects.filter( - group__name="public_services", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.bc_cnt = models.OtherObjects.objects.filter( - group__name="BC", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.tc_cnt = models.OtherObjects.objects.filter( - group__name="TC", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() - point.business_activity = models.OtherObjects.objects.filter( - group__name="business_activity", - wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( - param1__sum=Sum('param1'))['param1__sum'] - point.age_day = AGE_DAY_LIMIT + # point.rival_post_cnt = models.Post_and_pvz.objects.filter( + # category__name="Постаматы прочих сетей", include_in_ml=True, + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( + # category__name="ПВЗ", include_in_ml=True, + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate( + # dist=Dist('wkt', origin)).order_by('dist')[0].dist.m + # point.property_price_bargains = models.OtherObjects.objects.filter( + # group__name="bargains", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[ + # 'param1__avg'] + # offers_estate = models.OtherObjects.objects.filter( + # group__name="offers_estate", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + # param1__avg=Avg('param1'), param3__avg=Avg('param3')) + # point.property_price_offers = offers_estate['param1__avg'] + # point.property_mean_floor = offers_estate['param3__avg'] + # point.property_era = models.OtherObjects.objects.filter( + # group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[ + # 'param2'] + # point.flats_cnt = models.OtherObjects.objects.filter( + # group__name="flats_cnt", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + # param1__sum=Sum('param1'))['param1__sum'] + # popul_home_job = models.OtherObjects.objects.filter( + # group__name="popul_home_job", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + # param1__sum=Sum('param1'), param3__sum=Sum('param3')) + # point.popul_home = popul_home_job['param1__sum'] + # point.popul_job = popul_home_job['param3__sum'] + # yndx_food_cnt_amt = models.OtherObjects.objects.filter( + # group__name="yndx_food_cnt_amt", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + # param1__sum=Sum('param1'), param3__sum=Sum('param3')) + # point.yndxfood_sum = yndx_food_cnt_amt['param1__sum'] + # point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum'] + # point.school_cnt = models.OtherObjects.objects.filter( + # group__name="schools", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.kindergar_cnt = models.OtherObjects.objects.filter( + # group__name="kindergar", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.public_stop_cnt = models.OtherObjects.objects.filter( + # group__name="stops", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.sport_center_cnt = models.OtherObjects.objects.filter( + # group__name="sport_centers", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.pharmacy_cnt = models.OtherObjects.objects.filter( + # group__name="pharmacies", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.supermarket_cnt = models.OtherObjects.objects.filter( + # group__name="supermarkets", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.supermarket_premium_cnt = models.OtherObjects.objects.filter( + # group__name="supermarkets_premium", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.clinic_cnt = models.OtherObjects.objects.filter( + # group__name="clinics", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.bank_cnt = models.OtherObjects.objects.filter( + # group__name="banks", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.reca_cnt = models.OtherObjects.objects.filter( + # group__name="recas", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.lab_cnt = models.OtherObjects.objects.filter( + # group__name="labs", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.culture_cnt = models.OtherObjects.objects.filter( + # group__name="cultures", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.attraction_cnt = models.OtherObjects.objects.filter( + # group__name="attractions", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.mfc_cnt = models.OtherObjects.objects.filter( + # group__name="public_services", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.bc_cnt = models.OtherObjects.objects.filter( + # group__name="BC", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.tc_cnt = models.OtherObjects.objects.filter( + # group__name="TC", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() + # point.business_activity = models.OtherObjects.objects.filter( + # group__name="business_activity", + # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( + # param1__sum=Sum('param1'))['param1__sum'] + # point.age_day = AGE_DAY_LIMIT placement_point = models.PlacementPoint.objects.annotate( dist=Dist('geometry', origin)).order_by('dist') if placement_point: @@ -248,13 +250,16 @@ class PointService: point.save() for group in groups: self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance) - change_status(f'Шаг 2. Обогащено {points.filter(matching_status=MatchingStatus.Matched.name).count()} из {points.count()}', task_name) + change_status( + f'Шаг 2. Обогащено {points.filter(matching_status=MatchingStatus.Matched.name).count()} из {points.count()}', + task_name) run_psql_command() @staticmethod def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance): - post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( + post_object = models.Post_and_pvz.objects.filter(group__name=group.name, + group__category=group.category).annotate( distance=Dist("wkt", point.geometry)).order_by('distance').first() d = instance_type.objects.filter(placement_point=point, pvz_postamates_group=group).first() @@ -315,7 +320,8 @@ class PointService: data = pd.DataFrame(serializer.data) if not data.empty: if data['start_date'].any(): - data['start_date'] = data['start_date'].apply(lambda x : pd.to_datetime(x).tz_localize(None) if x else None) + data['start_date'] = data['start_date'].apply( + lambda x: pd.to_datetime(x).tz_localize(None) if x else None) if data['sample_trn'].any(): data['sample_trn'] = data['sample_trn'].astype(int) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) diff --git a/service/tasks.py b/service/tasks.py index 9e5a9b1..935afe6 100644 --- a/service/tasks.py +++ b/service/tasks.py @@ -424,7 +424,7 @@ def load_post_and_pvz(obj_id: int): for _j, point in enumerate(points): status.status = "Подсчет расстояний: " + str(int((num_points * _i + _j) / total * 100)) + "%" status.save() - post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( + post_object = models.Post_and_pvz.objects.filter(group__name=group.name,group__category=group.category).annotate( distance=Distance("wkt", point.geometry)).order_by('distance').first() d = models.PlacementPointPVZDistance.objects.filter(placement_point=point, pvz_postamates_group=group).first()