TRUNK
/
postnet
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix_dist_calc

Browse Source
dev
AlexP077 2 years ago
parent c4f78d0bc7
commit 4f102b754b
3 changed files with 108 additions and 102 deletions
Show Stats Download Patch File Download Diff File

4
service/management/commands/create_procedures.py
Unescape View File

@ -31,7 +31,7 @@ LANGUAGE plpgsql;
""" """
CMD_PIVOT_DIST_PRE = """CREATE OR REPLACE VIEW compact_preplacementpoint AS CMD_PIVOT_DIST_PRE = """CREATE OR REPLACE VIEW compact_preplacementpoint AS
SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpoint; SELECT id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, delta_first, delta_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpoint;
CREATE OR REPLACE procedure prepivot_dist() CREATE OR REPLACE procedure prepivot_dist()
AS $BODY$ AS $BODY$
DECLARE columnNames TEXT; DECLARE columnNames TEXT;
@ -52,7 +52,7 @@ FROM CROSSTAB(
LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id' LEFT JOIN compact_preplacementpoint ON preplacement_point_id=id'
,columnNames); ,columnNames);
ELSE ELSE
CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT placement_point_id, compact_preplacementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpointpvzdistance LEFT JOIN compact_preplacementpoint ON placement_point_id=compact_preplacementpoint.id; CREATE MATERIALIZED VIEW prepoints_with_dist AS SELECT placement_point_id, compact_preplacementpoint.id, status, category, age_day, fact, area_id, district_id, prediction_first, prediction_current, delta_first, delta_current,doors, flat_cnt, rival_post_cnt, rival_pvz_cnt, target_post_cnt, flats_cnt, tc_cnt, culture_cnt, mfc_cnt, public_stop_cnt, supermarket_cnt, target_dist, metro_dist, geometry FROM service_preplacementpointpvzdistance LEFT JOIN compact_preplacementpoint ON placement_point_id=compact_preplacementpoint.id;
END IF; END IF;
END; END;
$BODY$ $BODY$

204
service/service.py
Unescape View File

@ -5,7 +5,8 @@ import pandas as pd
from django.contrib.gis.measure import Distance from django.contrib.gis.measure import Distance
from django.db.models import F from django.db.models import F
from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT, STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT from postamates.settings import DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS, AGE_DAY_LIMIT, STATUS_TASK_NAME, \
STATUS_TASK_NAME_IMPORT
from service import models from service import models
from service.enums import PointStatus from service.enums import PointStatus
from service.utils import create_columns_dist, run_psql_command, change_status from service.utils import create_columns_dist, run_psql_command, change_status
@ -42,13 +43,13 @@ class PointService:
response = requests.get(req_url).json().get('items') response = requests.get(req_url).json().get('items')
if not response: if not response:
models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
status=PointStatus.Pending.name) status=PointStatus.Pending.name)
problem += 1 problem += 1
continue continue
coords = response[0].get('position') coords = response[0].get('position')
if not coords: if not coords:
models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name, models.PrePlacementPoint.objects.get_or_create(address=addr, matching_status=MatchingStatus.Error.name,
status=PointStatus.Pending.name) status=PointStatus.Pending.name)
problem += 1 problem += 1
continue continue
wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")" wkt = "POINT(" + str(coords['lng']) + " " + str(coords['lat']) + ")"
@ -70,9 +71,10 @@ class PointService:
if o.get('name'): if o.get('name'):
name = o.pop('name') name = o.pop('name')
pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
house_number=house_number,name=name, house_number=house_number,
matching_status=MatchingStatus.Matched.name, name=name,
defaults=o) matching_status=MatchingStatus.Matched.name,
defaults=o)
else: else:
pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street, pre_obj, _ = models.PrePlacementPoint.objects.get_or_create(street=street,
house_number=house_number, house_number=house_number,
@ -150,96 +152,96 @@ class PointService:
dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS) dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)
).count() ).count()
point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean
point.rival_post_cnt = models.Post_and_pvz.objects.filter( # point.rival_post_cnt = models.Post_and_pvz.objects.filter(
category__name="Постаматы прочих сетей", include_in_ml=True, # category__name="Постаматы прочих сетей", include_in_ml=True,
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( # point.rival_pvz_cnt = models.Post_and_pvz.objects.filter(
category__name="ПВЗ", include_in_ml=True, # category__name="ПВЗ", include_in_ml=True,
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate( # point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate(
dist=Dist('wkt', origin)).order_by('dist')[0].dist.m # dist=Dist('wkt', origin)).order_by('dist')[0].dist.m
point.property_price_bargains = models.OtherObjects.objects.filter( # point.property_price_bargains = models.OtherObjects.objects.filter(
group__name="bargains", # group__name="bargains",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[ # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[
'param1__avg'] # 'param1__avg']
offers_estate = models.OtherObjects.objects.filter( # offers_estate = models.OtherObjects.objects.filter(
group__name="offers_estate", # group__name="offers_estate",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__avg=Avg('param1'), param3__avg=Avg('param3')) # param1__avg=Avg('param1'), param3__avg=Avg('param3'))
point.property_price_offers = offers_estate['param1__avg'] # point.property_price_offers = offers_estate['param1__avg']
point.property_mean_floor = offers_estate['param3__avg'] # point.property_mean_floor = offers_estate['param3__avg']
point.property_era = models.OtherObjects.objects.filter( # point.property_era = models.OtherObjects.objects.filter(
group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[ # group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[
'param2'] # 'param2']
point.flats_cnt = models.OtherObjects.objects.filter( # point.flats_cnt = models.OtherObjects.objects.filter(
group__name="flats_cnt", # group__name="flats_cnt",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'))['param1__sum'] # param1__sum=Sum('param1'))['param1__sum']
popul_home_job = models.OtherObjects.objects.filter( # popul_home_job = models.OtherObjects.objects.filter(
group__name="popul_home_job", # group__name="popul_home_job",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'), param3__sum=Sum('param3')) # param1__sum=Sum('param1'), param3__sum=Sum('param3'))
point.popul_home = popul_home_job['param1__sum'] # point.popul_home = popul_home_job['param1__sum']
point.popul_job = popul_home_job['param3__sum'] # point.popul_job = popul_home_job['param3__sum']
yndx_food_cnt_amt = models.OtherObjects.objects.filter( # yndx_food_cnt_amt = models.OtherObjects.objects.filter(
group__name="yndx_food_cnt_amt", # group__name="yndx_food_cnt_amt",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'), param3__sum=Sum('param3')) # param1__sum=Sum('param1'), param3__sum=Sum('param3'))
point.yndxfood_sum = yndx_food_cnt_amt['param1__sum'] # point.yndxfood_sum = yndx_food_cnt_amt['param1__sum']
point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum'] # point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum']
point.school_cnt = models.OtherObjects.objects.filter( # point.school_cnt = models.OtherObjects.objects.filter(
group__name="schools", # group__name="schools",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.kindergar_cnt = models.OtherObjects.objects.filter( # point.kindergar_cnt = models.OtherObjects.objects.filter(
group__name="kindergar", # group__name="kindergar",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.public_stop_cnt = models.OtherObjects.objects.filter( # point.public_stop_cnt = models.OtherObjects.objects.filter(
group__name="stops", # group__name="stops",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.sport_center_cnt = models.OtherObjects.objects.filter( # point.sport_center_cnt = models.OtherObjects.objects.filter(
group__name="sport_centers", # group__name="sport_centers",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.pharmacy_cnt = models.OtherObjects.objects.filter( # point.pharmacy_cnt = models.OtherObjects.objects.filter(
group__name="pharmacies", # group__name="pharmacies",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.supermarket_cnt = models.OtherObjects.objects.filter( # point.supermarket_cnt = models.OtherObjects.objects.filter(
group__name="supermarkets", # group__name="supermarkets",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.supermarket_premium_cnt = models.OtherObjects.objects.filter( # point.supermarket_premium_cnt = models.OtherObjects.objects.filter(
group__name="supermarkets_premium", # group__name="supermarkets_premium",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.clinic_cnt = models.OtherObjects.objects.filter( # point.clinic_cnt = models.OtherObjects.objects.filter(
group__name="clinics", # group__name="clinics",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.bank_cnt = models.OtherObjects.objects.filter( # point.bank_cnt = models.OtherObjects.objects.filter(
group__name="banks", # group__name="banks",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.reca_cnt = models.OtherObjects.objects.filter( # point.reca_cnt = models.OtherObjects.objects.filter(
group__name="recas", # group__name="recas",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.lab_cnt = models.OtherObjects.objects.filter( # point.lab_cnt = models.OtherObjects.objects.filter(
group__name="labs", # group__name="labs",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.culture_cnt = models.OtherObjects.objects.filter( # point.culture_cnt = models.OtherObjects.objects.filter(
group__name="cultures", # group__name="cultures",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.attraction_cnt = models.OtherObjects.objects.filter( # point.attraction_cnt = models.OtherObjects.objects.filter(
group__name="attractions", # group__name="attractions",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.mfc_cnt = models.OtherObjects.objects.filter( # point.mfc_cnt = models.OtherObjects.objects.filter(
group__name="public_services", # group__name="public_services",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.bc_cnt = models.OtherObjects.objects.filter( # point.bc_cnt = models.OtherObjects.objects.filter(
group__name="BC", # group__name="BC",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.tc_cnt = models.OtherObjects.objects.filter( # point.tc_cnt = models.OtherObjects.objects.filter(
group__name="TC", # group__name="TC",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.business_activity = models.OtherObjects.objects.filter( # point.business_activity = models.OtherObjects.objects.filter(
group__name="business_activity", # group__name="business_activity",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( # wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'))['param1__sum'] # param1__sum=Sum('param1'))['param1__sum']
point.age_day = AGE_DAY_LIMIT # point.age_day = AGE_DAY_LIMIT
placement_point = models.PlacementPoint.objects.annotate( placement_point = models.PlacementPoint.objects.annotate(
dist=Dist('geometry', origin)).order_by('dist') dist=Dist('geometry', origin)).order_by('dist')
if placement_point: if placement_point:
@ -248,13 +250,16 @@ class PointService:
point.save() point.save()
for group in groups: for group in groups:
self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance) self.calculate_dist_for_group(point, group, instance_type=models.PrePlacementPointPVZDistance)
change_status(f'Шаг 2. Обогащено {points.filter(matching_status=MatchingStatus.Matched.name).count()} из {points.count()}', task_name) change_status(
f'Шаг 2. Обогащено {points.filter(matching_status=MatchingStatus.Matched.name).count()} из {points.count()}',
task_name)
run_psql_command() run_psql_command()
@staticmethod @staticmethod
def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance): def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance):
post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( post_object = models.Post_and_pvz.objects.filter(group__name=group.name,
group__category=group.category).annotate(
distance=Dist("wkt", point.geometry)).order_by('distance').first() distance=Dist("wkt", point.geometry)).order_by('distance').first()
d = instance_type.objects.filter(placement_point=point, d = instance_type.objects.filter(placement_point=point,
pvz_postamates_group=group).first() pvz_postamates_group=group).first()
@ -315,7 +320,8 @@ class PointService:
data = pd.DataFrame(serializer.data) data = pd.DataFrame(serializer.data)
if not data.empty: if not data.empty:
if data['start_date'].any(): if data['start_date'].any():
data['start_date'] = data['start_date'].apply(lambda x : pd.to_datetime(x).tz_localize(None) if x else None) data['start_date'] = data['start_date'].apply(
lambda x: pd.to_datetime(x).tz_localize(None) if x else None)
if data['sample_trn'].any(): if data['sample_trn'].any():
data['sample_trn'] = data['sample_trn'].astype(int) data['sample_trn'] = data['sample_trn'].astype(int)
data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True) data.rename(columns={'district_id': 'district', 'area_id': 'area'}, inplace=True)

2
service/tasks.py
Unescape View File

@ -424,7 +424,7 @@ def load_post_and_pvz(obj_id: int):
for _j, point in enumerate(points): for _j, point in enumerate(points):
status.status = "Подсчет расстояний: " + str(int((num_points * _i + _j) / total * 100)) + "%" status.status = "Подсчет расстояний: " + str(int((num_points * _i + _j) / total * 100)) + "%"
status.save() status.save()
post_object = models.Post_and_pvz.objects.filter(group__name=group.name).annotate( post_object = models.Post_and_pvz.objects.filter(group__name=group.name,group__category=group.category).annotate(
distance=Distance("wkt", point.geometry)).order_by('distance').first() distance=Distance("wkt", point.geometry)).order_by('distance').first()
d = models.PlacementPointPVZDistance.objects.filter(placement_point=point, d = models.PlacementPointPVZDistance.objects.filter(placement_point=point,
pvz_postamates_group=group).first() pvz_postamates_group=group).first()

Write Preview
Loading…
Cancel
Save