TRUNK
/
postnet
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'fix_2510' into 'dev'

Browse Source 
fix_2510

See merge request spatial/postamates!188
dev
Aleksandr Popov 2 years ago
parent 29fabd58e1 9fba7cd20f
commit 6197e82cc5
3 changed files with 119 additions and 105 deletions
Show Stats Download Patch File Download Diff File

2
docker-compose.yml
Unescape View File

@ -31,7 +31,7 @@ services:
python manage.py collectstatic --noinput && python manage.py collectstatic --noinput &&
python manage.py loaddata fixtures/groups.json && python manage.py loaddata fixtures/groups.json &&
python manage.py loaddata fixtures/post_and_pvz.json && python manage.py loaddata fixtures/post_and_pvz.json &&
python manage.py loaddata fixtures/post_and_pvz_groups.json && python manage.py loaddata fixtures/post_pvz_groups.json &&
python manage.py loaddata fixtures/otherobjectscategorys.json && python manage.py loaddata fixtures/otherobjectscategorys.json &&
python manage.py loaddata fixtures/otherobjectsgroups.json && python manage.py loaddata fixtures/otherobjectsgroups.json &&
python manage.py runserver 0.0.0.0:${DJANGO_PORT}" python manage.py runserver 0.0.0.0:${DJANGO_PORT}"

194
service/service.py
Unescape View File

@ -152,96 +152,107 @@ class PointService:
dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS) dist__lt=Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS)
).count() ).count()
point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean point.target_cnt_ao_mean = qs[0].target_cnt_ao_mean
# point.rival_post_cnt = models.Post_and_pvz.objects.filter( point.rival_post_cnt = models.Post_and_pvz.objects.filter(
# category__name="Постаматы прочих сетей", include_in_ml=True, category__name="Постаматы прочих сетей", include_in_ml=True,
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.rival_pvz_cnt = models.Post_and_pvz.objects.filter( point.rival_pvz_cnt = models.Post_and_pvz.objects.filter(
# category__name="ПВЗ", include_in_ml=True, category__name="ПВЗ", include_in_ml=True,
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.metro_dist = models.OtherObjects.objects.filter(group__name='metro_stations').annotate( metro = models.OtherObjects.objects.filter(group__name='metro_stations').annotate(
# dist=Dist('wkt', origin)).order_by('dist')[0].dist.m dist=Dist('wkt', origin)).order_by('dist')
# point.property_price_bargains = models.OtherObjects.objects.filter( if metro:
# group__name="bargains", point.metro_dist = metro[0].dist.m
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))[ bargains = models.OtherObjects.objects.filter(
# 'param1__avg'] group__name="bargains",
# offers_estate = models.OtherObjects.objects.filter( wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(Avg('param1'))
# group__name="offers_estate", if bargains:
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( point.property_price_bargains = bargains['param1__avg']
# param1__avg=Avg('param1'), param3__avg=Avg('param3')) offers_estate = models.OtherObjects.objects.filter(
# point.property_price_offers = offers_estate['param1__avg'] group__name="offers_estate",
# point.property_mean_floor = offers_estate['param3__avg'] wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
# point.property_era = models.OtherObjects.objects.filter( param1__avg=Avg('param1'), param3__avg=Avg('param3'))
# group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()[ if offers_estate:
# 'param2'] point.property_price_offers = offers_estate['param1__avg']
# point.flats_cnt = models.OtherObjects.objects.filter( point.property_mean_floor = offers_estate['param3__avg']
# group__name="flats_cnt", offers_estate = models.OtherObjects.objects.filter(
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( group__name="offers_estate").values('param2').annotate(cnt=Count('param2')).order_by('-cnt').first()
# param1__sum=Sum('param1'))['param1__sum'] if offers_estate:
# popul_home_job = models.OtherObjects.objects.filter( point.property_era = offers_estate['param2']
# group__name="popul_home_job", flats_cnt = models.OtherObjects.objects.filter(
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( group__name="flats_cnt",
# param1__sum=Sum('param1'), param3__sum=Sum('param3')) wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
# point.popul_home = popul_home_job['param1__sum'] param1__sum=Sum('param1'))
# point.popul_job = popul_home_job['param3__sum'] if flats_cnt:
# yndx_food_cnt_amt = models.OtherObjects.objects.filter( point.flats_cnt = flats_cnt['param1__sum']
# group__name="yndx_food_cnt_amt", popul_home_job = models.OtherObjects.objects.filter(
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( group__name="popul_home_job",
# param1__sum=Sum('param1'), param3__sum=Sum('param3')) wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
# point.yndxfood_sum = yndx_food_cnt_amt['param1__sum'] param1__sum=Sum('param1'), param3__sum=Sum('param3'))
# point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum'] if popul_home_job:
# point.school_cnt = models.OtherObjects.objects.filter( point.popul_home = popul_home_job['param1__sum']
# group__name="schools", point.popul_job = popul_home_job['param3__sum']
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() yndx_food_cnt_amt = models.OtherObjects.objects.filter(
# point.kindergar_cnt = models.OtherObjects.objects.filter( group__name="yndx_food_cnt_amt",
# group__name="kindergar", wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() param1__sum=Sum('param1'), param3__sum=Sum('param3'))
# point.public_stop_cnt = models.OtherObjects.objects.filter( if yndx_food_cnt_amt:
# group__name="stops", point.yndxfood_sum = yndx_food_cnt_amt['param1__sum']
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() point.yndxfood_cnt = yndx_food_cnt_amt['param3__sum']
# point.sport_center_cnt = models.OtherObjects.objects.filter( point.school_cnt = models.OtherObjects.objects.filter(
# group__name="sport_centers", group__name="schools",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.pharmacy_cnt = models.OtherObjects.objects.filter( point.kindergar_cnt = models.OtherObjects.objects.filter(
# group__name="pharmacies", group__name="kindergar",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.supermarket_cnt = models.OtherObjects.objects.filter( point.public_stop_cnt = models.OtherObjects.objects.filter(
# group__name="supermarkets", group__name="stops",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.supermarket_premium_cnt = models.OtherObjects.objects.filter( point.sport_center_cnt = models.OtherObjects.objects.filter(
# group__name="supermarkets_premium", group__name="sport_centers",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.clinic_cnt = models.OtherObjects.objects.filter( point.pharmacy_cnt = models.OtherObjects.objects.filter(
# group__name="clinics", group__name="pharmacies",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.bank_cnt = models.OtherObjects.objects.filter( point.supermarket_cnt = models.OtherObjects.objects.filter(
# group__name="banks", group__name="supermarkets",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.reca_cnt = models.OtherObjects.objects.filter( point.supermarket_premium_cnt = models.OtherObjects.objects.filter(
# group__name="recas", group__name="supermarkets_premium",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.lab_cnt = models.OtherObjects.objects.filter( point.clinic_cnt = models.OtherObjects.objects.filter(
# group__name="labs", group__name="clinics",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.culture_cnt = models.OtherObjects.objects.filter( point.bank_cnt = models.OtherObjects.objects.filter(
# group__name="cultures", group__name="banks",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.attraction_cnt = models.OtherObjects.objects.filter( point.reca_cnt = models.OtherObjects.objects.filter(
# group__name="attractions", group__name="recas",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.mfc_cnt = models.OtherObjects.objects.filter( point.lab_cnt = models.OtherObjects.objects.filter(
# group__name="public_services", group__name="labs",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.bc_cnt = models.OtherObjects.objects.filter( point.culture_cnt = models.OtherObjects.objects.filter(
# group__name="BC", group__name="cultures",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.tc_cnt = models.OtherObjects.objects.filter( point.attraction_cnt = models.OtherObjects.objects.filter(
# group__name="TC", group__name="attractions",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count() wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# point.business_activity = models.OtherObjects.objects.filter( point.mfc_cnt = models.OtherObjects.objects.filter(
# group__name="business_activity", group__name="public_services",
# wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate( wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
# param1__sum=Sum('param1'))['param1__sum'] point.bc_cnt = models.OtherObjects.objects.filter(
# point.age_day = AGE_DAY_LIMIT group__name="BC",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
point.tc_cnt = models.OtherObjects.objects.filter(
group__name="TC",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).count()
business_activity = models.OtherObjects.objects.filter(
group__name="business_activity",
wkt__distance_lt=(origin, Distance(m=DEFAULT_PLACEMENT_POINT_UPDATE_RADIUS))).aggregate(
param1__sum=Sum('param1'))
if business_activity:
point.business_activity = business_activity['param1__sum']
point.age_day = AGE_DAY_LIMIT
placement_point = models.PlacementPoint.objects.annotate( placement_point = models.PlacementPoint.objects.annotate(
dist=Dist('geometry', origin)).order_by('dist') dist=Dist('geometry', origin)).order_by('dist')
if placement_point: if placement_point:
@ -258,8 +269,7 @@ class PointService:
@staticmethod @staticmethod
def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance): def calculate_dist_for_group(point, group, instance_type=models.PlacementPointPVZDistance):
post_object = models.Post_and_pvz.objects.filter(group__name=group.name, post_object = models.Post_and_pvz.objects.filter(group__id=group.id).annotate(
group__category=group.category).annotate(
distance=Dist("wkt", point.geometry)).order_by('distance').first() distance=Dist("wkt", point.geometry)).order_by('distance').first()
d = instance_type.objects.filter(placement_point=point, d = instance_type.objects.filter(placement_point=point,
pvz_postamates_group=group).first() pvz_postamates_group=group).first()

28
service/tasks.py
Unescape View File

@ -115,7 +115,6 @@ def raschet(table_name='service_placementpoint', need_time=True, task_name=STATU
X_trn = pts_trn[feats].drop(columns=['id']) X_trn = pts_trn[feats].drop(columns=['id'])
Y_trn = pts_trn[['fact']] Y_trn = pts_trn[['fact']]
# status.status = 'Записи для инференса' # status.status = 'Записи для инференса'
# status.save() # status.save()
change_status('Записи для инференса', task_name=task_name) change_status('Записи для инференса', task_name=task_name)
@ -147,19 +146,23 @@ def raschet(table_name='service_placementpoint', need_time=True, task_name=STATU
target_feature_coords = np.array(target_feature_coords) target_feature_coords = np.array(target_feature_coords)
pts_inf['target_dist'] = pts_inf.apply( pts_inf['target_dist'] = pts_inf.apply(
lambda x: ((sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[1]) if ((x.status == 'Working') or (x.status == 'Installation')) else lambda x: (
(sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[1]) if (
(x.status == 'Working') or (x.status == 'Installation')) else
(sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])), (sorted(distance.cdist([[x['geometry'].x, x['geometry'].y]], target_feature_coords)[0])[0])),
axis=1, axis=1,
) )
pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700 pts_inf.loc[pts_inf.target_dist > 700, 'target_dist'] = 700
pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True) pts_inf = pts_inf.sort_values(by='id').reset_index(drop=True)
target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg({'cnt': 'count'}) target_post = gpd.sjoin(pts_inf, pts_target, op='contains').groupby('id', as_index=False).agg(
{'cnt': 'count'})
target_post = target_post.rename(columns={'cnt': 'target_post_cnt'}) target_post = target_post.rename(columns={'cnt': 'target_post_cnt'})
pts_inf = pts_inf.drop(columns=['target_post_cnt']) pts_inf = pts_inf.drop(columns=['target_post_cnt'])
pts_inf = pts_inf.join(target_post.set_index('id'), on='id') pts_inf = pts_inf.join(target_post.set_index('id'), on='id')
pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0) pts_inf['target_post_cnt'] = pts_inf['target_post_cnt'].fillna(0)
pts_inf['target_post_cnt'] = pts_inf.apply(lambda x: ((x.target_post_cnt - 1) if ((x.status == 'Working') or (x.status == 'Installation')) else x.target_post_cnt), axis=1) pts_inf['target_post_cnt'] = pts_inf.apply(lambda x: ((x.target_post_cnt - 1) if (
(x.status == 'Working') or (x.status == 'Installation')) else x.target_post_cnt), axis=1)
pts_inf['age_day_init'] = pts_inf['age_day'] pts_inf['age_day_init'] = pts_inf['age_day']
pts_inf['age_day'] = 240 pts_inf['age_day'] = 240
X_inf = pts_inf[feats] X_inf = pts_inf[feats]
@ -176,7 +179,8 @@ def raschet(table_name='service_placementpoint', need_time=True, task_name=STATU
for i in seeds: for i in seeds:
# status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%' # status.status = 'Обучение inference: ' + str(int((seeds.index(i) + 1) / len(seeds) * 100)) + '%'
# status.save() # status.save()
change_status(f'Обучение inference: {str(int((seeds.index(i) + 1) / len(seeds) * 100))}%', task_name=task_name) change_status(f'Обучение inference: {str(int((seeds.index(i) + 1) / len(seeds) * 100))}%',
task_name=task_name)
x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i) x_trn, x_test, y_trn, y_test = ms.train_test_split(X_trn, Y_trn, test_size=0.2, random_state=i)
model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i) model = catboost.CatBoostRegressor(cat_features=['property_era'], random_state=i)
model.fit(x_trn, y_trn, verbose=False) model.fit(x_trn, y_trn, verbose=False)
@ -195,14 +199,15 @@ def raschet(table_name='service_placementpoint', need_time=True, task_name=STATU
shap_values = explainer(X_inf.drop(columns=['id'])) shap_values = explainer(X_inf.drop(columns=['id']))
shap_fields = pd.DataFrame(shap_values.values) shap_fields = pd.DataFrame(shap_values.values)
shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap' shap_fields.columns = X_inf.drop(columns=['id']).columns + '_shap'
shap_fields = shap_fields.drop(columns = ['age_day_shap']) shap_fields = shap_fields.drop(columns=['age_day_shap'])
shap_fields['sum'] = abs(shap_fields).sum(axis=1) shap_fields['sum'] = abs(shap_fields).sum(axis=1)
shap_fields = round(shap_fields.iloc[:,:32].div(shap_fields['sum'], axis=0)*100, 2) shap_fields = round(shap_fields.iloc[:, :32].div(shap_fields['sum'], axis=0) * 100, 2)
# Обновление полей по результатам работы модели # Обновление полей по результатам работы модели
update_fields = pts_inf[ update_fields = pts_inf[
[ [
'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current', 'plan_first', 'id', 'age_day_init', 'status', 'fact', 'delta_current', 'delta_first', 'plan_current',
'plan_first',
'prediction_first', 'target_post_cnt', 'target_dist' 'prediction_first', 'target_post_cnt', 'target_dist'
] ]
] ]
@ -227,7 +232,8 @@ def raschet(table_name='service_placementpoint', need_time=True, task_name=STATU
axis=1, axis=1,
) )
update_fields['plan_current'] = update_fields.apply( update_fields['plan_current'] = update_fields.apply(
lambda x: (x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0), lambda x: (
x.prediction_current * interpolate.splev(x.age_day_init, spl) if x.status == 'Working' else 0),
axis=1, axis=1,
) )
update_fields['delta_first'] = update_fields.apply( update_fields['delta_first'] = update_fields.apply(
@ -387,7 +393,6 @@ def raschet(table_name='service_placementpoint', need_time=True, task_name=STATU
LastMLCall.objects.create() LastMLCall.objects.create()
@shared_task @shared_task
def load_post_and_pvz(obj_id: int): def load_post_and_pvz(obj_id: int):
file = models.TempFiles.objects.get(id=obj_id) file = models.TempFiles.objects.get(id=obj_id)
@ -424,7 +429,7 @@ def load_post_and_pvz(obj_id: int):
for _j, point in enumerate(points): for _j, point in enumerate(points):
status.status = "Подсчет расстояний: " + str(int((num_points * _i + _j) / total * 100)) + "%" status.status = "Подсчет расстояний: " + str(int((num_points * _i + _j) / total * 100)) + "%"
status.save() status.save()
post_object = models.Post_and_pvz.objects.filter(group__name=group.name,group__category=group.category).annotate( post_object = models.Post_and_pvz.objects.filter(group__id=group.id).annotate(
distance=Distance("wkt", point.geometry)).order_by('distance').first() distance=Distance("wkt", point.geometry)).order_by('distance').first()
d = models.PlacementPointPVZDistance.objects.filter(placement_point=point, d = models.PlacementPointPVZDistance.objects.filter(placement_point=point,
pvz_postamates_group=group).first() pvz_postamates_group=group).first()
@ -449,7 +454,6 @@ def load_post_and_pvz(obj_id: int):
run_psql_command() run_psql_command()
@shared_task() @shared_task()
def add_age_day(): def add_age_day():
qs = PlacementPoint.objects.filter(status='Working') qs = PlacementPoint.objects.filter(status='Working')

Write Preview
Loading…
Cancel
Save