postnet/service/utils.py

import os

import geojson
import numpy as np
import pandas as pd
from django.contrib.gis.geos import GEOSGeometry
from geojson import MultiPolygon

from service import models
import requests
from tqdm import tqdm
from django.core.cache import cache
from django.conf import settings
from rest_framework.response import Response
from rest_framework.viewsets import ReadOnlyModelViewSet
from django.db.models import Avg, Min, Max

from postamates.settings import STATUS_TASK_NAME, STATUS_TASK_NAME_IMPORT

import psycopg2
from postamates.settings import DB_URL

def change_status(status, task_name=STATUS_TASK_NAME):
    ts, _ = models.TaskStatus.objects.get_or_create(task_name=task_name)
    ts.status = status
    ts.save()


def run_sql_command(command):
    connection = psycopg2.connect(
        DB_URL
    )
    try:
        cursor = connection.cursor()
        cursor.execute(command)
        connection.commit()
    except psycopg2.Error as e:
        print("Error executing command:", e)
    finally:
        cursor.close()
        connection.close()


def run_psql_command():
    connection = psycopg2.connect(
        DB_URL
    )
    try:
        cursor = connection.cursor()
        command = "CALL public.pivot_dist();CALL public.prepivot_dist();REFRESH MATERIALIZED VIEW public.points_with_dist;REFRESH MATERIALIZED VIEW public.prepoints_with_dist;"
        cursor.execute(command)
        connection.commit()
    except psycopg2.Error as e:
        print("Error executing command:", e)
    finally:
        cursor.close()
        connection.close()


def load_ao_and_rayons(
        ao_filepath: str,
        rayons_filepath: str,
):
    models.AO.objects.all().delete()
    models.Rayon.objects.all().delete()
    gj = geojson.load(ao_filepath)
    objs = gj['features']
    for obj in tqdm(objs, desc='Loading AOs...'):
        name = obj['properties']['okrug']
        coords = obj['geometry']['coordinates']
        models.AO.objects.create(**{'name': name, 'polygon': GEOSGeometry(str(MultiPolygon(coords)))})
    gj = geojson.load(rayons_filepath)
    objs = gj['features']
    for obj in tqdm(objs, desc='Loading Rayons...'):
        name = obj['properties']['rayon']
        coords = obj['geometry']['coordinates']
        okr = obj['properties']['okrug']
        ao = models.AO.objects.get(name=okr)
        models.Rayon.objects.create(**{'name': name, 'polygon': GEOSGeometry(str(MultiPolygon(coords))), 'AO': ao})


def load_dist(filepath: str):
    models.PointDist.objects.all().delete()
    df = pd.read_csv(filepath)
    for row in df.to_dict('records'):
        row['id1'] = models.PlacementPoint.objects.get(pk=row.get('id1'))
        row['id2'] = models.PlacementPoint.objects.get(pk=row.get('id2'))
        models.PointDist.objects.create(**row)


def log(msg):
    print(msg)


def cached_func(key, func, timeout=settings.CACHE_TIMEOUT, *args, **kwargs):
    d = cache.get(key)
    if d is None:
        d = func(*args, **kwargs)
        cache.set(key, d, timeout)
    return d


def get_middle_bi_values():
    fields_to_aggregate = [
        'target_dist_shap',
        'target_post_cnt_shap',
        'target_cnt_ao_mean_shap',
        'rival_pvz_cnt_shap',
        'rival_post_cnt_shap',
        'metro_dist_shap',
        'property_price_bargains_shap',
        'property_price_offers_shap',
        'property_mean_floor_shap',
        'property_era_shap',
        'flats_cnt_shap',
        'popul_home_shap',
        'popul_job_shap',
        'yndxfood_sum_shap',
        'yndxfood_cnt_shap',
        'school_cnt_shap',
        'kindergar_cnt_shap',
        'public_stop_cnt_shap',
        'sport_center_cnt_shap',
        'pharmacy_cnt_shap',
        'supermarket_cnt_shap',
        'supermarket_premium_cnt_shap',
        'clinic_cnt_shap',
        'bank_cnt_shap',
        'reca_cnt_shap',
        'lab_cnt_shap',
        'culture_cnt_shap',
        'attraction_cnt_shap',
        'mfc_cnt_shap',
        'bc_cnt_shap',
        'tc_cnt_shap',
        'business_activity_shap'
    ]

    aggregations = {}
    for field_name in fields_to_aggregate:
        aggregations[f'avg_{field_name}'] = Avg(field_name)
        aggregations[f'min_{field_name}'] = Min(field_name)
        aggregations[f'max_{field_name}'] = Max(field_name)
    result = models.PlacementPoint.objects.aggregate(**aggregations)
    return result


class CustomReadOnlyModelViewSet(ReadOnlyModelViewSet):
    def list(self, request, *args, **kwargs):
        def f():
            return ReadOnlyModelViewSet.list(self, request, *args, **kwargs).data

        d = cached_func(self.__class__.__name__, f)
        return Response(d)


def create_columns_dist(row):
    return pd.Series(row['min_distance_to_group'])


def load_houses(filepath: str):
    models.House.objects.all().delete()
    df = pd.read_csv(filepath)
    df = df.replace(np.nan, None)
    df = df.replace('NaT', None)
    for row in df.to_dict('records'):
        models.House.objects.create(**row)