TRUNK
/
postnet
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

release 0

Browse Source
dev
timofejmalinin 3 years ago
parent 6a8e56237a
commit 870892d09d
21 changed files with 98 additions and 476 deletions
Show Stats Download Patch File Download Diff File

1
ao_and_rayons.json
View File

File diff suppressed because one or more lines are too long

6
postamates/settings.py
Unescape View File

@ -141,3 +141,9 @@ DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
CORS_ORIGIN_ALLOW_ALL = True # If this is used then `CORS_ORIGIN_WHITELIST` will not have any effect
CORS_ALLOW_CREDENTIALS = True
CORS_ORIGIN_ALLOW = True
DB_URL = 'postgresql://postamates_user:postamates_pass@postamates.spatiality.website:5481/postamates_db'
# DB_URL = f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@{os.getenv('POSTGRES_HOST', 'postgres')}:{os.getenv('POSTGRES_PORT', 'postgres')}/{os.getenv('POSTGRES_DB', 'postgres')}"
if os.getenv('local') is not None:
GDAL_LIBRARY_PATH = '/opt/homebrew/opt/gdal/lib/libgdal.dylib'
GEOS_LIBRARY_PATH = '/opt/homebrew/opt/geos/lib/libgeos_c.dylib'

2
postamates_frontend

@ -1 +1 @@
Subproject commit e2c5831c709de7ca53ce9993cb36e23ada7f4ca9
Subproject commit 71c070e3d72fd144f8d9f102a2a8a9a740ff5d0f

8
requirements.txt
Unescape View File

@ -1,3 +1,4 @@
catboost
Django==3.2
djangorestframework==3.11.1
psycopg2-binary==2.9.3
@ -9,4 +10,11 @@ django-cors-headers==3.12.0
pyshp==2.3.0
matplotlib==3.5.2
openpyxl==3.0.10
shapely
pygeos
xlsxwriter
django-json-widget
django-rest-registration
geopandas
sqlalchemy
numpy

9
service/admin.py
Unescape View File

@ -1,12 +1,3 @@
from django.contrib import admin
from service.models import Polygon, Point, Config
from django.db import models
from django_json_widget.widgets import JSONEditorWidget
# admin.site.register(Polygon)
# admin.site.register(Point)
@admin.register(Config)
class YourModelAdmin(admin.ModelAdmin):
formfield_overrides = {
models.JSONField: {'widget': JSONEditorWidget},
}

0
service/management/__init__.py
Unescape View File

1
service/management/commands/__init__.py
Unescape View File

@ -1 +0,0 @@

49
service/management/commands/create_fixture.py
Unescape View File

@ -1,49 +0,0 @@
from django.core.management.base import BaseCommand
import json
from service import models
def create_fixture(name, file_path, merge):
if merge:
with open(file_path, 'r') as f:
old_data = json.load(f)
max_pk = max([x['pk'] for x in old_data])
else:
max_pk = 0
model = getattr(models, name)
model_default_data = {
f.name: {
"label": f.verbose_name,
"legend": f.verbose_name,
"help": f.verbose_name
}
for f in model._meta.fields
}
data = {
"model": "service.config",
"pk": max_pk + 1,
"fields": {
"name": f"{name}",
"data": model_default_data
}
}
with open(file_path, 'w') as f:
if merge:
old_data.append(data)
json.dump(old_data, f, ensure_ascii=False, indent=4)
else:
json.dump([data], f, ensure_ascii=False, indent=4)
class Command(BaseCommand):
requires_system_checks = False
def add_arguments(self, parser):
parser.add_argument('-n', '--name', type=str)
parser.add_argument('-f', '--file', type=str)
parser.add_argument('-m', '--merge', type=bool, default=False)
def handle(self, name, file, merge, *args, **options):
create_fixture(name, file, merge)

33
service/management/commands/create_points.py
Unescape View File

@ -1,33 +0,0 @@
from django.core.management.base import BaseCommand
from service.models import Point
from django.contrib.gis.geos import Point as GeoPoint
from pandas import read_excel
from tqdm import tqdm
def import_points(file_path):
Point.objects.all().delete()
point_models = []
df = read_excel(file_path)
df = df.fillna(0)
df_rows = list(df.iterrows())
for id_, i in tqdm(df_rows):
data = i.to_dict()
lat, lng = data.pop('lat'),data.pop('lng')
geometry = GeoPoint(lng, lat, srid=4326)
try:
point_models.append(Point(point=geometry, **data))
except Exception as e:
print(e)
pass
Point.objects.bulk_create(point_models, batch_size=10000)
class Command(BaseCommand):
requires_system_checks = False
def add_arguments(self, parser):
parser.add_argument('-f', '--file', type=str)
def handle(self, file, *args, **options):
import_points(file)

28
service/management/commands/create_poly.py
Unescape View File

@ -1,28 +0,0 @@
from django.core.management.base import BaseCommand
from service.models import Polygon
import shapefile
from django.contrib.gis.geos import Polygon as GeoPolygon
from tqdm import tqdm
def import_poly(file_path):
Polygon.objects.all().delete()
poly_models = []
shape = shapefile.Reader(file_path)
shape_records = list(shape.shapeRecords())
for i in tqdm(shape_records):
polygon = i.shape.__geo_interface__
data = i.__geo_interface__['properties']
geom = GeoPolygon(polygon['coordinates'][0], srid=4326)
poly_models.append(Polygon(geometry=geom, **data))
Polygon.objects.bulk_create(poly_models, batch_size=10000)
class Command(BaseCommand):
requires_system_checks = False
def add_arguments(self, parser):
parser.add_argument('-f', '--file', type=str)
def handle(self, file, *args, **options):
import_poly(file)

44
service/migrations/0001_initial.py
Unescape View File

@ -1,44 +0,0 @@
# Generated by Django 3.2 on 2022-07-09 16:01
import django.contrib.gis.db.models.fields
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Point',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('point', django.contrib.gis.db.models.fields.PointField(srid=4326)),
('name', models.CharField(blank=True, max_length=256, null=True)),
('adress', models.CharField(blank=True, max_length=512, null=True)),
('district', models.CharField(blank=True, max_length=512, null=True)),
('area', models.FloatField()),
('price', models.IntegerField()),
],
),
migrations.CreateModel(
name='Poly',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('geometry', django.contrib.gis.db.models.fields.PolygonField(srid=4326)),
('cellid', models.IntegerField()),
('home', models.IntegerField()),
('job', models.IntegerField()),
('horeca', models.IntegerField()),
('brand', models.IntegerField()),
('univer', models.IntegerField()),
('hotel', models.IntegerField()),
('TC', models.IntegerField()),
('BC', models.IntegerField()),
('metro', models.FloatField()),
],
),
]

18
service/migrations/0002_point_link.py
Unescape View File

@ -1,18 +0,0 @@
# Generated by Django 3.2 on 2022-07-09 16:05
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='point',
name='link',
field=models.URLField(blank=True, null=True),
),
]

18
service/migrations/0003_alter_point_price.py
Unescape View File

@ -1,18 +0,0 @@
# Generated by Django 3.2 on 2022-07-09 16:17
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0002_point_link'),
]
operations = [
migrations.AlterField(
model_name='point',
name='price',
field=models.BigIntegerField(),
),
]

64
service/migrations/0004_auto_20220709_2004.py
Unescape View File

@ -1,64 +0,0 @@
# Generated by Django 3.2 on 2022-07-09 17:04
import django.contrib.gis.db.models.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0003_alter_point_price'),
]
operations = [
migrations.CreateModel(
name='Polygon',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('geometry', django.contrib.gis.db.models.fields.PolygonField(srid=4326, verbose_name='Геометрия')),
('cellid', models.IntegerField(verbose_name='ID клетки')),
('home', models.IntegerField(verbose_name='Дом')),
('job', models.IntegerField(verbose_name='Работа')),
('horeca', models.IntegerField(verbose_name='Рестораны')),
('brand', models.IntegerField(verbose_name='Бренды')),
('univer', models.IntegerField(verbose_name='Университеты')),
('hotel', models.IntegerField(verbose_name='Отели')),
('TC', models.IntegerField(verbose_name='ТЦ')),
('BC', models.IntegerField(verbose_name='БЦ')),
('metro', models.FloatField(verbose_name='Метро')),
],
),
migrations.DeleteModel(
name='Poly',
),
migrations.AlterField(
model_name='point',
name='adress',
field=models.CharField(blank=True, max_length=512, null=True, verbose_name='Адрес'),
),
migrations.AlterField(
model_name='point',
name='area',
field=models.FloatField(verbose_name='Площадь'),
),
migrations.AlterField(
model_name='point',
name='district',
field=models.CharField(blank=True, max_length=512, null=True, verbose_name='Район'),
),
migrations.AlterField(
model_name='point',
name='name',
field=models.CharField(blank=True, max_length=256, null=True, verbose_name='Название'),
),
migrations.AlterField(
model_name='point',
name='point',
field=django.contrib.gis.db.models.fields.PointField(srid=4326, verbose_name='Геометрия'),
),
migrations.AlterField(
model_name='point',
name='price',
field=models.BigIntegerField(verbose_name='Цена'),
),
]

25
service/migrations/0005_config.py
Unescape View File

@ -1,25 +0,0 @@
# Generated by Django 3.2 on 2022-07-11 09:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('service', '0004_auto_20220709_2004'),
]
operations = [
migrations.CreateModel(
name='Config',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=256, unique=True, verbose_name='Название модели')),
('data', models.JSONField(verbose_name='Данные')),
],
options={
'verbose_name': 'Конфигурация',
'verbose_name_plural': 'Конфигурация',
},
),
]

0
service/migrations/__init__.py
Unescape View File

38
service/models.py
Unescape View File

@ -1,38 +0,0 @@
from django.db import models
from django.contrib.gis.db import models as gis_models
class Polygon(models.Model):
geometry = gis_models.PolygonField(verbose_name='Геометрия')
cellid = models.IntegerField(verbose_name='ID клетки')
home = models.IntegerField(verbose_name='Дом')
job = models.IntegerField(verbose_name='Работа')
horeca = models.IntegerField(verbose_name='Рестораны')
brand = models.IntegerField(verbose_name='Бренды')
univer = models.IntegerField(verbose_name='Университеты')
hotel = models.IntegerField(verbose_name='Отели')
TC = models.IntegerField(verbose_name='ТЦ')
BC = models.IntegerField(verbose_name='БЦ')
metro = models.FloatField(verbose_name='Метро')
class Point(models.Model):
point = gis_models.PointField(verbose_name='Геометрия')
name = models.CharField(max_length=256, blank=True, null=True, verbose_name='Название')
adress = models.CharField(max_length=512, blank=True, null=True, verbose_name='Адрес')
district = models.CharField(max_length=512, blank=True, null=True, verbose_name='Район')
link = models.URLField(blank=True, null=True)
area = models.FloatField(verbose_name='Площадь')
price = models.BigIntegerField(verbose_name='Цена')
class Config(models.Model):
name = models.CharField(max_length=256, unique=True, verbose_name='Название модели')
data = models.JSONField(verbose_name='Данные')
def __str__(self):
return self.name
class Meta:
verbose_name = 'Конфигурация'
verbose_name_plural = 'Конфигурация'

17
service/serializers.py
Unescape View File

@ -1,17 +0,0 @@
from rest_framework import serializers
from . import models
from service.utils import get_model_column_names
class PolygonSerializer(serializers.ModelSerializer):
class Meta:
model = models.Polygon
fields = get_model_column_names(model, [])
class PointSerializer(serializers.ModelSerializer):
class Meta:
model = models.Point
fields = get_model_column_names(model, [])

7
service/urls.py
Unescape View File

@ -1,16 +1,15 @@
from django.urls import path
from django.conf.urls import url
from rest_framework import routers
from rest_framework.authtoken import views as rf_views
from . import views
app_name = 'HACK2'
app_name = 'postamates'
router = routers.DefaultRouter()
router.register('polygons', views.PolygonViewSet, basename='polygons')
router.register('points', views.PointViewSet, basename='points')
urlpatterns = router.urls
urlpatterns += [
path('ao_and_rayons/', views.ao_and_rayons.as_view(), name='ao_and_rayons'),
path('raschet/', views.raschet.as_view(), name='ao_and_rayons'),
]

43
service/utils.py
Unescape View File

@ -1,2 +1,41 @@
def get_model_column_names(model, exclude_fields_names):
return [x.column for x in model._meta.get_fields() if str(x.column) not in exclude_fields_names]
import pandas as pd
from django.conf import settings
import sqlalchemy
def raschet_real(df, koefs):
koef_summ = sum(koefs.values())
for k, v in koefs.items():
df[k] = (df[k] * v)
df['rate'] = df[list(koefs.keys())].apply(lambda x: sum([x[k]*koefs[k] for k in list(koefs.keys())]), axis=1)
df['rate'] = df['rate'] / koef_summ
return df
def raschet(tables, filters, koefs):
conn = sqlalchemy.create_engine(settings.DB_URL, connect_args={'options': '-csearch_path=public'})
msk_ao = filters.get('msk_ao')
msk_rayon = filters.get('msk_rayon')
category = filters.get('category', [])
categories = ','.join([f"'{c}'" for c in category])
points_df = None
nets_df = None
for table in tables:
if 'point' in table:
if msk_ao is not None:
query = f"select * from {table} where msk_ao={msk_ao} and category in ({categories});"
else:
query = f"select * from {table} where msk_rayon={msk_rayon} and category in ({categories});"
points = pd.read_sql(query, conn)
points_df = raschet_real(points, koefs)
if 'net' in table:
# if msk_ao is not None:
# query = f"select * from {table} where msk_ao={msk_ao};"
# else:
# query = f"select * from {table} where msk_rayon={msk_rayon};"
query = f"select * from {table};"
#
nets = pd.read_sql(query, conn)
nets_df = raschet_real(nets, koefs)
return points_df, nets_df

161
service/views.py
Unescape View File

@ -1,130 +1,43 @@
import datetime
from service import serializers
from service import models
from rest_framework import viewsets
from rest_framework.permissions import AllowAny
from django.core.serializers import serialize
from rest_framework.views import APIView
from rest_framework.response import Response
import json
from rest_framework import permissions
from django.core.cache import cache
from rest_framework import status
from rest_framework.decorators import action
from django.db.models import Max, Min
from django.conf import settings
from service.management.commands.create_poly import import_poly
from service.management.commands.create_points import import_points
from rest_framework.parsers import MultiPartParser
import os
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from service.utils import get_model_column_names
from django.http import HttpResponseRedirect
from django.contrib import messages
def get_min_max_filters(model, exclude_fields_names):
d = {}
l = []
fields = get_model_column_names(model, exclude_fields_names)
for i in fields:
l.append(Min(i))
l.append(Max(i))
qs = model.objects.aggregate(*l)
for i in fields:
d[i] = [qs[f'{i}__min'], qs[f'{i}__max']]
return d
def get_full_filters(model, exclude_fields_names):
d = {}
fields = get_model_column_names(model, exclude_fields_names)
values = model.objects.defer('pk').values(*fields)
for n, i in enumerate(fields):
d[i] = sorted([v[i] for v in values])
return d
def get_model_naming(model):
return models.Config.objects.get(name=f'{model.__name__}').data
class PolygonViewSet(viewsets.ReadOnlyModelViewSet):
serializer_class = serializers.PolygonSerializer
permission_classes = [AllowAny]
parser_classes = (MultiPartParser,)
@action(detail=False, methods=['get'])
def filters(self, request):
d = cache.get('polygon_filters')
if settings.DEBUG:
d = None
if d is None:
d = get_full_filters(models.Polygon, ['id', 'geometry'])
cache.set('polygon_filters', d, 60 * 60 * 24)
return Response(d, status=status.HTTP_200_OK)
@action(detail=False, methods=['get'])
def naming(self, request):
return Response(get_model_naming(models.Polygon), status=status.HTTP_200_OK)
@action(detail=False, methods=['post'])
def file_import(self, request):
try:
file = request.FILES['file'].file
path = default_storage.save(f'cells/cells_{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.zip', ContentFile(file.read()))
import_poly(os.path.join(settings.MEDIA_ROOT, path))
cache.delete('polygon_filters')
messages.add_message(request, messages.INFO, 'Данные успешно импортированы')
except Exception as e:
messages.add_message(request, messages.ERROR, f'Ошибка импорта: {e}')
return HttpResponseRedirect('/admin/')
import json
from service.utils import raschet as raschet_alg
import pandas as pd
from io import BytesIO
from django.http import HttpResponse
class PointViewSet(viewsets.ReadOnlyModelViewSet):
serializer_class = serializers.PointSerializer
permission_classes = [AllowAny]
queryset = models.Point.objects.all()
class ao_and_rayons(APIView):
permission_classes = [permissions.AllowAny]
def list(self, request, *args, **kwargs):
queryset = self.get_queryset()
d = cache.get('points')
if settings.DEBUG:
d = None
def get(self, request, format=None):
d = cache.get('ao_and_rayons')
if d is None:
d = json.loads(serialize('geojson', queryset,
geometry_field='point',
fields=get_model_column_names(models.Point,['point'])
))
cache.set('points', d, 60 * 60 * 24)
resp = Response(d)
resp["Access-Control-Allow-Origin"] = '*'
resp["Access-Control-Allow-Methods"] = 'GET,PUT, OPTIONS'
resp["Access-Control-Max-Age"] = '1000'
resp["Access-Control-Allow-Headers"] = 'X-Requested-With, Content-Type'
return resp
@action(detail=False, methods=['get'])
def filters(self, request):
resp = Response(get_min_max_filters(models.Point, ['id', 'point', 'is2025']))
resp["Access-Control-Allow-Origin"] = '*'
resp["Access-Control-Allow-Methods"] = 'GET,PUT, OPTIONS'
resp["Access-Control-Max-Age"] = '1000'
resp["Access-Control-Allow-Headers"] = 'X-Requested-With, Content-Type'
return resp
@action(detail=False, methods=['post'])
def file_import(self, request):
try:
file = request.FILES['file'].file
path = default_storage.save(f'points/points_{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.xlsx', ContentFile(file.read()))
import_points(os.path.join(settings.MEDIA_ROOT, path))
cache.delete('points')
messages.add_message(request, messages.INFO, 'Данные успешно импортированы')
except Exception as e:
messages.add_message(request, messages.ERROR, f'Ошибка импорта: {e}')
return HttpResponseRedirect('/admin/')
@action(detail=False, methods=['get'])
def naming(self, request):
return Response(get_model_naming(models.Point), status=status.HTTP_200_OK)
data = json.loads(open('ao_and_rayons.json', 'r').read())
cache.set('ao_and_rayons', data, 60 * 60 * 24)
return Response(d)
class raschet(APIView):
permission_classes = [permissions.AllowAny]
def post(self, request, format=None):
df_points, df_nets = raschet_alg(**request.data)
with BytesIO() as b:
# Use the StringIO object as the filehandle.
writer = pd.ExcelWriter(b, engine='xlsxwriter')
if df_points is not None:
df_points.to_excel(writer, sheet_name='Точки', index=False)
if df_nets is not None:
df_nets.to_excel(writer, sheet_name='Полигоны', index=False)
writer.save()
# Set up the Http response.
filename = f'Выгрузка.xlsx'
response = HttpResponse(
b.getvalue(),
content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return response

Write Preview
Loading…
Cancel
Save