diff --git a/postamates/settings.py b/postamates/settings.py index 79695fb..6ffd62a 100644 --- a/postamates/settings.py +++ b/postamates/settings.py @@ -174,7 +174,6 @@ SWAGGER_SETTINGS = { 'SWAGGER_PATH': 'django_static/swagger/swagger.yaml', } - SRID = 4326 # celery config @@ -188,3 +187,4 @@ AGE_DAY_LIMIT = 270 AGE_DAY_BORDER = 30 EXCEL_EXPORT_FILENAME = 'placement_points.xlsx' JSON_EXPORT_FILENAME = 'placement_points.json' +DATA_UPLOAD_MAX_NUMBER_FIELDS = None diff --git a/requirements.txt b/requirements.txt index d27fb41..64db2fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -90,3 +90,4 @@ virtualenv==20.20.0 wcwidth==0.2.6 xlrd==1.2.0 XlsxWriter==3.0.8 +django-filter==23.2 \ No newline at end of file diff --git a/service/migrations/0024_tempfiles.py b/service/migrations/0024_tempfiles.py new file mode 100644 index 0000000..6b2ec1d --- /dev/null +++ b/service/migrations/0024_tempfiles.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2 on 2023-08-03 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0023_auto_20230801_1654'), + ] + + operations = [ + migrations.CreateModel( + name='TempFiles', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data', models.TextField()), + ], + ), + ] diff --git a/service/migrations/0025_auto_20230804_1349.py b/service/migrations/0025_auto_20230804_1349.py new file mode 100644 index 0000000..6b07a2d --- /dev/null +++ b/service/migrations/0025_auto_20230804_1349.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2 on 2023-08-04 10:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('service', '0024_tempfiles'), + ] + + operations = [ + migrations.AlterField( + model_name='otherobjectsgroup', + name='category', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.otherobjectscategory'), + ), + migrations.AlterField( + model_name='post_and_pvzgroup', + name='category', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.post_and_pvzcategory'), + ), + ] diff --git a/service/models.py b/service/models.py index 1a26403..24c1399 100644 --- a/service/models.py +++ b/service/models.py @@ -165,7 +165,7 @@ class Post_and_pvzGroup(models.Model): return self.category.name + ' ' + self.name name = models.TextField(null=False, blank=False, verbose_name='Название группы') - category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='post_and_pvz_group', + category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='groups', on_delete=models.CASCADE) image = models.ImageField(blank=True, null=True, upload_to='post_and_pvz_group_images/', verbose_name='Картинка') visible = models.BooleanField(default=True) @@ -195,7 +195,7 @@ class OtherObjectsGroup(models.Model): return self.category.name + ' ' + self.name name = models.TextField(null=False, blank=False, verbose_name='Название группы') - category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='otherobjectsgroup', + category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='groups', on_delete=models.CASCADE) image = models.ImageField(blank=True, null=True, upload_to='other_objects_group_images/', verbose_name='Картинка') visible = models.BooleanField(default=True) @@ -221,3 +221,7 @@ class TaskStatus(models.Model): class LastMLCall(models.Model): dt = models.DateTimeField(auto_now_add=True) + + +class TempFiles(models.Model): + data = models.TextField(blank=False, null=False) diff --git a/service/serializers.py b/service/serializers.py index 1efb341..39ef0a4 100644 --- a/service/serializers.py +++ b/service/serializers.py @@ -9,6 +9,34 @@ class PlacementPointSerializer(serializers.ModelSerializer): fields = '__all__' +class PostAndPVZGroupSerializer(serializers.ModelSerializer): + class Meta: + model = models.Post_and_pvzGroup + fields = '__all__' + + +class PostAndPVZCategorySerializer(serializers.ModelSerializer): + groups = PostAndPVZGroupSerializer(read_only=True, many=True) + + class Meta: + model = models.Post_and_pvzCategory + fields = '__all__' + + +class OtherObjectsGroupSerializer(serializers.ModelSerializer): + class Meta: + model = models.OtherObjectsGroup + fields = '__all__' + + +class OtherObjectsCategorySerializer(serializers.ModelSerializer): + groups = OtherObjectsGroupSerializer(many=True, read_only=True) + + class Meta: + model = models.OtherObjectsCategory + fields = '__all__' + + class RayonSerializer(serializers.ModelSerializer): class Meta: model = models.AO diff --git a/service/service.py b/service/service.py index 644ce9d..ef56fe9 100644 --- a/service/service.py +++ b/service/service.py @@ -11,6 +11,12 @@ from service.enums import PointStatus from service.tasks import raschet +class LayerService: + @staticmethod + def get_post_and_pvz_categroies(): + return models.Post_and_pvzCategory.objects.all(), models.Post_and_pvzGroup.objects.all() + + class PointService: def update_fact(self, postamat_id: str, fact: int): qs = self.get_point_by_postamat_id(postamat_id) diff --git a/service/tasks.py b/service/tasks.py index 0d7bd22..23941d5 100644 --- a/service/tasks.py +++ b/service/tasks.py @@ -15,17 +15,13 @@ from sklearn import metrics from sklearn import model_selection as ms from sqlalchemy import text from django.contrib.gis.db.models.functions import Distance -import requests -from io import StringIO from postamates.settings import AGE_DAY_LIMIT from postamates.settings import DB_URL from service.models import PlacementPoint, LastMLCall from service import models - - -def log_to_telegram(msg): - requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage', - json={"chat_id": "-555238820", "text": msg}) +from service.utils import log_to_telegram +import base64 +from io import StringIO @shared_task() @@ -314,8 +310,33 @@ def raschet(): @shared_task -def calculate_group_distance(groups: list): - status, _ = models.TaskStatus.objects.get_or_create(task_name='Расчет ближайшего расстояния') +def load_post_and_pvz(obj_id: int): + file = models.TempFiles.objects.get(id=obj_id) + status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка ПВЗ и Постаматов') + excel_file = base64.b64decode(file.data) + df = pd.read_excel(excel_file) + df = df.replace(np.nan, None) + df = df.replace('NaT', None) + df.columns = df.columns.str.lower() + data_len = df.shape[0] + for _ind, row in enumerate(df.to_dict('records')): + status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%" + status.save() + category = row.get('category') + group = row.get('group') + if category: + cat, _ = models.Post_and_pvzCategory.objects.get_or_create(name=category) + if group: + gr, _ = models.Post_and_pvzGroup.objects.get_or_create(name=group, category=cat) + row['category'] = cat + row['group'] = gr + lon = str(row.pop('lon')) + lat = str(row.pop("lat")) + row['wkt'] = "POINT(" + lon + " " + lat + ")" + models.Post_and_pvz.objects.get_or_create(**row) + status.status = "Загрузка данных завершена" + status.save() + groups = df[['group', 'category']].drop_duplicates().to_dict(orient='records') points = models.PlacementPoint.objects.all() num_points = points.count() total = len(groups) * num_points @@ -349,3 +370,57 @@ def add_age_day(): c2 = PlacementPoint.objects.filter(sample_trn=True).count() if c2 - c1 != 0: raschet.delay() + + +@shared_task() +def load_other_objects(obj_id: int): + file = models.TempFiles.objects.get(id=obj_id) + status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Прочих объектов') + excel_file = base64.b64decode(file.data) + df = pd.read_excel(excel_file) + df = df.replace(np.nan, None) + df = df.replace('NaT', None) + df.columns = df.columns.str.lower() + data_len = df.shape[0] + for _ind, row in enumerate(df.to_dict('records')): + status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%" + status.save() + category = row.get('category') + group = row.get('group') + if category: + cat, _ = models.OtherObjectsCategory.objects.get_or_create(name=category) + if group: + gr, _ = models.OtherObjectsGroup.objects.get_or_create(name=group, category=cat) + row['category'] = cat + row['group'] = gr + lon = str(row.pop('lon')) + lat = str(row.pop("lat")) + row['wkt'] = "POINT(" + lon + " " + lat + ")" + models.OtherObjects.objects.get_or_create(**row) + status.status = "Загрузка данных завершена" + status.save() + + +@shared_task() +def load_data(obj_id: int): + status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Точек') + file = models.TempFiles.objects.get(id=obj_id) + csv_file = base64.b64decode(file.data) + models.PlacementPoint.objects.all().delete() + s = str(csv_file, 'utf-8') + data = StringIO(s) + df = pd.read_csv(data, delimiter=';') + df = df.replace(np.nan, None) + df = df.replace('NaT', None) + data_len = df.shape[0] + for _ind, row in enumerate(df.to_dict('records')): + status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%" + status.save() + data = { + k: row[k] for k in row.keys() if + k not in ['id', 'location_id', 'area', 'district', 'age_month'] + } + models.PlacementPoint.objects.create(**data) + status.status = "Загрузка данных завершена" + status.save() + models.TempFiles.objects.all().delete() diff --git a/service/urls.py b/service/urls.py index d0873c1..088901e 100644 --- a/service/urls.py +++ b/service/urls.py @@ -30,6 +30,9 @@ schema_view = get_schema_view( urlpatterns = [ path('placement_points/', include([*router.urls]), name='placement_points'), path('ao_rayons', views.AOViewSet.as_view({'get': 'list'}), name='ao_and_rayons'), + path('postamate_and_pvz_groups', views.PostAndPVZCategoryViewSet.as_view({'get': 'list'}), + name='postamate_and_pvz_groups'), + path('other_object_groups', views.OtherObjectsCategoryViewSet.as_view({'get': 'list'}), name='other_object_groups'), url(r'load_csv/', views.refresh_placement_points.as_view(), name='upload_placement_points'), url(r'upload_ao_and_rayons/', views.load_ao_and_rayons.as_view(), name='upload_ao_and_rayons'), url(r'upload_post_and_pvz/', views.upload_post_and_pvz, name='upload_post_and_pvz'), diff --git a/service/utils.py b/service/utils.py index 246c6c4..80bfbc2 100644 --- a/service/utils.py +++ b/service/utils.py @@ -3,23 +3,10 @@ import numpy as np import pandas as pd from django.contrib.gis.geos import GEOSGeometry from geojson import MultiPolygon -from tqdm import tqdm -from service.tasks import calculate_group_distance from service import models - - -def load_data(filepath: str): - models.PlacementPoint.objects.all().delete() - df = pd.read_csv(filepath) - df = df.replace(np.nan, None) - df = df.replace('NaT', None) - for row in tqdm(df.to_dict('records'), desc='Loading data...'): - data = { - k: row[k] for k in row.keys() if - k not in ['id', 'location_id', 'area', 'district', 'age_month'] - } - models.PlacementPoint.objects.create(**data) +import requests +from tqdm import tqdm def load_ao_and_rayons( @@ -44,52 +31,15 @@ def load_ao_and_rayons( models.Rayon.objects.create(**{'name': name, 'polygon': GEOSGeometry(str(MultiPolygon(coords))), 'AO': ao}) -def load_rivals(filepath: str): - df = pd.read_excel(filepath) - df = df.replace(np.nan, None) - df = df.replace('NaT', None) - df.columns = df.columns.str.lower() - for row in tqdm(df.to_dict('records'), desc='Loading data...'): - category = row.get('category') - group = row.get('group') - if category: - cat, _ = models.Post_and_pvzCategory.objects.get_or_create(name=category) - if group: - gr, _ = models.Post_and_pvzGroup.objects.get_or_create(name=group, category=cat) - row['category'] = cat - row['group'] = gr - lon = str(row.pop('lon')) - lat = str(row.pop("lat")) - row['wkt'] = "POINT(" + lon + " " + lat + ")" - models.Post_and_pvz.objects.get_or_create(**row) - new_groups = df[['group', 'category']].drop_duplicates().to_dict(orient='records') - calculate_group_distance.delay(new_groups) - - -def load_other_objects(filepath: str): - df = pd.read_excel(filepath) - df = df.replace(np.nan, None) - df = df.replace('NaT', None) - df.columns = df.columns.str.lower() - for row in tqdm(df.to_dict('records'), desc='Loading data...'): - category = row.get('category') - group = row.get('group') - if category: - cat, _ = models.OtherObjectsCategory.objects.get_or_create(name=category) - if group: - gr, _ = models.OtherObjectsGroup.objects.get_or_create(name=group, category=cat) - row['category'] = cat - row['group'] = gr - lon = str(row.pop('lon')) - lat = str(row.pop("lat")) - row['wkt'] = "POINT(" + lon + " " + lat + ")" - models.OtherObjects.objects.get_or_create(**row) - - def load_dist(filepath: str): models.PointDist.objects.all().delete() df = pd.read_csv(filepath) - for row in tqdm(df.to_dict('records'), desc='Loading data...'): + for row in df.to_dict('records'): row['id1'] = models.PlacementPoint.objects.get(pk=row.get('id1')) row['id2'] = models.PlacementPoint.objects.get(pk=row.get('id2')) models.PointDist.objects.create(**row) + + +def log_to_telegram(msg): + requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage', + json={"chat_id": "-555238820", "text": msg}) diff --git a/service/views.py b/service/views.py index 4c8ecf1..4cc9fe4 100644 --- a/service/views.py +++ b/service/views.py @@ -22,11 +22,12 @@ from service import utils from service.enums import PointStatus from service.permissions import UserPermission from service.service import PointService -from service.tasks import raschet -from service.utils import load_data +from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data from rest_framework.permissions import AllowAny from django.shortcuts import redirect from django.contrib import messages +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import filters class AOViewSet(ReadOnlyModelViewSet): @@ -35,11 +36,79 @@ class AOViewSet(ReadOnlyModelViewSet): permission_classes = [AllowAny] +class PostAndPVZCategoryViewSet(ReadOnlyModelViewSet): + serializer_class = serializers.PostAndPVZCategorySerializer + queryset = models.Post_and_pvzCategory.objects + + +class OtherObjectsCategoryViewSet(ReadOnlyModelViewSet): + serializer_class = serializers.OtherObjectsCategorySerializer + queryset = models.OtherObjectsCategory.objects + + class PlacementPointViewSet(ReadOnlyModelViewSet): serializer_class = serializers.PlacementPointSerializer queryset = models.PlacementPoint.objects pagination_class = pagination.MyPagination permission_classes = [UserPermission] + filter_backends = [DjangoFilterBackend, filters.OrderingFilter, filters.SearchFilter] + + @property + def filterset_fields(self): + model_cls = self.queryset.model + fieldset = { + field.name: [ + "exact", + "gt", + "gte", + "lt", + "lte", + "in", + "iexact", + "startswith", + "istartswith", + "endswith", + "iendswith", + "regex", + "iregex", + "isnull", + "contains", + "icontains", + ] + for field in [ + field + for field in model_cls._meta.get_fields() + if field.get_internal_type() + not in [ + "JSONField", + "ForeignKey", + "ManyToManyField", + "OneToOneField", + "PointField" + ] + ] + } + + # filters for model relations + for field in [ + field + for field in model_cls._meta.get_fields() + if field.get_internal_type() + in [ + "ForeignKey", + "ManyToManyField", + "OneToOneField", + ] + ]: + fieldset[field.name] = [ + "exact", + "gt", + "gte", + "lt", + "lte", + ] + + return fieldset def get_queryset(self): qs = self.queryset.all().order_by('id') @@ -59,6 +128,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): delta_current = self.request.GET.get('delta_current[]') rayons = self.request.GET.get('area[]') aos = self.request.GET.get('district[]') + group_dists = self.request.GET.getlist('dist_to_group') if location_ids: location_ids = list(location_ids.split(',')) qs = qs.filter(pk__in=location_ids) @@ -108,8 +178,20 @@ class PlacementPointViewSet(ReadOnlyModelViewSet): inclded = list(included.split(',')) qs2 = models.PlacementPoint.objects.filter(pk__in=inclded).all() qs = (qs | qs2).distinct() + if group_dists: + g_d = [list(g.split(',')) for g in group_dists] + for group in g_d: + filtered_points = list( + models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]), + dist__lt=int(group[1])).values_list( + 'placement_point__id', flat=True)) + qs = qs.filter(id__in=filtered_points) return qs + @action(methods=['get'], detail=False) + def get_filterset_fields(self, request, *args, **kwargs): + return Response(self.filterset_fields, status=HTTPStatus.OK) + @action(detail=False, methods=['get']) def filters(self, request): qs = self.get_queryset() @@ -231,8 +313,11 @@ class refresh_placement_points(APIView): @staticmethod def post(request): warnings.filterwarnings('ignore') - file = request.FILES['file'] - load_data(file) + file = request.FILES['file'].file + file_bytes = file.read() + csv_base64 = base64.b64encode(file_bytes).decode() + obj = models.TempFiles.objects.create(data=csv_base64) + load_data.delay(obj.id) messages.success(request, 'Файл точек успешно загружен') return redirect('/admin') @@ -248,21 +333,30 @@ class load_ao_and_rayons(APIView): return redirect('/admin') +import base64 + + @api_view(['POST']) def upload_post_and_pvz(request): warnings.filterwarnings('ignore') - file_rivals = request.FILES['file_post_and_pvz'] - utils.load_rivals(file_rivals) - messages.success(request, 'Файл ПВЗ и Постаматов успешно загружен') + file_rivals = request.FILES['file_post_and_pvz'].file + file_bytes = file_rivals.read() + excel_base64 = base64.b64encode(file_bytes).decode() + obj = models.TempFiles.objects.create(data=excel_base64) + load_post_and_pvz.delay(obj.id) + messages.success(request, 'Загрузка ПВЗ и Постаматов началась. Отслеживайте выполнение в Статусе фоновых задач') return redirect('/admin') @api_view(['POST']) def upload_other_objects(request): warnings.filterwarnings('ignore') - file_rivals = request.FILES['file_other_objects'] - utils.load_other_objects(file_rivals) - messages.success(request, 'Файл прочих объектов успешно загружен') + file = request.FILES['file_other_objects'] + file_bytes = file.read() + excel_base64 = base64.b64encode(file_bytes).decode() + obj = models.TempFiles.objects.create(data=excel_base64) + load_other_objects.delay(obj.id) + messages.success(request, 'Загрузка Прочих объектов началась. Отслеживайте выполнение в Статусе фоновых задач') return redirect('/admin') diff --git a/templates/admin/delete_selected_confirmation.html b/templates/admin/delete_selected_confirmation.html new file mode 100644 index 0000000..cb162f3 --- /dev/null +++ b/templates/admin/delete_selected_confirmation.html @@ -0,0 +1,51 @@ +{% extends "admin/base_site.html" %} +{% load i18n l10n admin_urls static %} + +{% block extrahead %} + {{ block.super }} + {{ media }} + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %} + +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ opts.verbose_name_plural|capfirst }} +› {% translate 'Delete multiple objects' %} +
+{% endblock %} + +{% block content %} +{% if perms_lacking %} +

{% blocktranslate %}Deleting the selected {{ objects_name }} would result in deleting related objects, but your account doesn't have permission to delete the following types of objects:{% endblocktranslate %}

+ +{% elif protected %} +

{% blocktranslate %}Deleting the selected {{ objects_name }} would require deleting the following protected related objects:{% endblocktranslate %}

+ +{% else %} +

{% blocktranslate %}Are you sure you want to delete the selected {{ objects_name }}? All of the following objects and their related items will be deleted:{% endblocktranslate %}

+ {% include "admin/includes/object_delete_summary.html" %} +

{% translate "Objects" %}

+ {% for deletable_object in deletable_objects %} + {% if 100 < deletable_object|length %} +

{{ deletable_object|length }} objects

+ {% else %} + + {% endif %} + {% endfor %} +
{% csrf_token %} +
+ {% for obj in queryset %} + + {% endfor %} + + + + {% translate "No, take me back" %} +
+
+{% endif %} +{% endblock %}