filters+groups

postamates/settings.py

@@ -174,7 +174,6 @@ SWAGGER_SETTINGS = {
     'SWAGGER_PATH': 'django_static/swagger/swagger.yaml',
 }
 
-
 SRID = 4326
 
 # celery config
@@ -188,3 +187,4 @@ AGE_DAY_LIMIT = 270
 AGE_DAY_BORDER = 30
 EXCEL_EXPORT_FILENAME = 'placement_points.xlsx'
 JSON_EXPORT_FILENAME = 'placement_points.json'
+DATA_UPLOAD_MAX_NUMBER_FIELDS = None

requirements.txt

@@ -90,3 +90,4 @@ virtualenv==20.20.0
 wcwidth==0.2.6
 xlrd==1.2.0
 XlsxWriter==3.0.8
+django-filter==23.2

service/migrations/0024_tempfiles.py

@@ -0,0 +1,20 @@
+# Generated by Django 3.2 on 2023-08-03 15:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('service', '0023_auto_20230801_1654'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TempFiles',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('data', models.TextField()),
+            ],
+        ),
+    ]

service/migrations/0025_auto_20230804_1349.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2 on 2023-08-04 10:49
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('service', '0024_tempfiles'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='otherobjectsgroup',
+            name='category',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.otherobjectscategory'),
+        ),
+        migrations.AlterField(
+            model_name='post_and_pvzgroup',
+            name='category',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='groups', to='service.post_and_pvzcategory'),
+        ),
+    ]

service/models.py

@@ -165,7 +165,7 @@ class Post_and_pvzGroup(models.Model):
         return self.category.name + ' ' + self.name
 
     name = models.TextField(null=False, blank=False, verbose_name='Название группы')
-    category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='post_and_pvz_group',
+    category = models.ForeignKey('Post_and_pvzCategory', default=None, related_name='groups',
                                  on_delete=models.CASCADE)
     image = models.ImageField(blank=True, null=True, upload_to='post_and_pvz_group_images/', verbose_name='Картинка')
     visible = models.BooleanField(default=True)
@@ -195,7 +195,7 @@ class OtherObjectsGroup(models.Model):
         return self.category.name + ' ' + self.name
 
     name = models.TextField(null=False, blank=False, verbose_name='Название группы')
-    category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='otherobjectsgroup',
+    category = models.ForeignKey('OtherObjectsCategory', default=None, related_name='groups',
                                  on_delete=models.CASCADE)
     image = models.ImageField(blank=True, null=True, upload_to='other_objects_group_images/', verbose_name='Картинка')
     visible = models.BooleanField(default=True)
@@ -221,3 +221,7 @@ class TaskStatus(models.Model):
 
 class LastMLCall(models.Model):
     dt = models.DateTimeField(auto_now_add=True)
+
+
+class TempFiles(models.Model):
+    data = models.TextField(blank=False, null=False)

service/serializers.py

@@ -9,6 +9,34 @@ class PlacementPointSerializer(serializers.ModelSerializer):
         fields = '__all__'
 
 
+class PostAndPVZGroupSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.Post_and_pvzGroup
+        fields = '__all__'
+
+
+class PostAndPVZCategorySerializer(serializers.ModelSerializer):
+    groups = PostAndPVZGroupSerializer(read_only=True, many=True)
+
+    class Meta:
+        model = models.Post_and_pvzCategory
+        fields = '__all__'
+
+
+class OtherObjectsGroupSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = models.OtherObjectsGroup
+        fields = '__all__'
+
+
+class OtherObjectsCategorySerializer(serializers.ModelSerializer):
+    groups = OtherObjectsGroupSerializer(many=True, read_only=True)
+
+    class Meta:
+        model = models.OtherObjectsCategory
+        fields = '__all__'
+
+
 class RayonSerializer(serializers.ModelSerializer):
     class Meta:
         model = models.AO

service/service.py

@@ -11,6 +11,12 @@ from service.enums import PointStatus
 from service.tasks import raschet
 
 
+class LayerService:
+    @staticmethod
+    def get_post_and_pvz_categroies():
+        return models.Post_and_pvzCategory.objects.all(), models.Post_and_pvzGroup.objects.all()
+
+
 class PointService:
     def update_fact(self, postamat_id: str, fact: int):
         qs = self.get_point_by_postamat_id(postamat_id)

service/tasks.py

@@ -15,17 +15,13 @@ from sklearn import metrics
 from sklearn import model_selection as ms
 from sqlalchemy import text
 from django.contrib.gis.db.models.functions import Distance
-import requests
-from io import StringIO
 from postamates.settings import AGE_DAY_LIMIT
 from postamates.settings import DB_URL
 from service.models import PlacementPoint, LastMLCall
 from service import models
-
-
-def log_to_telegram(msg):
-    requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage',
-                  json={"chat_id": "-555238820", "text": msg})
+from service.utils import log_to_telegram
+import base64
+from io import StringIO
 
 
 @shared_task()
@@ -314,8 +310,33 @@ def raschet():
 
 
 @shared_task
-def calculate_group_distance(groups: list):
-    status, _ = models.TaskStatus.objects.get_or_create(task_name='Расчет ближайшего расстояния')
+def load_post_and_pvz(obj_id: int):
+    file = models.TempFiles.objects.get(id=obj_id)
+    status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка ПВЗ и Постаматов')
+    excel_file = base64.b64decode(file.data)
+    df = pd.read_excel(excel_file)
+    df = df.replace(np.nan, None)
+    df = df.replace('NaT', None)
+    df.columns = df.columns.str.lower()
+    data_len = df.shape[0]
+    for _ind, row in enumerate(df.to_dict('records')):
+        status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%"
+        status.save()
+        category = row.get('category')
+        group = row.get('group')
+        if category:
+            cat, _ = models.Post_and_pvzCategory.objects.get_or_create(name=category)
+        if group:
+            gr, _ = models.Post_and_pvzGroup.objects.get_or_create(name=group, category=cat)
+        row['category'] = cat
+        row['group'] = gr
+        lon = str(row.pop('lon'))
+        lat = str(row.pop("lat"))
+        row['wkt'] = "POINT(" + lon + " " + lat + ")"
+        models.Post_and_pvz.objects.get_or_create(**row)
+    status.status = "Загрузка данных завершена"
+    status.save()
+    groups = df[['group', 'category']].drop_duplicates().to_dict(orient='records')
     points = models.PlacementPoint.objects.all()
     num_points = points.count()
     total = len(groups) * num_points
@@ -349,3 +370,57 @@ def add_age_day():
     c2 = PlacementPoint.objects.filter(sample_trn=True).count()
     if c2 - c1 != 0:
         raschet.delay()
+
+
+@shared_task()
+def load_other_objects(obj_id: int):
+    file = models.TempFiles.objects.get(id=obj_id)
+    status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Прочих объектов')
+    excel_file = base64.b64decode(file.data)
+    df = pd.read_excel(excel_file)
+    df = df.replace(np.nan, None)
+    df = df.replace('NaT', None)
+    df.columns = df.columns.str.lower()
+    data_len = df.shape[0]
+    for _ind, row in enumerate(df.to_dict('records')):
+        status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%"
+        status.save()
+        category = row.get('category')
+        group = row.get('group')
+        if category:
+            cat, _ = models.OtherObjectsCategory.objects.get_or_create(name=category)
+        if group:
+            gr, _ = models.OtherObjectsGroup.objects.get_or_create(name=group, category=cat)
+        row['category'] = cat
+        row['group'] = gr
+        lon = str(row.pop('lon'))
+        lat = str(row.pop("lat"))
+        row['wkt'] = "POINT(" + lon + " " + lat + ")"
+        models.OtherObjects.objects.get_or_create(**row)
+    status.status = "Загрузка данных завершена"
+    status.save()
+
+
+@shared_task()
+def load_data(obj_id: int):
+    status, _ = models.TaskStatus.objects.get_or_create(task_name='Загрузка Точек')
+    file = models.TempFiles.objects.get(id=obj_id)
+    csv_file = base64.b64decode(file.data)
+    models.PlacementPoint.objects.all().delete()
+    s = str(csv_file, 'utf-8')
+    data = StringIO(s)
+    df = pd.read_csv(data, delimiter=';')
+    df = df.replace(np.nan, None)
+    df = df.replace('NaT', None)
+    data_len = df.shape[0]
+    for _ind, row in enumerate(df.to_dict('records')):
+        status.status = "Загрузка данных: " + str(int(_ind / data_len * 100)) + "%"
+        status.save()
+        data = {
+            k: row[k] for k in row.keys() if
+            k not in ['id', 'location_id', 'area', 'district', 'age_month']
+        }
+        models.PlacementPoint.objects.create(**data)
+    status.status = "Загрузка данных завершена"
+    status.save()
+    models.TempFiles.objects.all().delete()

service/urls.py

@@ -30,6 +30,9 @@ schema_view = get_schema_view(
 urlpatterns = [
     path('placement_points/', include([*router.urls]), name='placement_points'),
     path('ao_rayons', views.AOViewSet.as_view({'get': 'list'}), name='ao_and_rayons'),
+    path('postamate_and_pvz_groups', views.PostAndPVZCategoryViewSet.as_view({'get': 'list'}),
+         name='postamate_and_pvz_groups'),
+    path('other_object_groups', views.OtherObjectsCategoryViewSet.as_view({'get': 'list'}), name='other_object_groups'),
     url(r'load_csv/', views.refresh_placement_points.as_view(), name='upload_placement_points'),
     url(r'upload_ao_and_rayons/', views.load_ao_and_rayons.as_view(), name='upload_ao_and_rayons'),
     url(r'upload_post_and_pvz/', views.upload_post_and_pvz, name='upload_post_and_pvz'),

service/utils.py

@@ -3,23 +3,10 @@ import numpy as np
 import pandas as pd
 from django.contrib.gis.geos import GEOSGeometry
 from geojson import MultiPolygon
-from tqdm import tqdm
-from service.tasks import calculate_group_distance
 
 from service import models
-
-
-def load_data(filepath: str):
-    models.PlacementPoint.objects.all().delete()
-    df = pd.read_csv(filepath)
-    df = df.replace(np.nan, None)
-    df = df.replace('NaT', None)
-    for row in tqdm(df.to_dict('records'), desc='Loading data...'):
-        data = {
-            k: row[k] for k in row.keys() if
-            k not in ['id', 'location_id', 'area', 'district', 'age_month']
-        }
-        models.PlacementPoint.objects.create(**data)
+import requests
+from tqdm import tqdm
 
 
 def load_ao_and_rayons(
@@ -44,52 +31,15 @@ def load_ao_and_rayons(
         models.Rayon.objects.create(**{'name': name, 'polygon': GEOSGeometry(str(MultiPolygon(coords))), 'AO': ao})
 
 
-def load_rivals(filepath: str):
-    df = pd.read_excel(filepath)
-    df = df.replace(np.nan, None)
-    df = df.replace('NaT', None)
-    df.columns = df.columns.str.lower()
-    for row in tqdm(df.to_dict('records'), desc='Loading data...'):
-        category = row.get('category')
-        group = row.get('group')
-        if category:
-            cat, _ = models.Post_and_pvzCategory.objects.get_or_create(name=category)
-        if group:
-            gr, _ = models.Post_and_pvzGroup.objects.get_or_create(name=group, category=cat)
-        row['category'] = cat
-        row['group'] = gr
-        lon = str(row.pop('lon'))
-        lat = str(row.pop("lat"))
-        row['wkt'] = "POINT(" + lon + " " + lat + ")"
-        models.Post_and_pvz.objects.get_or_create(**row)
-    new_groups = df[['group', 'category']].drop_duplicates().to_dict(orient='records')
-    calculate_group_distance.delay(new_groups)
-
-
-def load_other_objects(filepath: str):
-    df = pd.read_excel(filepath)
-    df = df.replace(np.nan, None)
-    df = df.replace('NaT', None)
-    df.columns = df.columns.str.lower()
-    for row in tqdm(df.to_dict('records'), desc='Loading data...'):
-        category = row.get('category')
-        group = row.get('group')
-        if category:
-            cat, _ = models.OtherObjectsCategory.objects.get_or_create(name=category)
-        if group:
-            gr, _ = models.OtherObjectsGroup.objects.get_or_create(name=group, category=cat)
-        row['category'] = cat
-        row['group'] = gr
-        lon = str(row.pop('lon'))
-        lat = str(row.pop("lat"))
-        row['wkt'] = "POINT(" + lon + " " + lat + ")"
-        models.OtherObjects.objects.get_or_create(**row)
-
-
 def load_dist(filepath: str):
     models.PointDist.objects.all().delete()
     df = pd.read_csv(filepath)
-    for row in tqdm(df.to_dict('records'), desc='Loading data...'):
+    for row in df.to_dict('records'):
         row['id1'] = models.PlacementPoint.objects.get(pk=row.get('id1'))
         row['id2'] = models.PlacementPoint.objects.get(pk=row.get('id2'))
         models.PointDist.objects.create(**row)
+
+
+def log_to_telegram(msg):
+    requests.post('https://api.telegram.org/bot6275517704:AAHVp_qv9d9NU740JJdOM2fJdgS4r1AgJrw/sendMessage',
+                  json={"chat_id": "-555238820", "text": msg})

service/views.py

@@ -22,11 +22,12 @@ from service import utils
 from service.enums import PointStatus
 from service.permissions import UserPermission
 from service.service import PointService
-from service.tasks import raschet
-from service.utils import load_data
+from service.tasks import raschet, load_post_and_pvz, load_other_objects, load_data
 from rest_framework.permissions import AllowAny
 from django.shortcuts import redirect
 from django.contrib import messages
+from django_filters.rest_framework import DjangoFilterBackend
+from rest_framework import filters
 
 
 class AOViewSet(ReadOnlyModelViewSet):
@@ -35,11 +36,79 @@ class AOViewSet(ReadOnlyModelViewSet):
     permission_classes = [AllowAny]
 
 
+class PostAndPVZCategoryViewSet(ReadOnlyModelViewSet):
+    serializer_class = serializers.PostAndPVZCategorySerializer
+    queryset = models.Post_and_pvzCategory.objects
+
+
+class OtherObjectsCategoryViewSet(ReadOnlyModelViewSet):
+    serializer_class = serializers.OtherObjectsCategorySerializer
+    queryset = models.OtherObjectsCategory.objects
+
+
 class PlacementPointViewSet(ReadOnlyModelViewSet):
     serializer_class = serializers.PlacementPointSerializer
     queryset = models.PlacementPoint.objects
     pagination_class = pagination.MyPagination
     permission_classes = [UserPermission]
+    filter_backends = [DjangoFilterBackend, filters.OrderingFilter, filters.SearchFilter]
+
+    @property
+    def filterset_fields(self):
+        model_cls = self.queryset.model
+        fieldset = {
+            field.name: [
+                "exact",
+                "gt",
+                "gte",
+                "lt",
+                "lte",
+                "in",
+                "iexact",
+                "startswith",
+                "istartswith",
+                "endswith",
+                "iendswith",
+                "regex",
+                "iregex",
+                "isnull",
+                "contains",
+                "icontains",
+            ]
+            for field in [
+                field
+                for field in model_cls._meta.get_fields()
+                if field.get_internal_type()
+                   not in [
+                       "JSONField",
+                       "ForeignKey",
+                       "ManyToManyField",
+                       "OneToOneField",
+                       "PointField"
+                   ]
+            ]
+        }
+
+        # filters for model relations
+        for field in [
+            field
+            for field in model_cls._meta.get_fields()
+            if field.get_internal_type()
+               in [
+                   "ForeignKey",
+                   "ManyToManyField",
+                   "OneToOneField",
+               ]
+        ]:
+            fieldset[field.name] = [
+                "exact",
+                "gt",
+                "gte",
+                "lt",
+                "lte",
+            ]
+
+        return fieldset
 
     def get_queryset(self):
         qs = self.queryset.all().order_by('id')
@@ -59,6 +128,7 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
         delta_current = self.request.GET.get('delta_current[]')
         rayons = self.request.GET.get('area[]')
         aos = self.request.GET.get('district[]')
+        group_dists = self.request.GET.getlist('dist_to_group')
         if location_ids:
             location_ids = list(location_ids.split(','))
             qs = qs.filter(pk__in=location_ids)
@@ -108,8 +178,20 @@ class PlacementPointViewSet(ReadOnlyModelViewSet):
             inclded = list(included.split(','))
             qs2 = models.PlacementPoint.objects.filter(pk__in=inclded).all()
             qs = (qs | qs2).distinct()
+        if group_dists:
+            g_d = [list(g.split(',')) for g in group_dists]
+            for group in g_d:
+                filtered_points = list(
+                    models.PlacementPointPVZDistance.objects.filter(pvz_postamates_group__id=int(group[0]),
+                                                                    dist__lt=int(group[1])).values_list(
+                        'placement_point__id', flat=True))
+                qs = qs.filter(id__in=filtered_points)
         return qs
 
+    @action(methods=['get'], detail=False)
+    def get_filterset_fields(self, request, *args, **kwargs):
+        return Response(self.filterset_fields, status=HTTPStatus.OK)
+
     @action(detail=False, methods=['get'])
     def filters(self, request):
         qs = self.get_queryset()
@@ -231,8 +313,11 @@ class refresh_placement_points(APIView):
     @staticmethod
     def post(request):
         warnings.filterwarnings('ignore')
-        file = request.FILES['file']
-        load_data(file)
+        file = request.FILES['file'].file
+        file_bytes = file.read()
+        csv_base64 = base64.b64encode(file_bytes).decode()
+        obj = models.TempFiles.objects.create(data=csv_base64)
+        load_data.delay(obj.id)
         messages.success(request, 'Файл точек успешно загружен')
         return redirect('/admin')
 
@@ -248,21 +333,30 @@ class load_ao_and_rayons(APIView):
         return redirect('/admin')
 
 
+import base64
+
+
 @api_view(['POST'])
 def upload_post_and_pvz(request):
     warnings.filterwarnings('ignore')
-    file_rivals = request.FILES['file_post_and_pvz']
-    utils.load_rivals(file_rivals)
-    messages.success(request, 'Файл ПВЗ и Постаматов успешно загружен')
+    file_rivals = request.FILES['file_post_and_pvz'].file
+    file_bytes = file_rivals.read()
+    excel_base64 = base64.b64encode(file_bytes).decode()
+    obj = models.TempFiles.objects.create(data=excel_base64)
+    load_post_and_pvz.delay(obj.id)
+    messages.success(request, 'Загрузка ПВЗ и Постаматов началась. Отслеживайте выполнение в Статусе фоновых задач')
     return redirect('/admin')
 
 
 @api_view(['POST'])
 def upload_other_objects(request):
     warnings.filterwarnings('ignore')
-    file_rivals = request.FILES['file_other_objects']
-    utils.load_other_objects(file_rivals)
-    messages.success(request, 'Файл прочих объектов успешно загружен')
+    file = request.FILES['file_other_objects']
+    file_bytes = file.read()
+    excel_base64 = base64.b64encode(file_bytes).decode()
+    obj = models.TempFiles.objects.create(data=excel_base64)
+    load_other_objects.delay(obj.id)
+    messages.success(request, 'Загрузка Прочих объектов началась. Отслеживайте выполнение в Статусе фоновых задач')
     return redirect('/admin')
 
 

templates/admin/delete_selected_confirmation.html

@@ -0,0 +1,51 @@
+{% extends "admin/base_site.html" %}
+{% load i18n l10n admin_urls static %}
+
+{% block extrahead %}
+    {{ block.super }}
+    {{ media }}
+    <script src="{% static 'admin/js/cancel.js' %}" async></script>
+{% endblock %}
+
+{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} delete-confirmation delete-selected-confirmation{% endblock %}
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+<a href="{% url 'admin:index' %}">{% translate 'Home' %}</a>
+&rsaquo; <a href="{% url 'admin:app_list' app_label=opts.app_label %}">{{ opts.app_config.verbose_name }}</a>
+&rsaquo; <a href="{% url opts|admin_urlname:'changelist' %}">{{ opts.verbose_name_plural|capfirst }}</a>
+&rsaquo; {% translate 'Delete multiple objects' %}
+</div>
+{% endblock %}
+
+{% block content %}
+{% if perms_lacking %}
+    <p>{% blocktranslate %}Deleting the selected {{ objects_name }} would result in deleting related objects, but your account doesn't have permission to delete the following types of objects:{% endblocktranslate %}</p>
+    <ul>{{ perms_lacking|unordered_list }}</ul>
+{% elif protected %}
+    <p>{% blocktranslate %}Deleting the selected {{ objects_name }} would require deleting the following protected related objects:{% endblocktranslate %}</p>
+    <ul>{{ protected|unordered_list }}</ul>
+{% else %}
+    <p>{% blocktranslate %}Are you sure you want to delete the selected {{ objects_name }}? All of the following objects and their related items will be deleted:{% endblocktranslate %}</p>
+    {% include "admin/includes/object_delete_summary.html" %}
+    <h2>{% translate "Objects" %}</h2>
+    {% for deletable_object in deletable_objects %}
+    {% if 100 < deletable_object|length %}
+        <p>{{ deletable_object|length }} objects</p>
+    {% else %}
+        <ul>{{ deletable_object|unordered_list }}</ul>
+    {% endif %}
+    {% endfor %}
+    <form method="post">{% csrf_token %}
+    <div>
+    {% for obj in queryset %}
+    <input type="hidden" name="{{ action_checkbox_name }}" value="{{ obj.pk|unlocalize }}">
+    {% endfor %}
+    <input type="hidden" name="action" value="delete_selected">
+    <input type="hidden" name="post" value="yes">
+    <input type="submit" value="{% translate 'Yes, I’m sure' %}">
+    <a href="#" class="button cancel-link">{% translate "No, take me back" %}</a>
+    </div>
+    </form>
+{% endif %}
+{% endblock %}