diff --git a/.gitignore b/.gitignore index 12a9d796..b6a0dbda 100644 --- a/.gitignore +++ b/.gitignore @@ -393,7 +393,7 @@ dist # vuepress build output .vuepress/dist -# vuepress v2.x temp and cache directory +# vuepress v2.x temporary and cache directory .temp # Docusaurus cache and generated files diff --git a/backend/donations/models/main.py b/backend/donations/models/main.py index 8f953587..5ee38808 100644 --- a/backend/donations/models/main.py +++ b/backend/donations/models/main.py @@ -348,8 +348,6 @@ class Donor(models.Model): ngo = models.ForeignKey(Ngo, verbose_name=_("NGO"), on_delete=models.SET_NULL, db_index=True, null=True) - # TODO: first name and last name have been swapped - # https://github.com/code4romania/redirectioneaza/issues/269 l_name = models.CharField(verbose_name=_("last name"), blank=True, null=False, default="", max_length=100) f_name = models.CharField(verbose_name=_("first name"), blank=True, null=False, default="", max_length=100) initial = models.CharField(verbose_name=_("initials"), blank=True, null=False, default="", max_length=5) diff --git a/backend/donations/views/donations_download.py b/backend/donations/views/donations_download.py index f9a97844..1d7dd8ac 100644 --- a/backend/donations/views/donations_download.py +++ b/backend/donations/views/donations_download.py @@ -133,8 +133,6 @@ def _package_donations(tmp_dir_name: str, donations: QuerySet[Donor], ngo: Ngo, detailed_address: Dict = _get_address_details(donation_object) donations_data.append( { - # TODO: first name and last name have been swapped - # https://github.com/code4romania/redirectioneaza/issues/269 "last_name": donation_object.l_name, "first_name": donation_object.f_name, "initial": donation_object.initial, diff --git a/backend/importer/__init__.py b/backend/importer/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/importer/apps.py b/backend/importer/apps.py deleted file mode 100644 index 92425812..00000000 --- a/backend/importer/apps.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.apps import AppConfig -from django.utils.translation import gettext_lazy as _ - - -class ImporterConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "importer" - verbose_name = _("Importer") diff --git a/backend/importer/extract.py b/backend/importer/extract.py deleted file mode 100644 index f89b82f5..00000000 --- a/backend/importer/extract.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -Extract the CNP and Address from the PDF form - -TODO: Integrate this with Django as a management command and use Django's FileField -""" - -from collections import namedtuple -from functools import partial -from io import BytesIO -from pypdf import PdfReader - - -Zone = namedtuple("Zone", ["start_y", "start_x", "end_x"]) - - -DATA_ZONES = { - "cnp": Zone(671, 336, 0), - "father": Zone(681, 300, 336), - "street_name": Zone(636, 67, 289), - "street_number": Zone(636, 289, 368), - "street_bl": Zone(614, 49, 108), - "street_sc": Zone(614, 108, 150), - "street_et": Zone(614, 150, 185), - "street_ap": Zone(614, 185, 255), - "percent": Zone(341, 146, 186), -} - - -def _visitor_builder(parts, start_x, start_y, end_x, text, cm, tm, fontDict, fontSize): - # print(tm, text) # Print the text matrix (for debugging) - x = tm[4] - y = tm[5] - # Check that the text is on the required line - if x >= start_x and y == start_y: - # Check that the text doesn't run over the end position - if (end_x and x < end_x) or not end_x: - parts.append(text.strip()) - - -def extract_data(page, zone: Zone): - parts = [] - visitor = partial(_visitor_builder, parts, zone.start_x, zone.start_y, zone.end_x) - page.extract_text(0, extraction_mode="plain", visitor_text=visitor) - return "".join(parts) - - -if __name__ == "__main__": - # This "open" to be replaced with Django's FileField.open() - pdf_file = open("document.pdf", "rb") - - reader = PdfReader(BytesIO(pdf_file.read())) - page = reader.pages[0] - - print("CNP =", extract_data(page, DATA_ZONES["cnp"])) - print("Father =", extract_data(page, DATA_ZONES["father"])) - - print("Street Name =", extract_data(page, DATA_ZONES["street_name"])) - print("Street Number =", extract_data(page, DATA_ZONES["street_number"])) - print("BL =", extract_data(page, DATA_ZONES["street_bl"])) - print("SC =", extract_data(page, DATA_ZONES["street_sc"])) - print("ET =", extract_data(page, DATA_ZONES["street_et"])) - print("AP =", extract_data(page, DATA_ZONES["street_ap"])) - - print("Percent =", extract_data(page, DATA_ZONES["percent"])) - - pdf_file.close() diff --git a/backend/importer/migrations/0001_initial.py b/backend/importer/migrations/0001_initial.py deleted file mode 100644 index 57fbbbc9..00000000 --- a/backend/importer/migrations/0001_initial.py +++ /dev/null @@ -1,48 +0,0 @@ -# Generated by Django 4.2.10 on 2024-02-15 17:15 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="ImportJob", - fields=[ - ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), - ( - "import_type", - models.CharField( - choices=[("users.User", "User"), ("donations.Ngo", "Ngo"), ("donations.Donor", "Donor")], - max_length=32, - verbose_name="Import type", - ), - ), - ( - "status", - models.CharField( - choices=[ - ("pending", "Pending"), - ("working", "Processing"), - ("done", "Done"), - ("error", "Error"), - ], - default="pending", - max_length=10, - verbose_name="Status", - ), - ), - ("has_header", models.BooleanField(default=True, verbose_name="Has header")), - ("csv_file", models.FileField(upload_to="imports/", verbose_name="File")), - ("uploaded_at", models.DateTimeField(auto_now_add=True, verbose_name="Uploaded at")), - ], - options={ - "verbose_name": "Import", - "verbose_name_plural": "Imports", - }, - ), - ] diff --git a/backend/importer/migrations/0002_alter_importjob_import_type.py b/backend/importer/migrations/0002_alter_importjob_import_type.py deleted file mode 100644 index 0745ed05..00000000 --- a/backend/importer/migrations/0002_alter_importjob_import_type.py +++ /dev/null @@ -1,27 +0,0 @@ -# Generated by Django 4.2.10 on 2024-02-16 14:43 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("importer", "0001_initial"), - ] - - operations = [ - migrations.AlterField( - model_name="importjob", - name="import_type", - field=models.CharField( - choices=[ - ("users.User", "User"), - ("donations.Ngo", "Ngo"), - ("donations.Donor", "Donor"), - ("partners.Partner", "Partner"), - ], - max_length=32, - verbose_name="Import type", - ), - ), - ] diff --git a/backend/importer/migrations/0003_alter_importjob_csv_file.py b/backend/importer/migrations/0003_alter_importjob_csv_file.py deleted file mode 100644 index b7341506..00000000 --- a/backend/importer/migrations/0003_alter_importjob_csv_file.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.10 on 2024-02-21 10:40 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("importer", "0002_alter_importjob_import_type"), - ] - - operations = [ - migrations.AlterField( - model_name="importjob", - name="csv_file", - field=models.FileField(upload_to="imports/%Y/%m/%d/", verbose_name="File"), - ), - ] diff --git a/backend/importer/migrations/0004_alter_importjob_options_remove_importjob_csv_file_and_more.py b/backend/importer/migrations/0004_alter_importjob_options_remove_importjob_csv_file_and_more.py deleted file mode 100644 index d9b34f93..00000000 --- a/backend/importer/migrations/0004_alter_importjob_options_remove_importjob_csv_file_and_more.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-31 11:01 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("importer", "0003_alter_importjob_csv_file"), - ] - - operations = [ - migrations.AlterModelOptions( - name="importjob", - options={}, - ), - migrations.RemoveField( - model_name="importjob", - name="csv_file", - ), - migrations.RemoveField( - model_name="importjob", - name="has_header", - ), - migrations.RemoveField( - model_name="importjob", - name="import_type", - ), - migrations.RemoveField( - model_name="importjob", - name="status", - ), - migrations.RemoveField( - model_name="importjob", - name="uploaded_at", - ), - ] diff --git a/backend/importer/migrations/0005_delete_importjob.py b/backend/importer/migrations/0005_delete_importjob.py deleted file mode 100644 index 2b8c5f6f..00000000 --- a/backend/importer/migrations/0005_delete_importjob.py +++ /dev/null @@ -1,16 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-31 11:02 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("importer", "0004_alter_importjob_options_remove_importjob_csv_file_and_more"), - ] - - operations = [ - migrations.DeleteModel( - name="ImportJob", - ), - ] diff --git a/backend/importer/migrations/__init__.py b/backend/importer/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/importer/tasks/__init__.py b/backend/importer/tasks/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/importer/tasks/donor_forms.py b/backend/importer/tasks/donor_forms.py deleted file mode 100644 index 47b9ba81..00000000 --- a/backend/importer/tasks/donor_forms.py +++ /dev/null @@ -1,62 +0,0 @@ -import logging -from typing import List - -from django.db.models import Count, Q, QuerySet -from django_q.tasks import async_task - -from donations.models.main import Donor, Ngo - -logger = logging.getLogger(__name__) - - -def import_donor_forms_task(batch_size: int = 50, run_async: bool = False, dry_run: bool = False): - logger.info("Starting a new donor form import task") - - ngos_by_number_of_donors: QuerySet[Ngo] = ( - Ngo.objects.all() - .annotate( - number_of_donors=Count( - "donor", - filter=(~Q(donor__pdf_url="") & Q(donor__pdf_file="") & Q(donor__date_created__gte="2023-12-31")), - ) - ) - .exclude(number_of_donors=0) - ) - - logger.info("Found %d NGOs with donors", ngos_by_number_of_donors.count()) - - index: int = 0 - processed_form_ids: List[int] = [] - for index, ngo in enumerate(ngos_by_number_of_donors): - donor_forms_for_ngo: List[int] = Donor.objects.filter( - ngo=ngo, pdf_file="", date_created__gte="2023-12-31" - ).values_list("pk", flat=True)[: batch_size + 1] - - processed_form_ids.extend(donor_forms_for_ngo) - - if len(processed_form_ids) > batch_size: - execute_import(index, processed_form_ids, run_async, dry_run) - - break - else: - if index: - execute_import(index, processed_form_ids, run_async, dry_run) - - return - - -def execute_import(index, processed_form_ids: List[int], run_async: bool, dry_run): - logger.info("Scheduling a new task for %d donors from %d NGOs", len(processed_form_ids), index + 1) - - if run_async: - async_task(import_donor_forms, ids=processed_form_ids, dry_run=dry_run) - else: - import_donor_forms(processed_form_ids, dry_run) - - -def import_donor_forms(ids: List[int], dry_run: bool): - """ - Download and re-upload the donation form files one by one - XXX: Will be removed - """ - ... diff --git a/backend/importer/tasks/repair_addresses.py b/backend/importer/tasks/repair_addresses.py deleted file mode 100644 index 072d53aa..00000000 --- a/backend/importer/tasks/repair_addresses.py +++ /dev/null @@ -1,67 +0,0 @@ -import ast -import logging -from json import JSONDecodeError -from typing import Dict - -from django.conf import settings -from django.db.models import QuerySet -from django_q.tasks import async_task - -from donations.models.main import Donor -from importer.tasks.utils import batch - -logger = logging.getLogger(__name__) - - -def repair_addresses_task(batch_size: int = 1000) -> None: - target_donor_ids: QuerySet[int] = ( - Donor.objects.exclude(encrypted_address="").values_list("pk", flat=True).order_by("pk") - ) - - for donor_ids in batch(target_donor_ids, batch_size): - logger.info("Starting task for the following list of donors: %s", len(donor_ids)) - async_task(repair_address_batch, donor_ids) - - -def repair_address_batch(ids: list) -> None: - target_donors: QuerySet[Donor] = Donor.objects.filter(pk__in=ids).order_by("pk") - - logger.info("Found %s donors with addresses to repair", target_donors.count()) - - for donor in target_donors: - logger.debug("Processing donor %s", donor.pk) - - try: - Donor.decrypt_address(donor.encrypted_address) - except JSONDecodeError: - repair_address(donor) - else: - continue - - -def repair_address(donor: Donor) -> None: - logger.debug("Repairing address for donor %s", donor.pk) - - encrypted_address: str = donor.encrypted_address - decoded_address: str = settings.FERNET_OBJECT.decrypt(encrypted_address.encode()).decode() - - try: - decoded_address: Dict = dict(ast.literal_eval(decoded_address)) - except ValueError: - logger.error("Received a ValueError trying to convert address to dict for donor %s", donor.pk) - return - except SyntaxError: - logger.error("Received a SyntaxError trying to convert address to dict for donor %s", donor.pk) - return - - address_dict = { - "street_name": decoded_address.get("str", ""), - "street_number": decoded_address.get("nr", ""), - "street_bl": decoded_address.get("bl", ""), - "street_sc": decoded_address.get("sc", ""), - "street_et": decoded_address.get("et", ""), - "street_ap": decoded_address.get("ap", ""), - } - - donor.set_address_helper(**address_dict) - donor.save(update_fields=["encrypted_address"]) diff --git a/backend/importer/tasks/utils.py b/backend/importer/tasks/utils.py deleted file mode 100644 index cb27980b..00000000 --- a/backend/importer/tasks/utils.py +++ /dev/null @@ -1,4 +0,0 @@ -def batch(iterable, batch_size=1) -> iter: - batch_length: int = len(iterable) - for index in range(0, batch_length, batch_size): - yield iterable[index : min(index + batch_size, batch_length)] diff --git a/backend/redirectioneaza/settings/base.py b/backend/redirectioneaza/settings/base.py index 9addbac1..2501035c 100644 --- a/backend/redirectioneaza/settings/base.py +++ b/backend/redirectioneaza/settings/base.py @@ -81,10 +81,9 @@ "allauth.socialaccount", "allauth.socialaccount.providers.amazon_cognito", # custom apps: + "users", "donations", "partners", - "users", - "importer", ] if not env.bool("USE_S3"):