feat: beta import function

.env.example

@@ -18,3 +18,9 @@ SQL_PORT=5432
 
 # Gunicorn
 WEB_CONCURRENCY=4
+
+# App Configs
+# Enable this if you want to keep deleted transactions in the database
+ENABLE_SOFT_DELETE=false
+# If ENABLE_SOFT_DELETE is true, transactions deleted for more than KEEP_DELETED_TRANSACTIONS_FOR days will be truly deleted. Set to 0 to keep all.
+KEEP_DELETED_TRANSACTIONS_FOR=365

app/WYGIWYH/settings.py

@@ -64,6 +64,7 @@
     "apps.accounts.apps.AccountsConfig",
     "apps.common.apps.CommonConfig",
     "apps.net_worth.apps.NetWorthConfig",
+    "apps.import_app.apps.ImportConfig",
     "apps.api.apps.ApiConfig",
     "cachalot",
     "rest_framework",
@@ -376,3 +377,6 @@
         "type": "image/png",
     },
 ]
+
+ENABLE_SOFT_DELETE = os.getenv("ENABLE_SOFT_DELETION", "false").lower() == "true"
+KEEP_DELETED_TRANSACTIONS_FOR = int(os.getenv("KEEP_DELETED_ENTRIES_FOR", "365"))

app/WYGIWYH/urls.py

@@ -48,4 +48,5 @@
     path("", include("apps.calendar_view.urls")),
     path("", include("apps.dca.urls")),
     path("", include("apps.mini_tools.urls")),
+    path("", include("apps.import_app.urls")),
 ]

app/apps/common/templatetags/json.py

@@ -0,0 +1,11 @@
+import json
+
+from django import template
+
+
+register = template.Library()
+
+
+@register.filter("json")
+def convert_to_json(value):
+    return json.dumps(value)

app/apps/import_app/admin.py

@@ -0,0 +1,6 @@
+from django.contrib import admin
+from apps.import_app import models
+
+# Register your models here.
+admin.site.register(models.ImportRun)
+admin.site.register(models.ImportProfile)

app/apps/import_app/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ImportConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "apps.import_app"

app/apps/import_app/forms.py

@@ -0,0 +1,64 @@
+from crispy_forms.bootstrap import FormActions
+from crispy_forms.helper import FormHelper
+from crispy_forms.layout import (
+    Layout,
+)
+from django import forms
+from django.utils.translation import gettext_lazy as _
+
+from apps.import_app.models import ImportProfile
+from apps.common.widgets.crispy.submit import NoClassSubmit
+
+
+class ImportProfileForm(forms.ModelForm):
+    class Meta:
+        model = ImportProfile
+        fields = [
+            "name",
+            "version",
+            "yaml_config",
+        ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout("name", "version", "yaml_config")
+
+        if self.instance and self.instance.pk:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Update"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+        else:
+            self.helper.layout.append(
+                FormActions(
+                    NoClassSubmit(
+                        "submit", _("Add"), css_class="btn btn-outline-primary w-100"
+                    ),
+                ),
+            )
+
+
+class ImportRunFileUploadForm(forms.Form):
+    file = forms.FileField(label=_("Select a file"))
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.helper = FormHelper()
+        self.helper.form_tag = False
+        self.helper.form_method = "post"
+        self.helper.layout = Layout(
+            "file",
+            FormActions(
+                NoClassSubmit(
+                    "submit", _("Import"), css_class="btn btn-outline-primary w-100"
+                ),
+            ),
+        )

app/apps/import_app/migrations/0001_initial.py

@@ -0,0 +1,51 @@
+# Generated by Django 5.1.5 on 2025-01-19 00:44
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('currencies', '0006_currency_exchange_currency'),
+        ('transactions', '0028_transaction_internal_note'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImportProfile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+                ('yaml_config', models.TextField(help_text='YAML configuration')),
+                ('version', models.IntegerField(choices=[(1, 'Version 1')], default=1, verbose_name='Version')),
+            ],
+            options={
+                'ordering': ['name'],
+            },
+        ),
+        migrations.CreateModel(
+            name='ImportRun',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('QUEUED', 'Queued'), ('PROCESSING', 'Processing'), ('FAILED', 'Failed'), ('FINISHED', 'Finished')], default='QUEUED', max_length=10, verbose_name='Status')),
+                ('file_name', models.CharField(help_text='File name', max_length=10000)),
+                ('logs', models.TextField(blank=True)),
+                ('processed_rows', models.IntegerField(default=0)),
+                ('total_rows', models.IntegerField(default=0)),
+                ('successful_rows', models.IntegerField(default=0)),
+                ('skipped_rows', models.IntegerField(default=0)),
+                ('failed_rows', models.IntegerField(default=0)),
+                ('started_at', models.DateTimeField(null=True)),
+                ('finished_at', models.DateTimeField(null=True)),
+                ('categories', models.ManyToManyField(related_name='import_runs', to='transactions.transactioncategory')),
+                ('currencies', models.ManyToManyField(related_name='import_runs', to='currencies.currency')),
+                ('entities', models.ManyToManyField(related_name='import_runs', to='transactions.transactionentity')),
+                ('profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='import_app.importprofile')),
+                ('tags', models.ManyToManyField(related_name='import_runs', to='transactions.transactiontag')),
+                ('transactions', models.ManyToManyField(related_name='import_runs', to='transactions.transaction')),
+            ],
+        ),
+    ]

app/apps/import_app/migrations/0002_alter_importprofile_name_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.1.5 on 2025-01-23 03:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('import_app', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='importprofile',
+            name='name',
+            field=models.CharField(max_length=100, unique=True, verbose_name='Name'),
+        ),
+        migrations.AlterField(
+            model_name='importprofile',
+            name='yaml_config',
+            field=models.TextField(verbose_name='YAML Configuration'),
+        ),
+    ]

app/apps/import_app/models.py

@@ -0,0 +1,87 @@
+import yaml
+
+from django.core.exceptions import ValidationError
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+from apps.import_app.schemas import version_1
+
+
+class ImportProfile(models.Model):
+    class Versions(models.IntegerChoices):
+        VERSION_1 = 1, _("Version") + " 1"
+
+    name = models.CharField(max_length=100, verbose_name=_("Name"), unique=True)
+    yaml_config = models.TextField(verbose_name=_("YAML Configuration"))
+    version = models.IntegerField(
+        choices=Versions,
+        default=Versions.VERSION_1,
+        verbose_name=_("Version"),
+    )
+
+    def __str__(self):
+        return self.name
+
+    class Meta:
+        ordering = ["name"]
+
+    def clean(self):
+        if self.version and self.version == self.Versions.VERSION_1:
+            try:
+                yaml_data = yaml.safe_load(self.yaml_config)
+                version_1.ImportProfileSchema(**yaml_data)
+            except Exception as e:
+                raise ValidationError({"yaml_config": _("Invalid YAML Configuration")})
+
+
+class ImportRun(models.Model):
+    class Status(models.TextChoices):
+        QUEUED = "QUEUED", _("Queued")
+        PROCESSING = "PROCESSING", _("Processing")
+        FAILED = "FAILED", _("Failed")
+        FINISHED = "FINISHED", _("Finished")
+
+    status = models.CharField(
+        max_length=10,
+        choices=Status,
+        default=Status.QUEUED,
+        verbose_name=_("Status"),
+    )
+    profile = models.ForeignKey(
+        ImportProfile,
+        on_delete=models.CASCADE,
+    )
+    file_name = models.CharField(
+        max_length=10000,
+        help_text=_("File name"),
+    )
+    transactions = models.ManyToManyField(
+        "transactions.Transaction", related_name="import_runs"
+    )
+    tags = models.ManyToManyField(
+        "transactions.TransactionTag", related_name="import_runs"
+    )
+    categories = models.ManyToManyField(
+        "transactions.TransactionCategory", related_name="import_runs"
+    )
+    entities = models.ManyToManyField(
+        "transactions.TransactionEntity", related_name="import_runs"
+    )
+    currencies = models.ManyToManyField(
+        "currencies.Currency", related_name="import_runs"
+    )
+
+    logs = models.TextField(blank=True)
+    processed_rows = models.IntegerField(default=0)
+    total_rows = models.IntegerField(default=0)
+    successful_rows = models.IntegerField(default=0)
+    skipped_rows = models.IntegerField(default=0)
+    failed_rows = models.IntegerField(default=0)
+    started_at = models.DateTimeField(null=True)
+    finished_at = models.DateTimeField(null=True)
+
+    @property
+    def progress(self):
+        if self.total_rows == 0:
+            return 0
+        return (self.processed_rows / self.total_rows) * 100

app/apps/import_app/schemas/__init__.py

@@ -0,0 +1 @@
+import apps.import_app.schemas.v1 as version_1