Switch to S3 storage refs #116530

Dockerfile

@@ -1,23 +1,32 @@
 # Pull base image
-FROM python:3.8-slim-buster as builder
+FROM python:3.9-slim-bookworm as builder
 
 # install python project dependencies pre-requisites
 RUN apt-get update && \
-    apt-get install -y python-dev libldap2-dev libsasl2-dev libssl-dev && \
-    apt-get install -y gcc default-libmysqlclient-dev
+    apt-get install -y libldap2-dev libsasl2-dev libssl-dev && \
+    apt-get install -y gcc default-libmysqlclient-dev pkg-config
 
-# Set environment variables
-COPY requirements.txt requirements.txt
+# Copy Pipfile dependency list
+COPY Pipfile Pipfile
+COPY Pipfile.lock Pipfile.lock
 
 # Install pipenv
-RUN set -ex && pip install --upgrade pip
+RUN set -ex && pip install --upgrade pip && pip install pipenv
 
 # Install dependencies
-RUN set -ex && pip install -r requirements.txt
+RUN set -ex && pipenv install --system
+
+# gunicorn
+RUN mkdir /var/log/gunicorn
+RUN touch /var/log/gunicorn/access-ninetofiver.log && touch /var/log/gunicorn/error-ninetofiver.log
+RUN mkdir /run/gunicorn
 
 FROM builder as final
 WORKDIR /code
 COPY . /app/
 
+# whitenoise
+RUN python /app/manage.py collectstatic --noinput
+
 RUN set -ex && bash -c "eval $(grep 'PYTHONDONTWRITEBYTECODE' .env)"
 RUN set -ex && bash -c "eval $(grep 'PYTHONUNBUFFERED' .env)"

Pipfile

@@ -17,7 +17,7 @@ cryptography = "*"
 defusedxml = "*"
 Deprecated = "*"
 diff-match-patch = "*"
-django = "<=4.0"
+django = "==4.2.5"
 django-admin-autocomplete-filter = "*"
 django-admin-list-filter-dropdown = "*"
 django-admin-rangefilter = "*"
@@ -101,12 +101,15 @@ wrapt = "*"
 xlrd = "*"
 xlwt = "*"
 zipp = "*"
+crispy-bootstrap4 = "*"
+django-minio-backend = "*"
 
 [dev-packages]
 isort = "*"
 pudb = "*"
 pylint = "*"
 tox = "*"
+minio = "*"
 
 [requires]
 python_version = "3.9"

ninetofiver/feeds.py

@@ -4,15 +4,15 @@
 from django_ical.views import ICalFeed
 from icalendar import vCalAddress
 from django_ical.views import ICAL_EXTRA_FIELDS
-from django_ical.feedgenerator import ITEM_EVENT_FIELD_MAP
+from django_ical.feedgenerator import ITEM_ELEMENT_FIELD_MAP
 from django_ical import feedgenerator
 from ninetofiver import models
 
 
 ICAL_EXTRA_FIELDS.append('status')
-ITEM_EVENT_FIELD_MAP += (('status', 'status'),)
-ITEM_EVENT_FIELD_MAP = [x for x in ITEM_EVENT_FIELD_MAP if x[0] != 'link']
-feedgenerator.ITEM_EVENT_FIELD_MAP = ITEM_EVENT_FIELD_MAP
+ITEM_ELEMENT_FIELD_MAP += (('status', 'status'),)
+ITEM_ELEMENT_FIELD_MAP = [x for x in ITEM_ELEMENT_FIELD_MAP if x[0] != 'link']
+feedgenerator.ITEM_ELEMENT_FIELD_MAP = ITEM_ELEMENT_FIELD_MAP
 
 
 class BaseFeed(ICalFeed):

ninetofiver/models.py

@@ -20,6 +20,8 @@
 from phonenumber_field.modelfields import PhoneNumberField
 from polymorphic.models import PolymorphicManager
 from polymorphic.models import PolymorphicModel
+from django_minio_backend import MinioBackend
+from django.conf import settings
 
 log = logging.getLogger(__name__)
 
@@ -171,7 +173,7 @@ def generate_file_path(instance, filename):
     address = models.TextField(max_length=255)
     country = CountryField()
     internal = models.BooleanField(default=False)
-    logo = models.FileField(upload_to=generate_file_path, blank=True, null=True)
+    logo = models.FileField(upload_to=generate_file_path, blank=True, null=True, storage=MinioBackend(bucket_name=settings.MINIO_MEDIA_FILES_BUCKET))
 
     class Meta(BaseModel.Meta):
         verbose_name_plural = 'companies'
@@ -184,7 +186,8 @@ def __str__(self):
 
     def get_logo_url(self):
         """Get a URL to the logo."""
-        return reverse('ninetofiver_api_v2:download_company_logo', kwargs={'pk': self.pk})
+        http = "https://" if settings.MINIO_EXTERNAL_ENDPOINT_USE_HTTPS else "http://"
+        return f"{http}{settings.MINIO_EXTERNAL_ENDPOINT}/media/{self.logo.name}" # TODO: will this work for private bucket? 🤨🤨🤨
 
 
 class WorkSchedule(BaseModel):
@@ -501,7 +504,7 @@ def generate_file_path(instance, filename):
     user = models.ForeignKey(auth_models.User, on_delete=models.PROTECT)
     name = models.CharField(max_length=255)
     description = models.TextField(max_length=255, blank=True, null=True)
-    file = models.FileField(upload_to=generate_file_path)
+    file = models.FileField(upload_to=generate_file_path, storage=MinioBackend(bucket_name=settings.MINIO_MEDIA_FILES_BUCKET))
     slug = models.SlugField(default=uuid.uuid4, editable=False)
 
     class Meta(BaseModel.Meta):
@@ -518,7 +521,8 @@ def __str__(self):
 
     def get_file_url(self):
         """Get a URL to the file."""
-        return reverse('ninetofiver_api_v2:download_attachment', kwargs={'slug': self.slug})
+        http = "https://" if settings.MINIO_EXTERNAL_ENDPOINT_USE_HTTPS else "http://"
+        return f"{http}{settings.MINIO_EXTERNAL_ENDPOINT}/media/{self.file.name}" # TODO: will this work for private bucket? 🤨🤨🤨
 
 
 class Holiday(BaseModel):

ninetofiver/settings.py

@@ -79,6 +79,7 @@ def _process_cfg(cls):
         # deprecated in django 2.0, replaced by django_select2
         # 'django_admin_select2',
         'dal',
+        'crispy_bootstrap4',
         'dal_select2',
         'django.contrib.admin',
         'django.contrib.auth',
@@ -96,6 +97,7 @@ def _process_cfg(cls):
         'crispy_forms',
         'django_gravatar',
         'django_countries',
+        'django_minio_backend.apps.DjangoMinioBackendConfig',
         'rangefilter',
         'django_admin_listfilter_dropdown',
         'admin_auto_filters',
@@ -347,6 +349,19 @@ def _process_cfg(cls):
     ROCKETCHAT_INCOMING_WEBHOOK_URL = values.Value(None)
     ROCKETCHAT_PERFORMANCE_REMINDER_NOTIFICATION_ENABLED = values.Value(True)
     ROCKETCHAT_TIMESHEET_REMINDER_NOTIFICATION_ENABLED = values.Value(True)
+
+    # minio storage
+    MINIO_ENDPOINT = "minio:9000"
+    MINIO_ACCESS_KEY = "minio" # change me in child class
+    MINIO_SECRET_KEY = "minio-client" # change me in child class
+    MINIO_USE_HTTPS = False
+    MINIO_PUBLIC_BUCKETS = [
+        "media"
+    ]
+    MINIO_MEDIA_FILES_BUCKET = "media"
+    MINIO_CONSISTENCY_CHECK_ON_START = False
+    MINIO_EXTERNAL_ENDPOINT = "127.0.0.1:9000"
+    MINIO_EXTERNAL_ENDPOINT_USE_HTTPS = False
 
 
 class Dev(Base):
@@ -441,7 +456,7 @@ class Dev(Base):
 
     REGISTRATION_OPEN = True
 
-    # EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
+    EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 
 
 class Prod(Base):

scripts/migrate_to_minio/README.md

@@ -0,0 +1,4 @@
+# Script to migrate attachments stored on local storage to a minio bucket
+Make sure to install dev dependencies (`pipenv sync -d`)
+
+Run with `pipenv run python ./migration.py mysql_hostname mysql_port mysql_user mysql_pass path_to_current_media_folder mysql_db_name minio_hostname_and_port minio_access_key minio_secret_key minio_bucket_name`