Merge pull request #90 from Wbarillon/architecture-overhaul

Architecture overhaul

README.txt

@@ -0,0 +1,17 @@
+Yet another readme :D
+
+Quickstart :
+
+git clone https://github.com/dataforgoodfr/12_bloom.git
+
+docker compose build
+
+docker compose pull
+
+copy and paste bloom/env.template at the same level than docker-compose.yaml and rename it .env
+
+docker compose run --service-ports bloom /bin/bash
+
+streamlit run Trawlwatcher.py
+
+working mmsi : 261084090

bloom/Dockerfile

@@ -0,0 +1,63 @@
+FROM python:3.10-slim-bullseye
+
+RUN apt-get update
+RUN apt-get -y install wget gcc g++
+RUN apt-get install -y rsyslog
+
+# Define working directory
+WORKDIR /source_code
+COPY bloom/ ./bloom/
+COPY data/ ./data/
+COPY app.py .
+COPY container.py .
+COPY docker-env/rsyslog.conf /etc/rsyslog.conf
+
+# Install requirements package for python with poetry
+ARG POETRY_VERSION=1.8.1
+ENV POETRY_VERSION=${POETRY_VERSION}
+RUN pip install --user "poetry==$POETRY_VERSION"
+ENV PATH="${PATH}:/root/.local/bin"
+COPY pyproject.toml poetry.lock ./
+
+RUN python -m venv /venv
+ENV PATH=/venv/bin:$PATH \
+    VIRTUAL_ENV=/venv
+RUN poetry install
+
+
+# Install chrome in the latest version
+ARG CHROME_VERSION="112.0.5615.165-1"
+RUN wget --no-verbose -O /tmp/chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROME_VERSION}_amd64.deb
+RUN apt-get install -y --fix-missing --no-install-recommends /tmp/chrome.deb
+RUN rm -f google-chrome-stable_current_amd64.deb
+
+
+# Launch cron services
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install -y cron \
+    # Remove package lists for smaller image sizes
+    && rm -rf /var/lib/apt/lists/* \
+    && which cron \
+    && rm -rf /etc/cron.*/*
+
+# Create cron task inside container
+# Due to the fact that cron process doesn't access to declared ENV vars and doesn't load user profiles
+# The entrypoint.sh script stores ENV vars at runtime in the ~/.env file as key=value pairs
+# Then the cron line include some command to load these ENV vars from file before launching app.py
+# This mecanism allows to give access to the same ENV vars for app.py launch in terminal and launch via cron
+RUN echo "*/15 * * * *	root export \$(cat ~/.env | grep -v '#' | xargs);/venv/bin/python3 /source_code/app.py 2>&1 | /usr/bin/logger -t bloom"  >> ./cron_scrapper
+RUN chmod 744 ./cron_scrapper
+
+# Move cron tab into the right directory
+RUN mv ./cron_scrapper /etc/cron.d/cron_scrapper
+
+# Run file
+RUN crontab /etc/cron.d/cron_scrapper
+
+COPY docker-env/entrypoint.sh /entrypoint.sh
+RUN ["chmod", "+x", "/entrypoint.sh"]
+
+# ENTRYPOINT ["/bin/bash", "/entrypoint.sh"]
+# https://manpages.ubuntu.com/manpages/trusty/man8/cron.8.html
+# -f | Stay in foreground mode, don't daemonize.
+# -L loglevel | Tell  cron  what to log about jobs (errors are logged regardless of this value) as the sum of the following values:
+CMD ["cron","-f", "-L", "2"]

Makefile → bloom/Makefile

@@ -1,6 +1,7 @@
 VERSION ?= 1.0.0
 
-BLOOM_DEV_DOCKER = @docker run --name bloom-test  --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net -p 8501:8501
+#BLOOM_DEV_DOCKER = @docker run --name bloom-test  --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net -p 8501:8501
+BLOOM_DEV_DOCKER = @docker run --name bloom-test  --mount type=bind,source=.,target=/source_code --env-file ./.env.template --network=bloom_net -p 8501:8501
 BLOOM_PRODUCTION_DOCKER = @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env --log-driver json-file --log-opt max-size=10M --log-opt max-file=3 --entrypoint /entrypoint.sh
 
 build:

Trawlwatcher.py → bloom/Trawlwatcher.py

@@ -3,6 +3,7 @@
 import streamlit as st
 from dotenv import load_dotenv
 
+
 st.set_page_config(
     page_title="Bloom Trawlwatcher Demo app",
     page_icon="🐟",

app.py → bloom/app.py

@@ -55,4 +55,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main()

data/vessels_subset_02022024.csv → bloom/data/vessels_subset_02022024.csv

@@ -1,11 +1,11 @@
-"id","country_iso3","cfr","IMO","registration_number","external_marking","ship_name","ircs","mmsi","loa","type","mt_activated"
-621,"ESP","27678 ESP","8516043","3VI-5-1-15","3VI-5-1-15","PARADANTA PRIMERO","EAAB",224378000,"58.38",">25m",False
-784,"FRA","FRA000919999","9828936","919999","CC919999","SCOMBRUS","FLVN",226347000,"81.37",">25m",False
-795,"FRA","NLD198300393","8224406","716900","FC716900","PRINS BERNHARD","FKHH",227302000,"88.24",">25m",False
-835,"FRA","FRA000932206","9741097","932206","CC932206","GEVRED","FIUO",228066900,"77",">25m",False
-863,"FRA","FRA000914221","9352913","914221","MA914221","VENT DU NORD II","FMEU",228215800,"41.92",">25m",False
-920,"GBR",NULL,"9877389","GY127",NULL,"HENDRIKA JACOBA",NULL,232031183,NULL,"Demersal seiners",False
-1194,"NLD","FRA000544858","7928835","FRA000544858","SL-9","JOHANNA","PHOE",244938000,"34.3",">25m",False
-1280,"NLD","NLD201001155","9599731","NLD201001155","UK-153","LUB SENIOR","PCGC",246749000,"28.5",">25m",False
-1431,"POL","IRL000I13000","9204556","PL02315","GDY-151","ANNELIES ILENA","SPG4839",261084090,"144.6",">25m",False
-1507,"PRT","PRT000023094","9297694","PTAVE-117345-N","PTAVE-117345-N","FRANÇA MORTE","CUFE7",263581000,"73.8",">25m",False
+"id","country_iso3","cfr","IMO","registration_number","external_marking","ship_name","ircs","mmsi","loa","type","mt_activated"
+621,"ESP","27678 ESP","8516043","3VI-5-1-15","3VI-5-1-15","PARADANTA PRIMERO","EAAB",224378000,"58.38",">25m",False
+784,"FRA","FRA000919999","9828936","919999","CC919999","SCOMBRUS","FLVN",226347000,"81.37",">25m",False
+795,"FRA","NLD198300393","8224406","716900","FC716900","PRINS BERNHARD","FKHH",227302000,"88.24",">25m",False
+835,"FRA","FRA000932206","9741097","932206","CC932206","GEVRED","FIUO",228066900,"77",">25m",False
+863,"FRA","FRA000914221","9352913","914221","MA914221","VENT DU NORD II","FMEU",228215800,"41.92",">25m",False
+920,"GBR",NULL,"9877389","GY127",NULL,"HENDRIKA JACOBA",NULL,232031183,NULL,"Demersal seiners",False
+1194,"NLD","FRA000544858","7928835","FRA000544858","SL-9","JOHANNA","PHOE",244938000,"34.3",">25m",False
+1280,"NLD","NLD201001155","9599731","NLD201001155","UK-153","LUB SENIOR","PCGC",246749000,"28.5",">25m",False
+1431,"POL","IRL000I13000","9204556","PL02315","GDY-151","ANNELIES ILENA","SPG4839",261084090,"144.6",">25m",False
+1507,"PRT","PRT000023094","9297694","PTAVE-117345-N","PTAVE-117345-N","FRANÇA MORTE","CUFE7",263581000,"73.8",">25m",False

docker-env/Dockerfile → bloom/docker-env/Dockerfile

@@ -28,7 +28,7 @@ RUN poetry install
 # Install chrome in the latest version
 ARG CHROME_VERSION="112.0.5615.165-1"
 RUN wget --no-verbose -O /tmp/chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROME_VERSION}_amd64.deb
-RUN apt-get install -y /tmp/chrome.deb
+RUN apt-get install -y --fix-missing /tmp/chrome.deb
 RUN rm -f google-chrome-stable_current_amd64.deb
 
 

pages/1_Vessel_Exploration.py → bloom/pages/1_Vessel_Exploration.py

@@ -80,7 +80,7 @@ def select_vessel(mmsi):
     st.write("Select a vessel from the dropdown list")
 
 else:
-    mmsi = st.session_state.vessel_mmsi
+    mmsi = str(st.session_state.vessel_mmsi)
     v = load_trajectory(mmsi, load_mpa)
 
     # if "mpas" not in st.session_state:

bloom/requirements.txt

@@ -0,0 +1,24 @@
+pandas==2.2.1
+selenium==4.18.1
+undetected-chromedriver==3.4.6
+openpyxl==3.1.0
+pydantic==1.10.14
+pyyaml==6.0
+shapely==2.0.1
+geopandas==0.14.3
+sqlalchemy==2.0
+alembic==1.10.2
+geoalchemy2==0.13.1
+psycopg2-binary==2.9.6
+dependency-injector==4.41.0
+dependency_injection==1.2.0
+requests==2.31
+gql==3.4.0
+requests-toolbelt==1.0
+slack-sdk==3.21.3
+folium==0.14.0
+matplotlib==3.8.0
+geopy==2.4.0
+streamlit==1.27.2
+streamlit-folium==0.15.0
+python-dotenv==1.0.0

docker-compose.yaml

@@ -0,0 +1,45 @@
+x-common-infos:
+  # Env variables stored in a .env file at same level than docker-compose.yaml
+  environment: &common-env
+    POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME}
+    POSTGRES_PORT: ${POSTGRES_PORT}
+    POSTGRES_DB: ${POSTGRES_DB}
+    POSTGRES_USER: ${POSTGRES_USER}
+    POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+
+services:
+  bloom:
+    container_name: bloom-test
+    build: ./bloom/
+    command: streamlit run Trawlwatcher.py
+    volumes:
+      - ./bloom:/source_code
+    environment:
+      <<: *common-env
+    ports:
+      - 8501:8501
+    networks:
+        - bloom_net
+    depends_on:
+      postgres:
+        condition: service_healthy # The service is working and still running
+
+  postgres:
+    container_name: postgres_bloom
+    image: docker.pkg.github.com/wbarillon/docker-packages/trawlwatcher_local_db:0.2
+    environment:
+      <<: *common-env
+    ports:
+      - ${POSTGRES_PORT}:5432
+    networks:
+        - bloom_net
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready --quiet --dbname=${POSTGRES_DB} --username=${POSTGRES_USER}']
+      interval: 100ms
+      timeout: 14s
+      retries: 140
+      start_period: 0s
+
+networks:
+  bloom_net:
+    name: bloom_net