Allow user to override envvars, don't overwrite dagster.yaml

deploy.sh

@@ -11,23 +11,25 @@ if [ -z "$POPGETTER_COUNTRIES" ]; then
     exit 1
 fi
 
-export IGNORE_EXPERIMENTAL_WARNINGS=1
-export DAGSTER_MODULE_NAME=popgetter
-if ! DAGSTER_HOME=$(mktemp -d); then
-    echo "Failed to create temporary directory for DAGSTER_HOME"
-    exit 1
+if [ -z "$DAGSTER_HOME" ]; then
+    echo "DAGSTER_HOME environment variable not set; setting to temporary directory"
+    if ! DAGSTER_HOME=$(mktemp -d); then
+        echo "Failed to create temporary directory for DAGSTER_HOME"
+        exit 1
+    fi
+    export DAGSTER_HOME
+    echo "DAGSTER_HOME set to to $DAGSTER_HOME"
 fi
-export DAGSTER_HOME
-echo "Setting DAGSTER_HOME to $DAGSTER_HOME"
-# Reduce Dagster outputs
-printf "python_logs:\n  python_log_level: INFO\n" > "$DAGSTER_HOME"/dagster.yaml
+
+export IGNORE_EXPERIMENTAL_WARNINGS=1
+export DAGSTER_MODULE_NAME="${DAGSTER_MODULE_NAME:=popgetter}"
 
 echo "Relevant environment variables:"
 echo "  - POPGETTER_COUNTRIES: $POPGETTER_COUNTRIES"
 echo "  - ENV: $ENV"
 if [ "$ENV" == "prod" ]; then
-    export AZURE_STORAGE_ACCOUNT=popgetter
-    export AZURE_CONTAINER=prod
+    export AZURE_STORAGE_ACCOUNT="${AZURE_STORAGE_ACCOUNT:=popgetter}"
+    export AZURE_CONTAINER="${AZURE_CONTAINER:=prod}"
     if ! AZURE_DIRECTORY=$(python -c 'import popgetter; print(popgetter.__version__)' 2>/dev/null); then
         echo "Failed to get popgetter version"
         exit 1

python/popgetter/run.py

@@ -77,6 +77,7 @@ def try_materialise(asset, upstream_deps, instance, fail_fast, partition_key=Non
             assets=[asset, *upstream_deps],
             selection=[asset],
             instance=instance,
+            run_config={"loggers": {"console": {"config": {"log_level": "INFO"}}}},
             **({"partition_key": partition_key} if partition_key is not None else {}),
         )
     except Exception as e: