Fix deployment problems

ansible/deploy.yml

@@ -1,6 +1,7 @@
 ---
 - name: Deploy binaries
-  hosts: lard-a # deploy to lard-a for now, might need to define a separate VM later?
+  # deploy to lard-a for now, might need to define a separate VM later?
+  hosts: lard-a
   remote_user: ubuntu
   roles:
     - role: deploy

ansible/migrate.yml

@@ -2,6 +2,7 @@
 - name: Perform DB migration
   hosts: servers
   remote_user: ubuntu
+  gather_facts: false
   vars:
     primary: lard-a
 
@@ -18,12 +19,12 @@
       become: true
       become_user: postgres
       block:
-        - name: Create data partition tables in lard
+        - name: Update public schema in lard
           community.postgresql.postgresql_script:
             db: lard
-            path: /etc/postgresql/16/db/partition.sql
+            path: /etc/postgresql/16/db/public.sql
 
-        - name: Update public schema in lard
+        - name: Create partition tables in lard
           community.postgresql.postgresql_script:
             db: lard
-            path: /etc/postgresql/16/db/public.sql
+            path: /etc/postgresql/16/db/partitions_generated.sql

ansible/roles/deploy/tasks/main.yml

@@ -43,6 +43,7 @@
     name: lard_ingestion
     state: restarted
     enabled: true
+  become: true
 
 - name: Unset environment variables
   ansible.builtin.command: systemctl unset-environment LARD_CONN_STRING STINFO_CONN_STRING

db/partitions_generated.sql

@@ -0,0 +1,73 @@
+-- Generated by simple script for testing
+CREATE TABLE IF NOT EXISTS data_y1950_to_y2000 PARTITION OF public.data
+FOR VALUES FROM ('1950-01-01 00:00:00+00') TO ('2000-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2000_to_y2010 PARTITION OF public.data
+FOR VALUES FROM ('2000-01-01 00:00:00+00') TO ('2010-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2010_to_y2015 PARTITION OF public.data
+FOR VALUES FROM ('2010-01-01 00:00:00+00') TO ('2015-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2015_to_y2016 PARTITION OF public.data
+FOR VALUES FROM ('2015-01-01 00:00:00+00') TO ('2016-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2016_to_y2017 PARTITION OF public.data
+FOR VALUES FROM ('2016-01-01 00:00:00+00') TO ('2017-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2017_to_y2018 PARTITION OF public.data
+FOR VALUES FROM ('2017-01-01 00:00:00+00') TO ('2018-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2018_to_y2019 PARTITION OF public.data
+FOR VALUES FROM ('2018-01-01 00:00:00+00') TO ('2019-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2019_to_y2020 PARTITION OF public.data
+FOR VALUES FROM ('2019-01-01 00:00:00+00') TO ('2020-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2020_to_y2021 PARTITION OF public.data
+FOR VALUES FROM ('2020-01-01 00:00:00+00') TO ('2021-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2021_to_y2022 PARTITION OF public.data
+FOR VALUES FROM ('2021-01-01 00:00:00+00') TO ('2022-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2022_to_y2023 PARTITION OF public.data
+FOR VALUES FROM ('2022-01-01 00:00:00+00') TO ('2023-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2023_to_y2024 PARTITION OF public.data
+FOR VALUES FROM ('2023-01-01 00:00:00+00') TO ('2024-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2024_to_y2025 PARTITION OF public.data
+FOR VALUES FROM ('2024-01-01 00:00:00+00') TO ('2025-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2025_to_y2026 PARTITION OF public.data
+FOR VALUES FROM ('2025-01-01 00:00:00+00') TO ('2026-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2026_to_y2027 PARTITION OF public.data
+FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2027-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2027_to_y2028 PARTITION OF public.data
+FOR VALUES FROM ('2027-01-01 00:00:00+00') TO ('2028-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2028_to_y2029 PARTITION OF public.data
+FOR VALUES FROM ('2028-01-01 00:00:00+00') TO ('2029-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS data_y2029_to_y2030 PARTITION OF public.data
+FOR VALUES FROM ('2029-01-01 00:00:00+00') TO ('2030-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y1950_to_y2000 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('1950-01-01 00:00:00+00') TO ('2000-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2000_to_y2010 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2000-01-01 00:00:00+00') TO ('2010-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2010_to_y2015 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2010-01-01 00:00:00+00') TO ('2015-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2015_to_y2016 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2015-01-01 00:00:00+00') TO ('2016-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2016_to_y2017 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2016-01-01 00:00:00+00') TO ('2017-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2017_to_y2018 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2017-01-01 00:00:00+00') TO ('2018-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2018_to_y2019 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2018-01-01 00:00:00+00') TO ('2019-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2019_to_y2020 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2019-01-01 00:00:00+00') TO ('2020-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2020_to_y2021 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2020-01-01 00:00:00+00') TO ('2021-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2021_to_y2022 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2021-01-01 00:00:00+00') TO ('2022-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2022_to_y2023 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2022-01-01 00:00:00+00') TO ('2023-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2023_to_y2024 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2023-01-01 00:00:00+00') TO ('2024-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2024_to_y2025 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2024-01-01 00:00:00+00') TO ('2025-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2025_to_y2026 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2025-01-01 00:00:00+00') TO ('2026-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2026_to_y2027 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2027-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2027_to_y2028 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2027-01-01 00:00:00+00') TO ('2028-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2028_to_y2029 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2028-01-01 00:00:00+00') TO ('2029-01-01 00:00:00+00');
+CREATE TABLE IF NOT EXISTS nonscalar_data_y2029_to_y2030 PARTITION OF public.nonscalar_data
+FOR VALUES FROM ('2029-01-01 00:00:00+00') TO ('2030-01-01 00:00:00+00');

ingestion/Cargo.toml

@@ -6,6 +6,8 @@ edition.workspace = true
 [features]
 integration_tests = []
 kafka = []
+# TODO: remove this when finished with testing Obsinn ingestion
+kafka_prod = []
 
 [dependencies]
 axum.workspace = true

ingestion/src/main.rs

@@ -2,14 +2,13 @@ use bb8_postgres::PostgresConnectionManager;
 use std::sync::{Arc, RwLock};
 use tokio_postgres::NoTls;
 
-#[cfg(feature = "kafka")]
-use lard_ingestion::kvkafka;
 use lard_ingestion::permissions;
 
 const PARAMCONV: &str = "resources/paramconversions.csv";
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    println!("LARD ingestion service starting up...");
     // TODO: use clap for argument parsing
     let args: Vec<String> = std::env::args().collect();
 
@@ -24,6 +23,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let permit_tables = Arc::new(RwLock::new(permissions::fetch_permits().await?));
     let background_permit_tables = permit_tables.clone();
 
+    println!("Spawing task to fetch permissions from StInfoSys...");
     // background task to refresh permit tables every 30 mins
     tokio::task::spawn(async move {
         let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(30 * 60));
@@ -48,10 +48,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let db_pool = bb8::Pool::builder().build(manager).await?;
 
     // Spawn kvkafka reader
-    let kafka_group = args[1].to_string();
-    #[cfg(feature = "kafka")]
-    tokio::spawn(kvkafka::read_and_insert(db_pool.clone(), kafka_group));
+    #[cfg(feature = "kafka_prod")]
+    {
+        let kafka_group = args[1].to_string();
+        println!("Spawing kvkafka reader...");
+        tokio::spawn(lard_ingestion::kvkafka::read_and_insert(
+            db_pool.clone(),
+            kafka_group,
+        ));
+    }
 
     // Set up and run our server + database
+    println!("Ingestion server started!");
     lard_ingestion::run(db_pool, PARAMCONV, permit_tables).await
 }