Merge pull request #15 from eco4cast/targets-to-bucket

Pushing targets to s3

challenge_configuration.yaml

@@ -25,7 +25,8 @@ target_metadata_gsheet: https://docs.google.com/spreadsheets/d/10YTX9ae_C1rFdLgE
 targets_thumbnail: 'https://raw.githubusercontent.com/eco4cast/neon4cast-ci/main/catalog/thumbnail_plots/neon_stream.jpg'
 targets_thumbnail_title: 'Test Image'
 targets_path: 'catalog/targets/'
-##
+# to here
+targets_file_name: 'river-chl-targets.csv.gz'
 challenge_url: https://projects.ecoforecast.org/usgsrc4cast-ci
 github_repo: eco4cast/usgsrc4cast-ci
 target_groups:

targets/_targets.R

@@ -8,10 +8,22 @@ tar_option_set(packages = c("dataRetrieval",
                             "tidyverse"))
 
 source("src/download_nwis_data.R")
+source("src/s3_utils.R")
 
 # End this file with a list of target objects.
 list(
 
+  tar_target(
+    config_file,
+    "../challenge_configuration.yaml",
+    format = "file"
+  ),
+
+  tar_target(
+    config,
+    yaml::read_yaml(config_file)
+  ),
+
   tar_target(
     site_list_file,
     "in/FY23_ecological_forecast_challenge_USGS_sites.csv",
@@ -125,17 +137,20 @@ list(
                site_id = paste0("USGS-", site_id),
                project_id = "usgsrc4cast",
                duration = "P1D") %>%
-        select(datetime, site_id, #project_id, duration,
-               variable, observation)
+        select(project_id, site_id, datetime,
+               duration, variable, observation)
       write_csv(out, file = out_file)
       return(out_file)
     },
     format = "file"
   ),
 
   tar_target(
-    push_to_data_repo,
-    "fill_in"
+    push_to_targets_s3,
+    push_to_s3(
+      config = config,
+      local_file_name = all_historic_data_csv,
+      s3_file_name = config$targets_file_name)
   )
 
 )

targets/src/s3_utils.R

@@ -0,0 +1,30 @@
+
+#' helper function for pushing file to s3
+#'
+#' @param config configuration file of the challenge
+#' @param local_file_name targets file name
+#' @param s3_file_name file to write to s3 #'
+push_to_s3 <- function(
+    config,
+    local_file_name,
+    s3_file_name
+){
+
+  targets <- read_csv(local_file_name)
+  # duration hard coded for now
+  bucket_path <- glue::glue("{config$targets_bucket}/project_id={config$project_id}/duration=P1D")
+
+  s3 <- arrow::s3_bucket(bucket_path,
+                         endpoint_override = config$endpoint,
+                         access_key = Sys.getenv("OSN_KEY"),
+                         secret_key = Sys.getenv("OSN_SECRET"))
+
+  sink <- s3$path(s3_file_name)
+
+  # write to s3
+  arrow::write_csv_arrow(x = targets,
+                         sink = sink)
+
+  return(sink)
+}
+