Skip to content

Commit

Permalink
Improve the data tracing structure
Browse files Browse the repository at this point in the history 
Adding dataset index
and also fix bugs in PCe approaches
  • Loading branch information
@realxinzhao
realxinzhao committed Oct 31, 2024
1 parent 288fc28 commit 097c700
Show file tree
Hide file tree
Showing 14 changed files with 232 additions and 271 deletions.
54 changes: 27 additions & 27 deletions R/xfaostat_L102_ProductionArea.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,11 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {
"FBSH_CBH_wide")

MODULE_OUTPUTS <-
c("QCL_PROD",
"QCL_AN_LIVEANIMAL",
"QCL_AN_PRIMARY_MILK",
"QCL_AN_LIVEANIMAL_MEATEQ",
"QCL_CROP_PRIMARY")
c("L102.QCL_PROD",
"L102.QCL_AN_LIVEANIMAL",
"L102.QCL_AN_PRIMARY_MILK",
"L102.QCL_AN_LIVEANIMAL_MEATEQ",
"L102.QCL_CROP_PRIMARY")

if(command == driver.DECLARE_INPUTS) {
return(MODULE_INPUTS)
Expand Down Expand Up @@ -378,7 +378,7 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {



# Bind QCL_ALL and save RDS ----
# Bind L102.QCL_ALL and save RDS ----
#*******************************************

QCL_CROP_PRIMARY %>% mutate(item_set = "QCL_COMM_CROP_PRIMARY") %>%
Expand All @@ -392,10 +392,10 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {
bind_rows(QCL_AN_LIVEANIMAL %>% mutate(item_set = "QCL_COMM_AN_LIVEANIMAL")) %>%
bind_rows(QCL_AN_LIVEANIMAL_MEATEQ %>% mutate(item_set = "QCL_COMM_AN_LIVEANIMAL_MEATEQ")) %>%
bind_rows(FBS_FISH %>% mutate(item_set = "FBS_COMM_FISH"))->
QCL_ALL
L102.QCL_ALL


QCL_ALL %>%
L102.QCL_ALL %>%
filter(item_set %in% c("QCL_COMM_AN_PRIMARY_MEAT1",
"QCL_COMM_AN_PRIMARY_MEAT2",
"QCL_COMM_AN_PRIMARY_EGG",
Expand All @@ -408,60 +408,60 @@ module_xfaostat_L102_ProductionArea <- function(command, ...) {
add_units("various") %>%
add_comments("Detailed FAO QCL data processing for live animal and production") %>%
add_precursors("QCL_wide", "FBS_wide", "FBSH_CBH_wide") ->
QCL_AN_LIVEANIMAL
L102.QCL_AN_LIVEANIMAL

QCL_AN_PRIMARY_MILK %>%
add_title("FAO milk animal stock and production") %>%
add_units("various") %>%
add_comments("Detailed FAO QCL data processing for dairy animal and production") %>%
same_precursors_as(QCL_AN_LIVEANIMAL) ->
QCL_AN_PRIMARY_MILK
same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
L102.QCL_AN_PRIMARY_MILK

QCL_AN_LIVEANIMAL_MEATEQ %>%
add_title("FAO live animal stock meat equivalent") %>%
add_units("various") %>%
add_comments("Detailed FAO QCL data processing for live animal stock meat equivalent") %>%
same_precursors_as(QCL_AN_LIVEANIMAL) ->
QCL_AN_LIVEANIMAL_MEATEQ
same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
L102.QCL_AN_LIVEANIMAL_MEATEQ

QCL_CROP_PRIMARY %>%
add_title("FAO primary crop area and production") %>%
add_units("various") %>%
add_comments("Detailed FAO QCL data processing for crop area and production") %>%
same_precursors_as(QCL_AN_LIVEANIMAL) ->
QCL_CROP_PRIMARY
same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
L102.QCL_CROP_PRIMARY


# Production only
QCL_ALL %>% filter(element_code == 5510) ->
QCL_PROD
L102.QCL_ALL %>% filter(element_code == 5510) ->
L102.QCL_PROD

QCL_PROD %>%
L102.QCL_PROD %>%
add_title("FAO primary production") %>%
add_units("tonnes") %>%
add_comments("FAO primary production") %>%
same_precursors_as(QCL_AN_LIVEANIMAL) ->
QCL_PROD
same_precursors_as(L102.QCL_AN_LIVEANIMAL) ->
L102.QCL_PROD

# No NA
assertthat::assert_that(QCL_ALL %>% filter(is.na(value)) %>% nrow() == 0)
assertthat::assert_that(L102.QCL_ALL %>% filter(is.na(value)) %>% nrow() == 0)

#QCL_ALL %>% FF_check_count_plot -> p; p
#L102.QCL_ALL %>% FF_check_count_plot -> p; p
# ggsave(file.path(DIR_DATAPROC_PLOT, "QCL_ALL.png"),
# plot = p + ggtitle("gcamdata-FAOSTAT (QCL & FBS) production data over time"),
# dpi = 200, width = 9, height = 5 )
# rm(p)

QCL_ALL %>% distinct(year); # 53 years
QCL_ALL %>% distinct(element, element_code, unit) # QCL_COMM_AN_LIVEANIMAL_MEATEQ has no element_code
QCL_ALL %>% distinct(item) # 158 primary crop + 45 primary an + 57 others + 17 +12
L102.QCL_ALL %>% distinct(year); # 53 years
L102.QCL_ALL %>% distinct(element, element_code, unit) # QCL_COMM_AN_LIVEANIMAL_MEATEQ has no element_code
L102.QCL_ALL %>% distinct(item) # 158 primary crop + 45 primary an + 57 others + 17 +12


# QCL_ALL %>%
# L102.QCL_ALL %>%
# add_title("FAO crop and livestock production and crop area") %>%
# add_units("various") %>%
# add_comments("Detailed FAO QCL data processing. FBS fish data is used") ->
# QCL_ALL
# L102.QCL_ALL


return_data(MODULE_OUTPUTS)
Expand Down
20 changes: 10 additions & 10 deletions R/xfaostat_L103_ProducerPrices.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
module_xfaostat_L103_ProducerPrices <- function(command, ...) {

MODULE_INPUTS <-
c("QCL_PROD",
c("L102.QCL_PROD",
"PP_wide")

MODULE_OUTPUTS <-
c("QCL_PRIMARY_PROD_PV")
c("L103.QCL_PRIMARY_PROD_PV")

if(command == driver.DECLARE_INPUTS) {
return(MODULE_INPUTS)
Expand All @@ -32,7 +32,7 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) {

year <- value <- Year <- Value <- FAO_country <- iso <- NULL # silence package check.
PP_wide <- element_code <- element <- area_code <- item_code <- area <-
item <- unit <- QCL_PROD <- item_set <- GCAM_commodity <- QCL_PRIMARY_item <-
item <- unit <- L102.QCL_PROD <- item_set <- GCAM_commodity <- QCL_PRIMARY_item <-
PP_item <- Production <- Prod_Value <- PP_index <- World_PP_index <-
PP_Multiplier <- PP_item_world <- NULL

Expand All @@ -50,15 +50,15 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) {


# Primary crops
QCL_PROD %>% filter(item_set == "QCL_COMM_CROP_PRIMARY") %>%
L102.QCL_PROD %>% filter(item_set == "QCL_COMM_CROP_PRIMARY") %>%
distinct(item, item_code) -> QCL_COMM_CROP_PRIMARY
# Primary animal products, including fat hides etc. 46
QCL_PROD %>% filter(grepl("AN_PRIMARY", item_set) ) %>%
L102.QCL_PROD %>% filter(grepl("AN_PRIMARY", item_set) ) %>%
distinct(item, item_code) -> QCL_COMM_AN_PRIMARY

QCL_COMM_CROP_PRIMARY %>%
bind_rows(QCL_COMM_AN_PRIMARY) %>%
left_join(QCL_PROD, by = c("item_code", "item")) ->
left_join(L102.QCL_PROD, by = c("item_code", "item")) ->
QCL_PRIMARY

QCL_PRIMARY %>% distinct(item, element, item_set, unit)
Expand Down Expand Up @@ -129,17 +129,17 @@ module_xfaostat_L103_ProducerPrices <- function(command, ...) {
left_join(UnitMap %>%
bind_rows(UnitMap %>% mutate(element = "Prod_Value", unit = "USD")),
by = "element") ->
QCL_PRIMARY_PROD_PV
L103.QCL_PRIMARY_PROD_PV

rm(QV1, QV)

QCL_PRIMARY_PROD_PV %>%
L103.QCL_PRIMARY_PROD_PV %>%
add_title("FAO crop and livestock production and crop area") %>%
add_units("USD and tonne") %>%
add_comments("Detailed FAO QCL data processing. FBS fish data is used") %>%
add_precursors("QCL_PROD",
add_precursors("L102.QCL_PROD",
"PP_wide") ->
QCL_PRIMARY_PROD_PV
L103.QCL_PRIMARY_PROD_PV

return_data(MODULE_OUTPUTS)

Expand Down
8 changes: 4 additions & 4 deletions R/xfaostat_L104_ProductionAreaAddFodder.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ module_xfaostat_L104_ProductionAreaAddFodder <- function(command, ...) {
"QCL_area_code_map")

MODULE_OUTPUTS <-
c("QCL_FODDERCROP")
c("L104.QCL_FODDERCROP")

if(command == driver.DECLARE_INPUTS) {
return(MODULE_INPUTS)
Expand Down Expand Up @@ -95,17 +95,17 @@ module_xfaostat_L104_ProductionAreaAddFodder <- function(command, ...) {
# Remove area x year that should no exist
FAOSTAT_AREA_RM_NONEXIST %>%
left_join(UnitMap, by = "element") ->
QCL_FODDERCROP
L104.QCL_FODDERCROP

rm(FAO_fodder_Prod_t_HA_ha_PRODSTAT_2011, UnitMap)

QCL_FODDERCROP %>%
L104.QCL_FODDERCROP %>%
add_title("Processed fodder crop production and area") %>%
add_units("tonne and ha") %>%
add_comments("Data is from old GCAM data v5.4") %>%
add_precursors(file.path(DIR_RAW_DATA_FAOSTAT, "Other_supplementary/FAO_fodder_Prod_t_HA_ha_PRODSTAT_2011"),
"QCL_area_code_map")->
QCL_FODDERCROP
L104.QCL_FODDERCROP


# FAO_ag_items_PRODSTAT is not read in anymore
Expand Down
Loading

0 comments on commit 097c700

Please sign in to comment.