I fixed some computation problems. When I directly combined the SPAM datasets...

I fixed some computation problems. When I directly combined the SPAM datasets before running all the aggregation functions on them, my computer with 100 GB ram could not process it. So I changed it back to the old version with single files and then at last combine the datasets. Now it worked. Issue: No difference when disaggregating with harv and prod propotions on final values. Check why!

SPAM_Data_Transformation/SPAM2010_aggregationADM1.R

@@ -210,7 +210,7 @@ ADM1_SPAM_GGS <- st_as_sf(x = ADM1_SPAM_GGS,
 
 
 # Writing Adm1 zone files tabular and spatial -----------------------------
-# write_csv(FINAL_adm1_spam2010_GGS_distinct, here::here("Input_data", "Core_data",  "ADM1_SPAM2010_GGS.csv"))
+write_csv(FINAL_adm1_spam2010_GGS_distinct, here::here("Input_data", "Core_data",  "ADM1_SPAM2010_GGS.csv"))
 # st_write(ADM1_SPAM_GGS, here::here("Input_data", "Core_data",  "ADM1_SPAM2010_GGS_geo.shp"),
 # driver = "ESRI Shapefile", options = "ENCODING=UTF-8")
 
@@ -350,10 +350,10 @@ treenut_pd = lst_ex_pd[[2]] %>% as_tibble() %>%
   
   
 # validation of aggregation 
-Validate_agg = xcv %>%dplyr::select(tnuts_vec, iso3, name_adm1) %>% 
-  pivot_longer(cols = starts_with("sum."),
-               names_to = "Nuts", 
-               values_to = "value")
+# Validate_agg = xcv %>%dplyr::select(tnuts_vec, iso3, name_adm1) %>% 
+  # pivot_longer(cols = starts_with("sum."),
+               # names_to = "Nuts", 
+               # values_to = "value")
 
 # calcluating the harvested area values for treenuts --------------------------
 treenut_ha = lst_ex_ha[[2]] %>% as_tibble() %>% 
@@ -432,7 +432,9 @@ TnutsRatio_ha_pd_GGS = prod.tib.adm1 %>%
   dplyr::rename(iso3 = iso3.x, name_adm1 = name_adm1.x) %>%
   dplyr::select(iso3, name_adm1, cell5m, tnuts_ratio_pd, tnuts_ratio_ha)
 
-
+# write and read treenuts data ----------------------------------------------
+write_csv(TnutsRatio_ha_pd_GGS, here::here("Input_data", "ratio_tnuts_ha_pd.csv"))
+TnutsRatio_ha_pd_GGS= read_csv(here::here("Input_data", "ratio_tnuts_ha_pd.csv"))#,locale = locale(encoding = "WINDOWS-1252"))
 
 # VEGGIES DISAGGREGATION  -------------------------------------------------
 # Loading veggie raster files  ------------------------------------------------
@@ -442,7 +444,7 @@ TnutsRatio_ha_pd_GGS = prod.tib.adm1 %>%
 #                      "yield" = "_YieldPerHectare\\.tif",
 #                      "area" = "_HarvestedAreaHectares\\.tif",
 #                      "prod" = "_Production\\.tif")
-#   path <- list.files(path = path, 
+#   path <- list.files(path = path,
 #                      full.names = TRUE,
 #                      pattern = pattern)
 #   l_raster <- lapply(path, raster::raster)
@@ -450,16 +452,16 @@ TnutsRatio_ha_pd_GGS = prod.tib.adm1 %>%
 #   # names(l_raster) <- crop_names
 #   return(l_raster)
 # }
-# 
-# # Exctracting veggies and rest area and production value per adm1 zones 
+# # 
+# # Exctracting veggies and rest area and production value per adm1 zones
 # FunZone_sum=function(rast){ # raster and vector
-#   c=zonal.stats(ADM1_SPAM_GGS, rast, stats = c("sum")) 
+#   c=zonal.stats(ADM1_SPAM_GGS, rast, stats = c("sum"))
 #   c$name_adm1 <- ADM1_SPAM_GGS$name_adm1  # assigning the identifier of the original df to the extracted df
 #   dat <- left_join(ADM1_SPAM_GGS, c, by = 'name_adm1') # merge the two files together
 # }
-
+# 
 # FUN_select_values = function(DF_list){
-#   dplyr::select(DF_list, starts_with("sum")) %>% as_tibble() %>% 
+#   dplyr::select(DF_list, starts_with("sum")) %>% as_tibble() %>%
 #     dplyr::select(-geometry)
 # }
 
@@ -606,8 +608,8 @@ veggie_ha = lst_ex_ha_veg[[2]] %>% as_tibble() %>%
 # veggie_ha = read_sf(here::here("Input_data", "GSS_veggie_pd.shp"), options = "ENCODING=WINDOWS-1252")
 # veggie_pd = read_sf(here::here("Input_data", "GSS_veggie_ha.shp"), options = "ENCODING=WINDOWS-1252")
   # # Converting to tibble
-  sph.prod.veg.tib = veggie_pd %>% as_tibble()
-  sph.harv.veg.tib = veggie_ha %>% as_tibble()
+  # sph.prod.veg.tib = veggie_pd %>% as_tibble()
+  # sph.harv.veg.tib = veggie_ha %>% as_tibble()
   
   # Loading Shapefile GGS
   ADM1_SPAM_GGS = st_read(here::here("Input_data", "Core_data",  "ADM1_SPAM2010_GGS_geo.shp"), options = "ENCODING=WINDOWS-1252")
@@ -650,7 +652,37 @@ ratio_veg_ha_pd = prod.tib.veg.adm1 %>%
 
 # write and read veggie data ----------------------------------------------
 write_csv(ratio_veg_ha_pd, here::here("Input_data", "ratio_veg_ha_pd.csv"))
-ratio_veg_ha_pd_import= read_csv(here::here("Input_data", "ratio_veg_ha_pd.csv"),options = "ENCODING=WINDOWS-1252")
+ratio_veg_ha_pd = read_csv(here::here("Input_data", "ratio_veg_ha_pd.csv"))
+
+
+
+# Load SPAM single SPAM files (uses less RAM for pulling through t --------
+
+
+# Production
+dat_TA_pr <- read_csv("spam2010V2r0_global_P_TA.csv",locale = locale(encoding = "WINDOWS-1252")) # LATIN2 encoding is used for proper display of nordic and slavic names
+dat_TI_pr <- read_csv("spam2010V2r0_global_P_TI.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TH_pr <- read_csv("spam2010V2r0_global_P_TH.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TL_pr <- read_csv("spam2010V2r0_global_P_TL.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TS_pr <- read_csv("spam2010V2r0_global_P_TS.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TR_pr <- read_csv("spam2010V2r0_global_P_TR.csv",locale = locale(encoding = "WINDOWS-1252"))
+
+
+# Physical area--------------------------------
+dat_TA_pa <- read_csv("spam2010V2r0_global_A_TA.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TI_pa <- read_csv("spam2010V2r0_global_A_TI.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TH_pa <- read_csv("spam2010V2r0_global_A_TH.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TL_pa <- read_csv("spam2010V2r0_global_A_TL.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TS_pa <- read_csv("spam2010V2r0_global_A_TS.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TR_pa <- read_csv("spam2010V2r0_global_A_TR.csv",locale = locale(encoding = "WINDOWS-1252"))
+
+# Harvesting area----------------------------------
+dat_TA_ha <- read_csv("spam2010V2r0_global_H_TA.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TI_ha <- read_csv("spam2010V2r0_global_H_TI.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TH_ha <- read_csv("spam2010V2r0_global_H_TH.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TL_ha <- read_csv("spam2010V2r0_global_H_TL.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TS_ha <- read_csv("spam2010V2r0_global_H_TS.csv",locale = locale(encoding = "WINDOWS-1252"))
+dat_TR_ha <- read_csv("spam2010V2r0_global_H_TR.csv",locale = locale(encoding = "WINDOWS-1252"))
 
 #Aggregating SPAM to Adm1 level  -----------------------------------------------------
 ChangeNames <- function(x) { ## Renaming function
@@ -771,8 +803,8 @@ l %>% dplyr::select(-iso3.y) %>%
 TransformFun_prod <- function(d){
   p <- ChangeNames(d)
   u <- Adm1MergeSPAM_GSSFUN(p)
-  b <- TnutsFun_prod(u)
-  z <- VeggieFun_prod(b)
+  b <- TnutsFun_area(u)
+  z <- VeggieFun_area(b)
   l <- GatherFun(z)
   r <- AggregateFun_sum(l)
   y <- cell5mFUN(z,r)
@@ -799,7 +831,7 @@ TR_prd <- TransformFun_prod(dat_TR_pr)
 TS_prd <- TransformFun_prod(dat_TS_pr)
 lst.prd <- list(TA_prd, TH_prd, TI_prd, TL_prd, TR_prd, TS_prd)
 dat_prd <- do.call("rbind", lst.prd) # merging all tech types
-write_csv(dat_prd, here::here("Input_data", "Core_data", "production_all_rec_tech_adm1.csv")) #careful with encoding when writing and reading files.
+# write_csv(dat_prd, here::here("Input_data", "Core_data", "production_all_rec_tech_adm1.csv")) #careful with encoding when writing and reading files.
 
 #Applying transformation to physical_tech types
 TA_phy <- TransformFun_area(dat_TA_pa)
@@ -810,9 +842,9 @@ TR_phy <- TransformFun_area(dat_TR_pa)
 TS_phy <- TransformFun_area(dat_TS_pa)
 lst.phy <- list(TA_phy, TH_phy, TI_phy, TL_phy, TR_phy, TS_phy)
 dat_phy <- do.call("rbind", lst.phy) # merging all tech types
-write_csv(dat_phy, here::here("Input_data", "Core_data", "physicalarea_all_rec_tech_adm1.csv")) #careful with encoding when writing and reading files.
+# write_csv(dat_phy, here::here("Input_data", "Core_data", "physicalarea_all_rec_tech_adm1.csv")) #careful with encoding when writing and reading files.
 
-#Applying transformation to physical_tech types
+#Applying transformation to harvest area types
 TA_har <- TransformFun_area(dat_TA_ha)
 TH_har <- TransformFun_area(dat_TH_ha)
 TI_har <- TransformFun_area(dat_TI_ha)
@@ -821,23 +853,25 @@ TR_har <- TransformFun_area(dat_TR_ha)
 TS_har <- TransformFun_area(dat_TS_ha)
 lst.har <- list(TA_har, TH_har, TI_har, TL_har, TR_har, TS_har)
 dat_har <- do.call("rbind", lst.har) # merging all tech types
-write_csv(dat_har, here::here("Input_data", "Core_data", "harvestedarea_all_rec_tech_adm1.csv"))#careful with encoding when writing and reading files.
+# write_csv(dat_har, here::here("Input_data", "Core_data", "harvestedarea_all_rec_tech_adm1.csv"))#careful with encoding when writing and reading files.
 
 # Validating aggregation  -------------------------------------------------
 # 1. Sum up adm1 units and compare if you get same aggregation results  
 # 2. Do this for all rec and tech types 
-dat_har %>%  as_tibble()
-dat_phy %>%  as_tibble()
-dat_prd %>%  as_tibble()
+# dat_har %>%  as_tibble()
+# dat_phy %>%  as_tibble()
+# dat_prd %>%  as_tibble()
 
 
 # Independent -- Merging treenuts with SPAM -------------------------------
-prd_dat <- read_csv(here::here("Input_data", "Core_data", "production_all_rec_tech_adm1.csv"), locale = locale(encoding = "LATIN2")) 
-phy_dat <- read_csv(here::here("Input_data", "Core_data", "physicalarea_all_rec_tech_adm1.csv"), locale = locale(encoding = "LATIN2"))
-har_dat <- read_csv(here::here("Input_data", "Core_data", "harvestedarea_all_rec_tech_adm1.csv"), locale = locale(encoding = "LATIN2"))
+# prd_dat <- read_csv(here::here("Input_data", "Core_data", "production_all_rec_tech_adm1.csv"))#, locale = locale(encoding = "LATIN2"))
+# phy_dat <- read_csv(here::here("Input_data", "Core_data", "physicalarea_all_rec_tech_adm1.csv"))#, locale = locale(encoding = "LATIN2"))
+# har_dat <- read_csv(here::here("Input_data", "Core_data", "harvestedarea_all_rec_tech_adm1.csv"))#, locale = locale(encoding = "LATIN2"))
 
 # Final global transformed SPAM2010 dataset. This df is in lign with GGS adm1 units.
 SPAM_GGS_all = rbind(dat_prd, dat_har, dat_phy)
+
+# Changing format and calculating yields
 dat_spam_global = SPAM_GGS_all %>%
   pivot_wider(!unit,
               names_from = "rec_type", 
@@ -846,7 +880,7 @@ dat_spam_global = SPAM_GGS_all %>%
   rowwise() %>% 
   mutate(Y = (P*1000/H))%>% #checked for result. We are transforming mt to kg  when multiplying Production by 1000 to get unit kg/ha for yields
   mutate_at(vars(Y), ~replace(., is.nan(.), 0))
-write.csv(dat_spam_global, here::here("Input_data", "Core_data", "SPAM_GGS_all_global_recspread.csv"))
+write.csv(dat_spam_global, here::here("Input_data", "Core_data", "SPAM_GGS_all_global_recspread_harvrationly.csv"))
 
 
 # Subsetting EU -----------------------------------------------------------