From f53d5ac8d89596bffe726dd6fccf7e161046a1db Mon Sep 17 00:00:00 2001
From: Jan Engel MPI-BGC <jengel@bgc-jena.mpg.de>
Date: Wed, 5 Jun 2024 19:31:34 +0200
Subject: [PATCH 1/3] modify data preprocessing script and related functions to
work with 2024
* not yet tested
---
.../functions/create_meas_period_metadata_df.R | 6 ++++++
...lot_ghg_conc_2023.R => plot_ghg_conc_2023_2024.R} | 0
...ead_raw_data_2023.R => read_raw_data_2023_2024.R} | 4 ++++
...23.R => sort_n2o_conc_by_meas_period_2023_2024.R} | 0
preprocessing/main_data_preprocessing.R | 12 ++++++------
preprocessing/run_all_main.R | 8 ++++----
6 files changed, 20 insertions(+), 10 deletions(-)
rename preprocessing/functions/{plot_ghg_conc_2023.R => plot_ghg_conc_2023_2024.R} (100%)
rename preprocessing/functions/{read_raw_data_2023.R => read_raw_data_2023_2024.R} (98%)
rename preprocessing/functions/{sort_n2o_conc_by_meas_period_2023.R => sort_n2o_conc_by_meas_period_2023_2024.R} (100%)
diff --git a/preprocessing/functions/create_meas_period_metadata_df.R b/preprocessing/functions/create_meas_period_metadata_df.R
index d10bdaa..eb6c1e2 100644
--- a/preprocessing/functions/create_meas_period_metadata_df.R
+++ b/preprocessing/functions/create_meas_period_metadata_df.R
@@ -33,6 +33,12 @@ create_measPeriodMetaDataDF <- function(expYear, measCamp, field_mp_meta_data_DF
measCampInt <- 2023070000
} else if(measCamp == "202309"){
measCampInt <- 2023090000
+ } else if(measCamp == "202406"){
+ measCampInt <- 2024060000
+ } else if(measCamp == "202407"){
+ measCampInt <- 2024070000
+ } else if(measCamp == "202408"){
+ measCampInt <- 2024080000
} else {
# error, unknown measCamp
print("")
diff --git a/preprocessing/functions/plot_ghg_conc_2023.R b/preprocessing/functions/plot_ghg_conc_2023_2024.R
similarity index 100%
rename from preprocessing/functions/plot_ghg_conc_2023.R
rename to preprocessing/functions/plot_ghg_conc_2023_2024.R
diff --git a/preprocessing/functions/read_raw_data_2023.R b/preprocessing/functions/read_raw_data_2023_2024.R
similarity index 98%
rename from preprocessing/functions/read_raw_data_2023.R
rename to preprocessing/functions/read_raw_data_2023_2024.R
index 0ce45f1..ee8c50c 100644
--- a/preprocessing/functions/read_raw_data_2023.R
+++ b/preprocessing/functions/read_raw_data_2023_2024.R
@@ -21,6 +21,8 @@ get_datalogger_raw_data <- function(mainDir, dataDir, measCamp){
inputFile <- "20230720-27_stordalen_datalogger_data.csv" # äuä nid vouständig
} else if (measCamp == "202309"){
inputFile <- "20230922_stordalen_datalogger_data.csv"
+ } else if (measCamp == "202406"){
+ inputFile <- "202406_test2309_stordalen_datalogger_data.csv"
}
inputFile <- paste0(dataDir, "/" , measCamp, "/", inputFile)
datalogger_raw_data <- read.csv(inputFile)
@@ -159,6 +161,8 @@ get_field_mp_meta_data_DF <- function(mainDir, dataDir, measCamp){
inputFile <- "20230720-27_field-data.csv"
} else if (measCamp == "202309"){
inputFile <- "202309_field-data.csv"
+ } else if (measCamp == "202406"){
+ inputFile <- "202406_test2309_field-data.csv"
}
inputFile <- paste0(dataDir, "/" , measCamp, "/", inputFile)
diff --git a/preprocessing/functions/sort_n2o_conc_by_meas_period_2023.R b/preprocessing/functions/sort_n2o_conc_by_meas_period_2023_2024.R
similarity index 100%
rename from preprocessing/functions/sort_n2o_conc_by_meas_period_2023.R
rename to preprocessing/functions/sort_n2o_conc_by_meas_period_2023_2024.R
diff --git a/preprocessing/main_data_preprocessing.R b/preprocessing/main_data_preprocessing.R
index ab34199..75bba7b 100644
--- a/preprocessing/main_data_preprocessing.R
+++ b/preprocessing/main_data_preprocessing.R
@@ -41,8 +41,8 @@ if (expYear == "2022"){
source("functions/read_raw_data_2022.R")
stop("improve and add function for 2022 read raw data")
-} else if (expYear == "2023"){
- source("functions/read_raw_data_2023.R")
+} else if (expYear == "2023" | expYear == "2024"){
+ source("functions/read_raw_data_2023_2024.R")
# get measurement data
# ch4 = methane, co2 = carbon dioxide, conc = concentration,
# st = soil temperature, sm = soil moisture, n2o = nitrous oxide
@@ -90,7 +90,7 @@ if (expYear == "2022"){
gas_ID$n2o_aeris)
}
#--- 2023 -----------------------------------------------------------------------
-} else if (expYear == "2023"){
+} else if (expYear == "2023" | expYear == "2024"){
if (process_co2 == "T"){
source("functions/sort_co2_conc_by_meas_period_2023.R")
# create DF with licor CO2 measured data, sorted by meas_ID
@@ -113,7 +113,7 @@ if (expYear == "2022"){
}
if (process_n2o == "T"){
- source("functions/sort_n2o_conc_by_meas_period_2023.R")
+ source("functions/sort_n2o_conc_by_meas_period_2023_2024.R")
# create DF with Aeris N2O measured data, sorted by meas_ID
list_hlp <- f_create_n2o_conc_sorted_by_measurements(field_mp_meta_data_DF,
measPeriodMetaDataDF,
@@ -143,8 +143,8 @@ if (create_plots_ghg_conc == "T"){
if (expYear == "2022"){
source("functions/plot_ghg_conc.R")
#--- 2023 -----------------------------------------------------------------------
- } else if (expYear == "2023"){
- source("functions/plot_ghg_conc_2023.R")
+ } else if (expYear == "2023" | expYear == "2024"){
+ source("functions/plot_ghg_conc_2023_2024.R")
# N2O
if (process_n2o == "T"){
f_plot_ghg_conc_2023_n2o_non_mod_mp(figsGasConcDir,
diff --git a/preprocessing/run_all_main.R b/preprocessing/run_all_main.R
index c5139b2..0248956 100644
--- a/preprocessing/run_all_main.R
+++ b/preprocessing/run_all_main.R
@@ -13,10 +13,10 @@
# 1. Script settings ----
# list of measurement campaigns to run [YYYYMM] and [month]
-measCampList <- c("202309")
-measCampMonthList <- c("September") # must be consistent with 'measCampList'
-#measCampList <- c("202305", "202307", "202309")
-#measCampMonthList <- c("May", "July", "September")
+measCampList <- c("202406")
+measCampMonthList <- c("June") # must be consistent with 'measCampList'
+#measCampList <- c("202305", "202307", "202309", "202406", "202407", "202408")
+#measCampMonthList <- c("May", "July", "September", "June", "July", "August")
# gas type to process [T,F]
--
GitLab
From ace271285e7509e5abec4fccca133c66039f9121 Mon Sep 17 00:00:00 2001
From: Jan Engel MPI-BGC <jengel@bgc-jena.mpg.de>
Date: Wed, 5 Jun 2024 20:08:26 +0200
Subject: [PATCH 2/3] add new "additional file" providing input-data filesnames
* for years 2023 & 2024 only
* new file: preprocessing/additional_info/input_data_filenames.txt
* sourced in: functions/read_raw_data_2023_2024.R
* not yet tested
---
.../additional_info/input_data_filenames.txt | 25 +++++++++++++++
.../functions/read_raw_data_2023_2024.R | 32 ++++++++++++++-----
2 files changed, 49 insertions(+), 8 deletions(-)
create mode 100644 preprocessing/additional_info/input_data_filenames.txt
diff --git a/preprocessing/additional_info/input_data_filenames.txt b/preprocessing/additional_info/input_data_filenames.txt
new file mode 100644
index 0000000..77b4737
--- /dev/null
+++ b/preprocessing/additional_info/input_data_filenames.txt
@@ -0,0 +1,25 @@
+#
+# filenames of datalogger-data and meta-data input files for each individual measurement campaign
+#
+# this file is sourced in: functions/read_raw_data_2023_2024.R
+#
+
+## datalogger data
+### 2023
+dataLoggerDataInputFile_202305 <- "20230511-30_stordalen_datalogger_data.csv"
+dataLoggerDataInputFile_202307 <- "20230720-27_stordalen_datalogger_data.csv" # äuä nid vouständig
+dataLoggerDataInputFile_202309 <- "20230922_stordalen_datalogger_data.csv"
+### 2024
+dataLoggerDataInputFile_202406 <- "202406_test2309_stordalen_datalogger_data.csv"
+dataLoggerDataInputFile_202407 <- "no_file_yet"
+dataLoggerDataInputFile_202408 <- "no_file_yet"
+
+## meta data
+### 2023
+metaDataInputFile_202305 <- "202305_field-data.csv"
+metaDataInputFile_202307 <- "20230720-27_field-data.csv"
+metaDataInputFile_202309 <- "202309_field-data.csv"
+### 2024
+metaDataInputFile_202406 <- "202406_test2309_field-data.csv"
+metaDataInputFile_202407 <- "no_file_yet"
+metaDataInputFile_202408 <- "no_file_yet"
\ No newline at end of file
diff --git a/preprocessing/functions/read_raw_data_2023_2024.R b/preprocessing/functions/read_raw_data_2023_2024.R
index ee8c50c..6e605f2 100644
--- a/preprocessing/functions/read_raw_data_2023_2024.R
+++ b/preprocessing/functions/read_raw_data_2023_2024.R
@@ -12,17 +12,25 @@ get_datalogger_raw_data <- function(mainDir, dataDir, measCamp){
print("start get_datalogger_raw_data")
+
+ # get data-input filenames
+ source("../additional_info/input_data_filenames.txt")
+
# import data from data logger (CO2 + CH4 fluxes, soil moisture and temp data)
if (measCamp == "202209"){
inputFile <- "20221101_stordalen_ch4-co2.csv"
} else if (measCamp == "202305"){
- inputFile <- "20230511-30_stordalen_datalogger_data.csv"
+ inputFile <- dataLoggerDataInputFile_202305
} else if (measCamp == "202307"){
- inputFile <- "20230720-27_stordalen_datalogger_data.csv" # äuä nid vouständig
+ inputFile <- dataLoggerDataInputFile_202307 # äuä nid vouständig
} else if (measCamp == "202309"){
- inputFile <- "20230922_stordalen_datalogger_data.csv"
+ inputFile <- dataLoggerDataInputFile_202309
} else if (measCamp == "202406"){
- inputFile <- "202406_test2309_stordalen_datalogger_data.csv"
+ inputFile <- dataLoggerDataInputFile_202406
+ } else if (measCamp == "202407"){
+ inputFile <- dataLoggerDataInputFile_202407
+ } else if (measCamp == "202408"){
+ inputFile <- dataLoggerDataInputFile_202408
}
inputFile <- paste0(dataDir, "/" , measCamp, "/", inputFile)
datalogger_raw_data <- read.csv(inputFile)
@@ -153,16 +161,24 @@ get_aeris_raw_data <- function(mainDir, dataDir, measCamp){
get_field_mp_meta_data_DF <- function(mainDir, dataDir, measCamp){
print("start get_field_mp_meta_data_DF")
+
+ # get data-input filenames
+ source("../additional_info/input_data_filenames.txt")
+
if (measCamp == "202209"){
inputFile <- "202208-09_field-data.csv"
} else if (measCamp == "202305"){
- inputFile <- "202305_field-data.csv"
+ inputFile <- metaDataInputFile_202305
} else if (measCamp == "202307"){
- inputFile <- "20230720-27_field-data.csv"
+ inputFile <- metaDataInputFile_202307
} else if (measCamp == "202309"){
- inputFile <- "202309_field-data.csv"
+ inputFile <- metaDataInputFile_202309
} else if (measCamp == "202406"){
- inputFile <- "202406_test2309_field-data.csv"
+ inputFile <- metaDataInputFile_202406
+ } else if (measCamp == "202407"){
+ inputFile <- metaDataInputFile_202407
+ } else if (measCamp == "202408"){
+ inputFile <- metaDataInputFile_202408
}
inputFile <- paste0(dataDir, "/" , measCamp, "/", inputFile)
--
GitLab
From 54be9baa3bd886706f41802cc04daf72db014e7d Mon Sep 17 00:00:00 2001
From: Jan Engel MPI-BGC <jengel@bgc-jena.mpg.de>
Date: Wed, 5 Jun 2024 20:12:15 +0200
Subject: [PATCH 3/3] correction for previous commit: file path
---
preprocessing/functions/read_raw_data_2023_2024.R | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/preprocessing/functions/read_raw_data_2023_2024.R b/preprocessing/functions/read_raw_data_2023_2024.R
index 6e605f2..3623a3d 100644
--- a/preprocessing/functions/read_raw_data_2023_2024.R
+++ b/preprocessing/functions/read_raw_data_2023_2024.R
@@ -14,7 +14,7 @@ get_datalogger_raw_data <- function(mainDir, dataDir, measCamp){
print("start get_datalogger_raw_data")
# get data-input filenames
- source("../additional_info/input_data_filenames.txt")
+ source("additional_info/input_data_filenames.txt")
# import data from data logger (CO2 + CH4 fluxes, soil moisture and temp data)
if (measCamp == "202209"){
@@ -163,7 +163,7 @@ get_field_mp_meta_data_DF <- function(mainDir, dataDir, measCamp){
print("start get_field_mp_meta_data_DF")
# get data-input filenames
- source("../additional_info/input_data_filenames.txt")
+ source("additional_info/input_data_filenames.txt")
if (measCamp == "202209"){
inputFile <- "202208-09_field-data.csv"
--
GitLab