diff --git a/R/get_bioclim_data.R b/R/get_bioclim_data.R new file mode 100644 index 0000000..eaf7fc4 --- /dev/null +++ b/R/get_bioclim_data.R @@ -0,0 +1,52 @@ +#' Title +#' +#' @param output_dir +#' @param output_filename +#' @param raster_template +#' +#' @return +#' @export +#' +#' @examples +get_bioclim_data <- function(output_dir, + output_filename, + raster_template) { + + template <- terra::unwrap(raster_template) + bioclim_data <- geodata::worldclim_global(var = "bio", res = 2.5, path = output_dir) + + bioclim_data <- transform_raster(bioclim_data, template) + + bioclim_names <- c( + "Annual_Mean_Temperature", "Mean_Diurnal_Range", "Isothermality", + "Temperature_Seasonality", "Max_Temperature_of_Warmest_Month", + "Min_Temperature_of_Coldest_Month", "Temperature_Annual_Range", + "Mean_Temperature_of_Wettest_Quarter", "Mean_Temperature_of_Driest_Quarter", + "Mean_Temperature_of_Warmest_Quarter", "Mean_Temperature_of_Coldest_Quarter", + "Annual_Precipitation", "Precipitation_of_Wettest_Month", + "Precipitation_of_Driest_Month", "Precipitation_Seasonality", + "Precipitation_of_Wettest_Quarter", "Precipitation_of_Driest_Quarter", + "Precipitation_of_Warmest_Quarter", "Precipitation_of_Coldest_Quarter") + + # Assign the new names to the layers + names(bioclim_data) <- bioclim_names + + filename = paste(output_dir, output_filename, sep = "/") + + if(grepl("\\.parquet", filename)) { + # Convert to dataframe + dat <- as.data.frame(bioclim_data, xy = TRUE) |> as_tibble() + + # Save as parquet + arrow::write_parquet(dat, filename, compression = "gzip", compression_level = 5) + + terra::writeRaster(bioclim_data, filename=gsub("parquet", "tif", filename), overwrite=T, gdal=c("COMPRESS=LZW")) + + } else { + terra::writeRaster(bioclim_data, filename=filename, overwrite=T, gdal=c("COMPRESS=LZW")) + } + + unlink(paste(output_dir, "climate", sep = "/"), recursive=TRUE) + + return(filename) +} \ No newline at end of file diff --git a/R/get_elevation.R b/R/get_elevation.R deleted file mode 100644 index 2061e2c..0000000 --- a/R/get_elevation.R +++ /dev/null @@ -1,38 +0,0 @@ -#' .. content for \description{} (no empty lines) .. -#' -#' .. content for \details{} .. -#' -#' @title -#' @param -#' @return -#' @author Whitney Bagge, Nathan Layman -#' @export -library(paws) -get_elevation<- function(output_dir, continent_raster_template) { - - # Create directory if it does not yet exist - dir.create(output_dir, recursive = TRUE, showWarnings = FALSE) - - template <- terra::unwrap(continent_raster_template) - sf_use_s2(FALSE) - - CopernicusDEM::aoi_geom_save_tif_matches(sf_or_file = sf::st_as_sfc(sf::st_bbox(template)), - dir_save_tifs = output_dir, - resolution = 90, - crs_value = 4326, - threads = parallel::detectCores(), - verbose = TRUE) - - # Read big raster into memory - elevation_rast <- terra::rast(filename) - - # Save as compressed raster - terra::writeRaster(gdal=c("COMPRESS=LZW") - - # Clean up big uncompressed raster to save hard drive space - unlink(filename) - - # Return path to compressed raster - return(filename) - -} \ No newline at end of file diff --git a/R/get_elevation_data.R b/R/get_elevation_data.R new file mode 100644 index 0000000..3987f6c --- /dev/null +++ b/R/get_elevation_data.R @@ -0,0 +1,46 @@ +#' Title +#' +#' @param output_dir +#' @param output_filename +#' @param raster_template +#' +#' @return +#' @export +#' +#' @examples +get_elevation_data <- function(output_dir, + output_filename, + raster_template) { + + # Create directory if it does not yet exist + dir.create(output_dir, recursive = TRUE, showWarnings = FALSE) + + template <- terra::unwrap(raster_template) + + # Create a bounding bounding box template + elevation_data <- geodata::elevation_global(res = 0.5 , + path = output_dir) + + elevation_data <- transform_raster(elevation_data, + template = template) + + filename = paste(output_dir, output_filename, sep = "/") + + if(grepl("\\.parquet", filename)) { + # Convert to dataframe + dat <- as.data.frame(elevation_data, xy = TRUE) |> as_tibble() + + # Save as parquet + arrow::write_parquet(dat, filename, compression = "gzip", compression_level = 5) + + terra::writeRaster(elevation_data, filename=gsub("parquet", "tif", filename), overwrite=T, gdal=c("COMPRESS=LZW")) + + } else { + terra::writeRaster(elevation_data, filename=filename, overwrite=T, gdal=c("COMPRESS=LZW")) + } + + unlink(paste(output_dir, "elevation", sep = "/"), recursive=TRUE) + + # Return path to compressed raster + return(filename) +} \ No newline at end of file diff --git a/R/get_landcover_data.R b/R/get_landcover_data.R new file mode 100644 index 0000000..7932a42 --- /dev/null +++ b/R/get_landcover_data.R @@ -0,0 +1,39 @@ +get_landcover_data <- function(output_dir, + output_filename, + raster_template) { + + template <- terra::unwrap(raster_template) + + landcover_types <- c("trees", "grassland", "shrubs", "cropland", "built", "bare", "snow", "water", "wetland", "mangroves", "moss") + + # Fetch each layer, process them and stack them into a single SpatRaster + # Cleaning up files as we go to save space. + landcover_data <- map(landcover_types, function(l) { + landcover <- geodata::landcover(var = l, path = output_dir) + file <- sources(landcover) + landcover <- transform_raster(landcover, template) + unlink(file) + landcover + }) + + landcover_data <- do.call(c, landcover_data) + + filename = paste(output_dir, output_filename, sep = "/") + + if(grepl("\\.parquet", filename)) { + # Convert to dataframe + dat <- as.data.frame(landcover_data, xy = TRUE) |> as_tibble() + + # Save as parquet + arrow::write_parquet(dat, filename, compression = "gzip", compression_level = 5) + + terra::writeRaster(landcover_data, filename=gsub("parquet", "tif", filename), overwrite=T, gdal=c("COMPRESS=LZW")) + + } else { + terra::writeRaster(landcover_data, filename=filename, overwrite=T, gdal=c("COMPRESS=LZW")) + } + + unlink(paste(output_dir, "landuse", sep = "/"), recursive=TRUE) + + return(filename) +} \ No newline at end of file diff --git a/R/preprocess_soil.R b/R/preprocess_soil.R index bb0f85a..84689a6 100644 --- a/R/preprocess_soil.R +++ b/R/preprocess_soil.R @@ -9,7 +9,10 @@ #' @export library(DBI) library(RSQLite) -preprocess_soil <- function(soil_directory_dataset, soil_directory_raw, continent_raster_template, soil_downloaded) { +preprocess_soil <- function(soil_directory_dataset, + soil_directory_raw, + continent_raster_template, + soil_downloaded) { #read in the raster file @@ -86,6 +89,7 @@ preprocess_soil <- function(soil_directory_dataset, soil_directory_raw, continen texture = as.numeric(records$TEXTURE_USDA)) #classify the raster (transformed_raster) using the matrix of values - TEXTURE CLASS + # CLASIFFY DOESN'T SEEM TO BE WORKING LEFT OFF HERE hwsd.zhnj.texture <- classify(transformed_raster, rcl.matrix.texture) hwsd.zhnj.texture <- as.factor(hwsd.zhnj.texture) levels(hwsd.zhnj.texture) <- levels(records$TEXTURE_USDA) diff --git a/_targets.R b/_targets.R index 90ffcb6..4fff0d3 100644 --- a/_targets.R +++ b/_targets.R @@ -46,7 +46,6 @@ static_targets <- tar_plan( # modis ndvi = 0.01 tar_target(rsa_polygon, rgeoboundaries::geoboundaries("South Africa", "adm2")), - # SOIL ----------------------------------------------------------- tar_target(soil_directory_raw, create_data_directory(directory_path = "data/soil")), @@ -84,7 +83,6 @@ static_targets <- tar_plan( "slope_thirty" = "https://www.fao.org/fileadmin/user_upload/soils/HWSD%20Viewer/GloSlopesCl7_30as.rar", "slope_fortyfive" = "https://www.fao.org/fileadmin/user_upload/soils/HWSD%20Viewer/GloSlopesCl8_30as.rar")), - # Slope is the fraction of cells tar_target(slope_preprocessed, get_remote_rasters(urls = slope_urls, output_dir = "data/slope_dataset", output_filename = "slope.parquet", @@ -105,24 +103,34 @@ static_targets <- tar_plan( tar_target(glw_preprocessed, preprocess_glw_data(glw_directory_dataset, glw_directory_raw, glw_downloaded, continent_raster_template)), - # ELEVATION ----------------------------------------------------------- -tar_target(elevation_directory_raw, - create_data_directory(directory_path = "data/elevation")), -tar_target(elevation_downloaded, get_elevation(elevation_directory_raw, continent_raster_template, overwrite = FALSE), - format = "file", - repository = "local"), -tar_target(elevation_directory_dataset, - create_data_directory(directory_path = "data/elevation_dataset")), tar_target(elevation_preprocessed, - process_elevation(elevation_directory_dataset, elevation_downloaded, elevation_directory_raw, continent_raster_template)), + get_elevation_data(output_dir = "data/elevation_dataset", + output_filename = "africa_elevation.parquet", + raster_template = continent_raster_template), + format = "file", + repository = "local"), + +# BIOCLIM ----------------------------------------------------------- +tar_target(bioclim_preprocessed, + get_bioclim_data(output_dir = "data/bioclim_dataset", + output_filename = "bioclim.parquet", + raster_template = continent_raster_template), + format = "file", + repository = "local"), + +# LANDCOVER ----------------------------------------------------------- +tar_target(landcover_preprocessed, + get_landcover_data(output_dir = "data/landcover_dataset", + output_filename = "landcover.parquet", + raster_template = continent_raster_template), + format = "file", + repository = "local"), -# Any missing static layers? -# bioclim -# forest cover -# ) + + # Dynamic Data Download ----------------------------------------------------------- dynamic_targets <- tar_plan( diff --git a/_targets/meta/meta b/_targets/meta/meta index 9f6411d..6191c26 100644 --- a/_targets/meta/meta +++ b/_targets/meta/meta @@ -1,5 +1,5 @@ name|type|data|command|depend|seed|path|time|size|bytes|format|repository|iteration|parent|children|seconds|warnings|error -.Random.seed|object|41fc3c40921ed810||||||||||||||| +.Random.seed|object|e60e22eacf4aeb0d||||||||||||||| aggregate_augmented_data_by_adm|function|0ba2d0669462f71c||||||||||||||| aggregated_data_rsa|pattern|6dbb43f34bc42ac8|ea0a3280c45817ae||-1696630435||||137860|qs|aws|vector||aggregated_data_rsa_94f732f8*aggregated_data_rsa_5e501efa*aggregated_data_rsa_34327510*aggregated_data_rsa_40e9884c*aggregated_data_rsa_5ababfa3|288.617|| aggregated_data_rsa_34327510|branch|c1e4fd089c317f41|ea0a3280c45817ae|1b56fcad8b0f2922|1726064958|bucket=open-rvfcast-data*region=us-east-1*key=_targets/aggregated_data_rsa_34327510*endpoint=TlVMTA*version=|t19755.8889066061s||27572|qs|aws|vector|aggregated_data_rsa||56.413|| @@ -443,6 +443,7 @@ augmented_data_rsa_fe899acf|branch|0226f41a6223ab99|e34b3be3b303277e|0de1cf59847 augmented_data_rsa_feb1b830|branch|4ff2ba553efa663a|e34b3be3b303277e|f0a3cacbb7dbfdb4|1077197804|bucket=open-rvfcast-data*region=us-east-1*key=_targets/augmented_data_rsa_feb1b830*endpoint=TlVMTA*version=|t19755.8512676687s||97|qs|aws|vector|augmented_data_rsa||1.99|| aws_bucket|object|d9cf2c5ff7cc1be4||||||||||||||| aws_s3_upload_single_type|function|6d277b68ccbb67a2||||||||||||||| +bioclim_preprocessed|stem|1b1a48600ccae883|5f41665e5cc07c28|b47b73e50af4bc33|1289283391|data/bioclim_dataset/bioclim.parquet|t19924.6244828159s|70d824b00d5241b0|24881359|file|local|vector|||47.599|| cache_aws_branched_target|function|6e2abfa4969de1bf||||||||||||||| calc_daily_outbreak_history|function|b822309d11aabbee||||||||||||||| calc_outbreak_history|function|c294695b47f785d8||||||||||||||| @@ -553,8 +554,8 @@ ecmwf_forecasts_transformed_upload_aws_s3_f77cea8f|branch|6e170a6bc685cec2|af110 ecmwf_forecasts_transformed_upload_aws_s3_fe9ab7e6|branch|6e170a6bc685cec2|af1104937d66021a|d17be2bde46638f3|1048328926|bucket=open-rvfcast-data*region=NULL*key=_targets/ecmwf_forecasts_transformed_upload_aws_s3_fe9ab7e6*endpoint=TlVMTA*version=|t19661.0328043273s||34|qs|aws|vector|ecmwf_forecasts_transformed_upload_aws_s3||146.633|| elevation_directory_dataset|stem|707b9f535de2b985|36b8986f351764ad|3d3d9feae01275db|-1939550309|bucket=open-rvfcast-data*region=us-east-1*key=_targets/elevation_directory_dataset*endpoint=TlVMTA*version=|t19920.7241538229s||54|qs|aws|vector|||0.001|| elevation_directory_raw|stem|6de45bc98e62fe5a|43d984583322c2a0|3d3d9feae01275db|-1529505470|bucket=open-rvfcast-data*region=us-east-1*key=_targets/elevation_directory_raw*endpoint=TlVMTA*version=|t19920.7240788535s||46|qs|aws|vector|||0.002|| -elevation_downloaded|stem|0f7af5732eb7c6a0|5532e9bcd7843c33|161f165ebe9e09cd|881307834|data/elevation/srtm_africa.tif|t19900.8442125982s||1.11253697931774e-308|file|local|vector|||0.026||Path exists and overwrite is FALSE -elevation_preprocessed|stem||7545b182d2c6c248|e16674377db3bcf2|-449770024||t19898.8997716689s||0|qs|aws|vector|||0.054|TIFFFillTileRead error at row 4294967295, col 4294967295, tile 43264 got 0 bytes, expected 532 GDAL error 1. TIFFReadEncodedTile failed. GDAL error 1. UsersnathanlaymanDocumentsAcademiaEHAProjectsopenrvfcastdataelevationsrtm_africa.tif, band 1 IReadBlock failed at X offset 100, Y offset 66 TIFFReadEncodedTile failed. GDAL error 1|project warp failure +elevation_downloaded|stem|0f7af5732eb7c6a0|5532e9bcd7843c33|161f165ebe9e09cd|881307834|data/elevation/srtm_africa.tif|t19900.8442125982s||2.22505688254164e-308|file|local|vector|||0.026||Path exists and overwrite is FALSE +elevation_preprocessed|stem|e09f5fc8886199e9|41a249f0486dcfba|f611c49b2e065bc4|-449770024|data/elevation_dataset/africa_elevation.parquet|t19924.6304703577s|436096d7609465b1|1491699|file|local|vector|||20.353|| env_file|object|5e2c4c2bf6df65f0||||||||||||||| f|object|6298f8b06e6691ad||||||||||||||| filter_augmented_data|function|ed3fa76da2ee8f6f||||||||||||||| @@ -2268,8 +2269,9 @@ forecasts_anomalies_validate_upload_aws_s3_feb7dc70|branch|98d6911e9a3e27d2|b68e forecasts_anomalies_validate_upload_aws_s3_fecdee98|branch|1975df037a7d7371|b68e902edc98acf1|4d6164baab82b8cc|-393215332|bucket=open-rvfcast-data*region=us-east-1*key=_targets/forecasts_anomalies_validate_upload_aws_s3_fecdee98*endpoint=TlVMTA*version=|t19727.7280430236s||192|qs|aws|vector|forecasts_anomalies_validate_upload_aws_s3||12.115|| forecasts_anomalies_validate_upload_aws_s3_ff020a54|branch|b9dd4291cfa4d993|b68e902edc98acf1|3792874f79dfe901|1395503998|bucket=open-rvfcast-data*region=us-east-1*key=_targets/forecasts_anomalies_validate_upload_aws_s3_ff020a54*endpoint=TlVMTA*version=|t19727.7017176573s||192|qs|aws|vector|forecasts_anomalies_validate_upload_aws_s3||12.098|| forecasts_validate_directory|stem|316bbb9f1c700453|c3e4b2e8448616d5|3d3d9feae01275db|535511402|bucket=open-rvfcast-data*region=us-east-1*key=_targets/forecasts_validate_directory*endpoint=TlVMTA*version=|t19724.8348698892s||56|qs|aws|vector|||0|| +get_bioclim_data|function|b19d9f8efc5a9b82||||||||||||||| get_country_bounding_boxes|function|82b21d03b36ce8fe||||||||||||||| -get_elevation|function|c65279d2ecf0315f||||||||||||||| +get_elevation|function|8ade129c4b6565d6||||||||||||||| get_glw|function|c56e9ad7b5763189||||||||||||||| get_glw_data|function|e0d1d91328785495||||||||||||||| get_modis_ndvi_api_parameters|function|c30ed04e0978b176||||||||||||||| @@ -6555,8 +6557,8 @@ soil_directory_dataset|stem|5367f73442caebca|6b87e3cd8dc09f31|3d3d9feae01275db|1 soil_directory_raw|stem|6d0bef71eae7763c|7be6d1311e8674de|3d3d9feae01275db|781979019|bucket=open-rvfcast-data*region=us-east-1*key=_targets/soil_directory_raw*endpoint=TlVMTA*version=|t19920.7240877431s||40|qs|aws|vector|||0.002|| soil_download|function|371f9065fa1611a3||||||||||||||| soil_downloaded|stem|678485c3217017dc|73e8f917a50d005c|bc396cd8e73e5cc4|-1047214501|data/soil|t19920.7441717959s|4fa881245fd93336|320|file|local|vector|||14.281|| -soil_preprocessed|stem|5367f73442caebca|7eacd6654f0fbf4a|992a2f44982341eb|1256321187|bucket=open-rvfcast-data*region=us-east-1*key=_targets/soil_preprocessed*endpoint=TlVMTA*version=|t19920.7441778077s||49|qs|aws|vector|||1.407|set.cats setting categories like this is deprecated use a twocolumn data.frame instead. set.cats setting categories like this is deprecated use a twocolumn data.frame instead| -static_targets|object|e029f9dca75c665b||||||||||||||| +soil_preprocessed|stem|5367f73442caebca|7eacd6654f0fbf4a|d0ba406f5d022c00|1256321187|bucket=open-rvfcast-data*region=us-east-1*key=_targets/soil_preprocessed*endpoint=TlVMTA*version=|t19924.1920359525s||49|qs|aws|vector|||1.708|set.cats setting categories like this is deprecated use a twocolumn data.frame instead. set.cats setting categories like this is deprecated use a twocolumn data.frame instead| +static_targets|object|9b0f09cc6837cca2||||||||||||||| submit_modis_ndvi_bundle_request|function|70d5dcdcf3510fa0||||||||||||||| submit_modis_ndvi_request|function|e6d950c0c17bd8cd||||||||||||||| submit_modis_ndvi_task_request|function|a97a6292a38c2323||||||||||||||| diff --git a/data/elevation_dataset/.gitkeep b/data/elevation_dataset/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/data/slope_aspect_dataset/slope_dataset b/data/slope_aspect_dataset/slope_dataset new file mode 100644 index 0000000..44c4c0d Binary files /dev/null and b/data/slope_aspect_dataset/slope_dataset differ diff --git a/data/soil_dataset/soil_drainage b/data/soil_dataset/soil_drainage index e7cd82a..6d8f014 100644 Binary files a/data/soil_dataset/soil_drainage and b/data/soil_dataset/soil_drainage differ diff --git a/renv.lock b/renv.lock index eb8e339..c4506ec 100644 --- a/renv.lock +++ b/renv.lock @@ -996,6 +996,17 @@ ], "Hash": "15e9634c0fcd294799e9b2e929ed1b86" }, + "geodata": { + "Package": "geodata", + "Version": "0.6-2", + "Source": "Repository", + "Repository": "RSPM", + "Requirements": [ + "R", + "terra" + ], + "Hash": "4d43570de08ab49ba86bb2ccb6fe117d" + }, "gert": { "Package": "gert", "Version": "2.0.0",