Skip to content

Commit

Permalink
add in bioclim and landcover data
Browse files Browse the repository at this point in the history
  • Loading branch information
n8layman committed Jul 20, 2024
1 parent a8c90a6 commit e1891e6
Show file tree
Hide file tree
Showing 11 changed files with 184 additions and 60 deletions.
52 changes: 52 additions & 0 deletions R/get_bioclim_data.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#' Title
#'
#' @param output_dir
#' @param output_filename
#' @param raster_template
#'
#' @return
#' @export
#'
#' @examples
get_bioclim_data <- function(output_dir,
output_filename,
raster_template) {

template <- terra::unwrap(raster_template)
bioclim_data <- geodata::worldclim_global(var = "bio", res = 2.5, path = output_dir)

bioclim_data <- transform_raster(bioclim_data, template)

bioclim_names <- c(
"Annual_Mean_Temperature", "Mean_Diurnal_Range", "Isothermality",
"Temperature_Seasonality", "Max_Temperature_of_Warmest_Month",
"Min_Temperature_of_Coldest_Month", "Temperature_Annual_Range",
"Mean_Temperature_of_Wettest_Quarter", "Mean_Temperature_of_Driest_Quarter",
"Mean_Temperature_of_Warmest_Quarter", "Mean_Temperature_of_Coldest_Quarter",
"Annual_Precipitation", "Precipitation_of_Wettest_Month",
"Precipitation_of_Driest_Month", "Precipitation_Seasonality",
"Precipitation_of_Wettest_Quarter", "Precipitation_of_Driest_Quarter",
"Precipitation_of_Warmest_Quarter", "Precipitation_of_Coldest_Quarter")

# Assign the new names to the layers
names(bioclim_data) <- bioclim_names

filename = paste(output_dir, output_filename, sep = "/")

if(grepl("\\.parquet", filename)) {
# Convert to dataframe
dat <- as.data.frame(bioclim_data, xy = TRUE) |> as_tibble()

# Save as parquet
arrow::write_parquet(dat, filename, compression = "gzip", compression_level = 5)

terra::writeRaster(bioclim_data, filename=gsub("parquet", "tif", filename), overwrite=T, gdal=c("COMPRESS=LZW"))

} else {
terra::writeRaster(bioclim_data, filename=filename, overwrite=T, gdal=c("COMPRESS=LZW"))
}

unlink(paste(output_dir, "climate", sep = "/"), recursive=TRUE)

return(filename)
}
38 changes: 0 additions & 38 deletions R/get_elevation.R

This file was deleted.

46 changes: 46 additions & 0 deletions R/get_elevation_data.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#' Title
#'
#' @param output_dir
#' @param output_filename
#' @param raster_template
#'
#' @return
#' @export
#'
#' @examples
get_elevation_data <- function(output_dir,
output_filename,
raster_template) {

# Create directory if it does not yet exist
dir.create(output_dir, recursive = TRUE, showWarnings = FALSE)

template <- terra::unwrap(raster_template)

# Create a bounding bounding box template
elevation_data <- geodata::elevation_global(res = 0.5 ,
path = output_dir)

elevation_data <- transform_raster(elevation_data,
template = template)

filename = paste(output_dir, output_filename, sep = "/")

if(grepl("\\.parquet", filename)) {
# Convert to dataframe
dat <- as.data.frame(elevation_data, xy = TRUE) |> as_tibble()

# Save as parquet
arrow::write_parquet(dat, filename, compression = "gzip", compression_level = 5)

terra::writeRaster(elevation_data, filename=gsub("parquet", "tif", filename), overwrite=T, gdal=c("COMPRESS=LZW"))

} else {
terra::writeRaster(elevation_data, filename=filename, overwrite=T, gdal=c("COMPRESS=LZW"))
}

unlink(paste(output_dir, "elevation", sep = "/"), recursive=TRUE)

# Return path to compressed raster
return(filename)
}
39 changes: 39 additions & 0 deletions R/get_landcover_data.R
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
get_landcover_data <- function(output_dir,
output_filename,
raster_template) {

template <- terra::unwrap(raster_template)

landcover_types <- c("trees", "grassland", "shrubs", "cropland", "built", "bare", "snow", "water", "wetland", "mangroves", "moss")

# Fetch each layer, process them and stack them into a single SpatRaster
# Cleaning up files as we go to save space.
landcover_data <- map(landcover_types, function(l) {
landcover <- geodata::landcover(var = l, path = output_dir)
file <- sources(landcover)
landcover <- transform_raster(landcover, template)
unlink(file)
landcover
})

landcover_data <- do.call(c, landcover_data)

filename = paste(output_dir, output_filename, sep = "/")

if(grepl("\\.parquet", filename)) {
# Convert to dataframe
dat <- as.data.frame(landcover_data, xy = TRUE) |> as_tibble()

# Save as parquet
arrow::write_parquet(dat, filename, compression = "gzip", compression_level = 5)

terra::writeRaster(landcover_data, filename=gsub("parquet", "tif", filename), overwrite=T, gdal=c("COMPRESS=LZW"))

} else {
terra::writeRaster(landcover_data, filename=filename, overwrite=T, gdal=c("COMPRESS=LZW"))
}

unlink(paste(output_dir, "landuse", sep = "/"), recursive=TRUE)

return(filename)
}
6 changes: 5 additions & 1 deletion R/preprocess_soil.R
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@
#' @export
library(DBI)
library(RSQLite)
preprocess_soil <- function(soil_directory_dataset, soil_directory_raw, continent_raster_template, soil_downloaded) {
preprocess_soil <- function(soil_directory_dataset,
soil_directory_raw,
continent_raster_template,
soil_downloaded) {

#read in the raster file

Expand Down Expand Up @@ -86,6 +89,7 @@ preprocess_soil <- function(soil_directory_dataset, soil_directory_raw, continen
texture = as.numeric(records$TEXTURE_USDA))

#classify the raster (transformed_raster) using the matrix of values - TEXTURE CLASS
# CLASIFFY DOESN'T SEEM TO BE WORKING LEFT OFF HERE
hwsd.zhnj.texture <- classify(transformed_raster, rcl.matrix.texture)
hwsd.zhnj.texture <- as.factor(hwsd.zhnj.texture)
levels(hwsd.zhnj.texture) <- levels(records$TEXTURE_USDA)
Expand Down
38 changes: 23 additions & 15 deletions _targets.R
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ static_targets <- tar_plan(
# modis ndvi = 0.01
tar_target(rsa_polygon, rgeoboundaries::geoboundaries("South Africa", "adm2")),


# SOIL -----------------------------------------------------------
tar_target(soil_directory_raw,
create_data_directory(directory_path = "data/soil")),
Expand Down Expand Up @@ -84,7 +83,6 @@ static_targets <- tar_plan(
"slope_thirty" = "https://www.fao.org/fileadmin/user_upload/soils/HWSD%20Viewer/GloSlopesCl7_30as.rar",
"slope_fortyfive" = "https://www.fao.org/fileadmin/user_upload/soils/HWSD%20Viewer/GloSlopesCl8_30as.rar")),

# Slope is the fraction of cells
tar_target(slope_preprocessed, get_remote_rasters(urls = slope_urls,
output_dir = "data/slope_dataset",
output_filename = "slope.parquet",
Expand All @@ -105,24 +103,34 @@ static_targets <- tar_plan(
tar_target(glw_preprocessed,
preprocess_glw_data(glw_directory_dataset, glw_directory_raw, glw_downloaded, continent_raster_template)),


# ELEVATION -----------------------------------------------------------
tar_target(elevation_directory_raw,
create_data_directory(directory_path = "data/elevation")),
tar_target(elevation_downloaded, get_elevation(elevation_directory_raw, continent_raster_template, overwrite = FALSE),
format = "file",
repository = "local"),
tar_target(elevation_directory_dataset,
create_data_directory(directory_path = "data/elevation_dataset")),
tar_target(elevation_preprocessed,
process_elevation(elevation_directory_dataset, elevation_downloaded, elevation_directory_raw, continent_raster_template)),
get_elevation_data(output_dir = "data/elevation_dataset",
output_filename = "africa_elevation.parquet",
raster_template = continent_raster_template),
format = "file",
repository = "local"),

# BIOCLIM -----------------------------------------------------------
tar_target(bioclim_preprocessed,
get_bioclim_data(output_dir = "data/bioclim_dataset",
output_filename = "bioclim.parquet",
raster_template = continent_raster_template),
format = "file",
repository = "local"),

# LANDCOVER -----------------------------------------------------------
tar_target(landcover_preprocessed,
get_landcover_data(output_dir = "data/landcover_dataset",
output_filename = "landcover.parquet",
raster_template = continent_raster_template),
format = "file",
repository = "local"),

# Any missing static layers?
# bioclim
# forest cover
#

)


# Dynamic Data Download -----------------------------------------------------------
dynamic_targets <- tar_plan(

Expand Down
14 changes: 8 additions & 6 deletions _targets/meta/meta
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name|type|data|command|depend|seed|path|time|size|bytes|format|repository|iteration|parent|children|seconds|warnings|error
.Random.seed|object|41fc3c40921ed810|||||||||||||||
.Random.seed|object|e60e22eacf4aeb0d|||||||||||||||
aggregate_augmented_data_by_adm|function|0ba2d0669462f71c|||||||||||||||
aggregated_data_rsa|pattern|6dbb43f34bc42ac8|ea0a3280c45817ae||-1696630435||||137860|qs|aws|vector||aggregated_data_rsa_94f732f8*aggregated_data_rsa_5e501efa*aggregated_data_rsa_34327510*aggregated_data_rsa_40e9884c*aggregated_data_rsa_5ababfa3|288.617||
aggregated_data_rsa_34327510|branch|c1e4fd089c317f41|ea0a3280c45817ae|1b56fcad8b0f2922|1726064958|bucket=open-rvfcast-data*region=us-east-1*key=_targets/aggregated_data_rsa_34327510*endpoint=TlVMTA*version=|t19755.8889066061s||27572|qs|aws|vector|aggregated_data_rsa||56.413||
Expand Down Expand Up @@ -443,6 +443,7 @@ augmented_data_rsa_fe899acf|branch|0226f41a6223ab99|e34b3be3b303277e|0de1cf59847
augmented_data_rsa_feb1b830|branch|4ff2ba553efa663a|e34b3be3b303277e|f0a3cacbb7dbfdb4|1077197804|bucket=open-rvfcast-data*region=us-east-1*key=_targets/augmented_data_rsa_feb1b830*endpoint=TlVMTA*version=|t19755.8512676687s||97|qs|aws|vector|augmented_data_rsa||1.99||
aws_bucket|object|d9cf2c5ff7cc1be4|||||||||||||||
aws_s3_upload_single_type|function|6d277b68ccbb67a2|||||||||||||||
bioclim_preprocessed|stem|1b1a48600ccae883|5f41665e5cc07c28|b47b73e50af4bc33|1289283391|data/bioclim_dataset/bioclim.parquet|t19924.6244828159s|70d824b00d5241b0|24881359|file|local|vector|||47.599||
cache_aws_branched_target|function|6e2abfa4969de1bf|||||||||||||||
calc_daily_outbreak_history|function|b822309d11aabbee|||||||||||||||
calc_outbreak_history|function|c294695b47f785d8|||||||||||||||
Expand Down Expand Up @@ -553,8 +554,8 @@ ecmwf_forecasts_transformed_upload_aws_s3_f77cea8f|branch|6e170a6bc685cec2|af110
ecmwf_forecasts_transformed_upload_aws_s3_fe9ab7e6|branch|6e170a6bc685cec2|af1104937d66021a|d17be2bde46638f3|1048328926|bucket=open-rvfcast-data*region=NULL*key=_targets/ecmwf_forecasts_transformed_upload_aws_s3_fe9ab7e6*endpoint=TlVMTA*version=|t19661.0328043273s||34|qs|aws|vector|ecmwf_forecasts_transformed_upload_aws_s3||146.633||
elevation_directory_dataset|stem|707b9f535de2b985|36b8986f351764ad|3d3d9feae01275db|-1939550309|bucket=open-rvfcast-data*region=us-east-1*key=_targets/elevation_directory_dataset*endpoint=TlVMTA*version=|t19920.7241538229s||54|qs|aws|vector|||0.001||
elevation_directory_raw|stem|6de45bc98e62fe5a|43d984583322c2a0|3d3d9feae01275db|-1529505470|bucket=open-rvfcast-data*region=us-east-1*key=_targets/elevation_directory_raw*endpoint=TlVMTA*version=|t19920.7240788535s||46|qs|aws|vector|||0.002||
elevation_downloaded|stem|0f7af5732eb7c6a0|5532e9bcd7843c33|161f165ebe9e09cd|881307834|data/elevation/srtm_africa.tif|t19900.8442125982s||1.11253697931774e-308|file|local|vector|||0.026||Path exists and overwrite is FALSE
elevation_preprocessed|stem||7545b182d2c6c248|e16674377db3bcf2|-449770024||t19898.8997716689s||0|qs|aws|vector|||0.054|TIFFFillTileRead error at row 4294967295, col 4294967295, tile 43264 got 0 bytes, expected 532 GDAL error 1. TIFFReadEncodedTile failed. GDAL error 1. UsersnathanlaymanDocumentsAcademiaEHAProjectsopenrvfcastdataelevationsrtm_africa.tif, band 1 IReadBlock failed at X offset 100, Y offset 66 TIFFReadEncodedTile failed. GDAL error 1|project warp failure
elevation_downloaded|stem|0f7af5732eb7c6a0|5532e9bcd7843c33|161f165ebe9e09cd|881307834|data/elevation/srtm_africa.tif|t19900.8442125982s||2.22505688254164e-308|file|local|vector|||0.026||Path exists and overwrite is FALSE
elevation_preprocessed|stem|e09f5fc8886199e9|41a249f0486dcfba|f611c49b2e065bc4|-449770024|data/elevation_dataset/africa_elevation.parquet|t19924.6304703577s|436096d7609465b1|1491699|file|local|vector|||20.353||
env_file|object|5e2c4c2bf6df65f0|||||||||||||||
f|object|6298f8b06e6691ad|||||||||||||||
filter_augmented_data|function|ed3fa76da2ee8f6f|||||||||||||||
Expand Down Expand Up @@ -2268,8 +2269,9 @@ forecasts_anomalies_validate_upload_aws_s3_feb7dc70|branch|98d6911e9a3e27d2|b68e
forecasts_anomalies_validate_upload_aws_s3_fecdee98|branch|1975df037a7d7371|b68e902edc98acf1|4d6164baab82b8cc|-393215332|bucket=open-rvfcast-data*region=us-east-1*key=_targets/forecasts_anomalies_validate_upload_aws_s3_fecdee98*endpoint=TlVMTA*version=|t19727.7280430236s||192|qs|aws|vector|forecasts_anomalies_validate_upload_aws_s3||12.115||
forecasts_anomalies_validate_upload_aws_s3_ff020a54|branch|b9dd4291cfa4d993|b68e902edc98acf1|3792874f79dfe901|1395503998|bucket=open-rvfcast-data*region=us-east-1*key=_targets/forecasts_anomalies_validate_upload_aws_s3_ff020a54*endpoint=TlVMTA*version=|t19727.7017176573s||192|qs|aws|vector|forecasts_anomalies_validate_upload_aws_s3||12.098||
forecasts_validate_directory|stem|316bbb9f1c700453|c3e4b2e8448616d5|3d3d9feae01275db|535511402|bucket=open-rvfcast-data*region=us-east-1*key=_targets/forecasts_validate_directory*endpoint=TlVMTA*version=|t19724.8348698892s||56|qs|aws|vector|||0||
get_bioclim_data|function|b19d9f8efc5a9b82|||||||||||||||
get_country_bounding_boxes|function|82b21d03b36ce8fe|||||||||||||||
get_elevation|function|c65279d2ecf0315f|||||||||||||||
get_elevation|function|8ade129c4b6565d6|||||||||||||||
get_glw|function|c56e9ad7b5763189|||||||||||||||
get_glw_data|function|e0d1d91328785495|||||||||||||||
get_modis_ndvi_api_parameters|function|c30ed04e0978b176|||||||||||||||
Expand Down Expand Up @@ -6555,8 +6557,8 @@ soil_directory_dataset|stem|5367f73442caebca|6b87e3cd8dc09f31|3d3d9feae01275db|1
soil_directory_raw|stem|6d0bef71eae7763c|7be6d1311e8674de|3d3d9feae01275db|781979019|bucket=open-rvfcast-data*region=us-east-1*key=_targets/soil_directory_raw*endpoint=TlVMTA*version=|t19920.7240877431s||40|qs|aws|vector|||0.002||
soil_download|function|371f9065fa1611a3|||||||||||||||
soil_downloaded|stem|678485c3217017dc|73e8f917a50d005c|bc396cd8e73e5cc4|-1047214501|data/soil|t19920.7441717959s|4fa881245fd93336|320|file|local|vector|||14.281||
soil_preprocessed|stem|5367f73442caebca|7eacd6654f0fbf4a|992a2f44982341eb|1256321187|bucket=open-rvfcast-data*region=us-east-1*key=_targets/soil_preprocessed*endpoint=TlVMTA*version=|t19920.7441778077s||49|qs|aws|vector|||1.407|set.cats setting categories like this is deprecated use a twocolumn data.frame instead. set.cats setting categories like this is deprecated use a twocolumn data.frame instead|
static_targets|object|e029f9dca75c665b|||||||||||||||
soil_preprocessed|stem|5367f73442caebca|7eacd6654f0fbf4a|d0ba406f5d022c00|1256321187|bucket=open-rvfcast-data*region=us-east-1*key=_targets/soil_preprocessed*endpoint=TlVMTA*version=|t19924.1920359525s||49|qs|aws|vector|||1.708|set.cats setting categories like this is deprecated use a twocolumn data.frame instead. set.cats setting categories like this is deprecated use a twocolumn data.frame instead|
static_targets|object|9b0f09cc6837cca2|||||||||||||||
submit_modis_ndvi_bundle_request|function|70d5dcdcf3510fa0|||||||||||||||
submit_modis_ndvi_request|function|e6d950c0c17bd8cd|||||||||||||||
submit_modis_ndvi_task_request|function|a97a6292a38c2323|||||||||||||||
Expand Down
Empty file removed data/elevation_dataset/.gitkeep
Empty file.
Binary file added data/slope_aspect_dataset/slope_dataset
Binary file not shown.
Binary file modified data/soil_dataset/soil_drainage
Binary file not shown.
11 changes: 11 additions & 0 deletions renv.lock
Original file line number Diff line number Diff line change
Expand Up @@ -996,6 +996,17 @@
],
"Hash": "15e9634c0fcd294799e9b2e929ed1b86"
},
"geodata": {
"Package": "geodata",
"Version": "0.6-2",
"Source": "Repository",
"Repository": "RSPM",
"Requirements": [
"R",
"terra"
],
"Hash": "4d43570de08ab49ba86bb2ccb6fe117d"
},
"gert": {
"Package": "gert",
"Version": "2.0.0",
Expand Down

0 comments on commit e1891e6

Please sign in to comment.