generated from ecohealthalliance/container-template
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #59 from ecohealthalliance/modis
Modis
- Loading branch information
Showing
14 changed files
with
3,279 additions
and
314 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
#' .. content for \description{} (no empty lines) .. | ||
#' | ||
#' .. content for \details{} .. | ||
#' | ||
#' @title | ||
#' @param modis_ndvi_downloaded | ||
#' @param continent_raster_template | ||
#' @param modis_ndvi_directory_dataset | ||
#' @param overwrite | ||
#' @return | ||
#' @author Emma Mendelsohn | ||
#' @export | ||
create_modis_ndvi_dataset <- function(modis_ndvi_downloaded, | ||
continent_raster_template, | ||
modis_ndvi_directory_dataset, overwrite = | ||
FALSE) { | ||
|
||
filename <- basename(modis_ndvi_downloaded) | ||
|
||
year_doy <- sub(".*doy(\\d+).*", "\\1", filename) | ||
start_date <- as.Date(year_doy, format = "%Y%j") # confirmed this is start date through manual download tests | ||
end_date <- start_date + 16 | ||
save_filename <- glue::glue("transformed_modis_NDVI_{start_date}_to_{end_date}.gz.parquet") | ||
|
||
existing_files <- list.files(modis_ndvi_directory_dataset) | ||
|
||
message(paste0("Transforming ", save_filename)) | ||
|
||
if(save_filename %in% existing_files & !overwrite){ | ||
message("file already exists, skipping transform") | ||
return(file.path(modis_ndvi_directory_dataset, save_filename)) | ||
} | ||
|
||
transformed_raster <- transform_raster(raw_raster = rast(modis_ndvi_downloaded), | ||
template = rast(continent_raster_template)) | ||
|
||
# Convert to dataframe | ||
dat_out <- as.data.frame(transformed_raster, xy = TRUE) |> | ||
as_tibble() |> | ||
rename(ndvi = 3) |> | ||
mutate(start_date = start_date, | ||
end_date = end_date) | ||
|
||
# Save as parquet | ||
write_parquet(dat_out, here::here(modis_ndvi_directory_dataset, save_filename), compression = "gzip", compression_level = 5) | ||
|
||
return(file.path(modis_ndvi_directory_dataset, save_filename)) | ||
|
||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
#' .. content for \description{} (no empty lines) .. | ||
#' | ||
#' .. content for \details{} .. | ||
#' | ||
#' @title | ||
#' @param modis_ndvi_token | ||
#' @param modis_ndvi_bundle_request | ||
#' @param download_directory | ||
#' @param overwrite | ||
#' @return | ||
#' @author Emma Mendelsohn | ||
#' @export | ||
download_modis_ndvi_delete <- function(modis_ndvi_token, modis_ndvi_bundle_request, | ||
download_directory = modis_ndvi_directory_raw, | ||
overwrite = FALSE) { | ||
|
||
return(modis_ndvi_token) | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
#' .. content for \description{} (no empty lines) .. | ||
#' | ||
#' .. content for \details{} .. | ||
#' | ||
#' @title | ||
|
||
#' @return | ||
#' @author Emma Mendelsohn | ||
#' @export | ||
get_modis_ndvi_token <- function() { | ||
|
||
secret <- base64_enc(paste(Sys.getenv("APPEEARS_USERNAME"), Sys.getenv("APPEEARS_PASSWORD"), sep = ":")) #TODO make project auth | ||
token_response <- POST("https://appeears.earthdatacloud.nasa.gov/api/login", | ||
add_headers("Authorization" = paste("Basic", gsub("\n", "", secret)), | ||
"Content-Type" = "application/x-www-form-urlencoded;charset=UTF-8"), | ||
body = "grant_type=client_credentials") | ||
token_response <- prettify(toJSON(content(token_response), auto_unbox = TRUE)) | ||
token <- paste("Bearer", fromJSON(token_response)$token) | ||
|
||
return(token) | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
#' .. content for \description{} (no empty lines) .. | ||
#' | ||
#' .. content for \details{} .. | ||
#' | ||
#' @title | ||
#' @param modis_ndvi_token | ||
#' @param modis_ndvi_task_id_list | ||
#' @return | ||
#' @author Emma Mendelsohn | ||
#' @export | ||
submit_modis_ndvi_bundle_request <- function(modis_ndvi_token, modis_ndvi_task_id_continent, timeout = 1000) { | ||
|
||
task_id <- modis_ndvi_task_id_continent$task_id | ||
|
||
# Get sys time for the loop timeout | ||
sys_start_time <- Sys.time() | ||
|
||
# Function to check task status | ||
check_task_status <- function() { | ||
task_response <- GET("https://appeears.earthdatacloud.nasa.gov/api/task", add_headers(Authorization = modis_ndvi_token)) | ||
task_response <- fromJSON(toJSON(content(task_response))) |> filter(task_id == !!task_id) | ||
task_status <- task_response |> pull(status) |> unlist() | ||
assertthat::assert_that(task_status %in% c("queued", "pending", "processing", "done")) | ||
return(task_status) | ||
} | ||
|
||
# Check task status in a loop | ||
task_status <- "" | ||
while (task_status != "done") { | ||
task_status <- check_task_status() | ||
|
||
# Print current task status | ||
message(paste("task status:", task_status)) | ||
|
||
# Check timeout | ||
elapsed_time <- difftime(Sys.time(), sys_start_time, units = "secs") | ||
if (task_status != "done" & elapsed_time >= timeout) { | ||
message("timeout reached") | ||
break | ||
} | ||
|
||
# Sleep for a few seconds before checking again | ||
if(task_status != "done"){ | ||
message("pausing 60 seconds before checking again") | ||
Sys.sleep(60) | ||
} | ||
} | ||
|
||
bundle_response <- GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", task_id, sep = ""), add_headers(Authorization = modis_ndvi_token)) | ||
bundle_response <- fromJSON(toJSON(content(bundle_response))) | ||
|
||
bundle_response_files <- bundle_response$files |> | ||
mutate(file_type = unlist(file_type)) |> | ||
mutate(file_name = unlist(file_name)) |> | ||
mutate(file_id = unlist(file_id)) |> | ||
filter(file_type == "tif") |> | ||
mutate(task_id = task_id) | ||
|
||
return(bundle_response_files) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
#' .. content for \description{} (no empty lines) .. | ||
#' | ||
#' .. content for \details{} .. | ||
#' | ||
#' @title | ||
|
||
#' @return | ||
#' @author Emma Mendelsohn | ||
#' @export | ||
submit_modis_ndvi_task_request <- function(modis_ndvi_start_year, | ||
modis_ndvi_end_year, | ||
modis_ndvi_token, | ||
country_bounding_boxes) { | ||
|
||
task_name <- country_bounding_boxes$country_iso3c | ||
bbox_coords <- unlist(country_bounding_boxes$bounding_box) | ||
|
||
# create the task list | ||
task <- list(task_type = "area", | ||
task_name = task_name, | ||
startDate = paste0("01-01-", modis_ndvi_start_year), | ||
endDate = paste0("12-31-", modis_ndvi_end_year), | ||
layer = "MOD13A2.061,_1_km_16_days_NDVI", | ||
file_type = "geotiff", | ||
projection_name = "native", | ||
bbox = paste(bbox_coords, collapse = ",")) | ||
|
||
# post the task request | ||
task_response <- POST("https://appeears.earthdatacloud.nasa.gov/api/task", query = task, add_headers(Authorization = modis_ndvi_token)) | ||
task_response <- content(task_response) | ||
task_response$country_iso3c <- task_name | ||
|
||
return(list(task_response)) | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
#' .. content for \description{} (no empty lines) .. | ||
#' | ||
#' .. content for \details{} .. | ||
#' | ||
#' @title | ||
#' @param modis_ndvi_start_year | ||
#' @param modis_ndvi_end_year | ||
#' @param modis_ndvi_token | ||
#' @param bbox_coords | ||
#' @return | ||
#' @author Emma Mendelsohn | ||
#' @export | ||
submit_modis_ndvi_task_request_continent <- function(modis_ndvi_start_year, | ||
modis_ndvi_end_year, | ||
modis_ndvi_token, | ||
bbox_coords) { | ||
|
||
task_name <- "africa" | ||
|
||
# create the task list | ||
task <- list(task_type = "area", | ||
task_name = task_name, | ||
startDate = paste0("01-01-", modis_ndvi_start_year), | ||
endDate = paste0("12-31-", modis_ndvi_end_year), | ||
layer = "MOD13A2.061,_1_km_16_days_NDVI", | ||
file_type = "geotiff", | ||
projection_name = "native", | ||
bbox = paste(bbox_coords, collapse = ",")) | ||
|
||
# post the task request | ||
task_response <- POST("https://appeears.earthdatacloud.nasa.gov/api/task", query = task, add_headers(Authorization = modis_ndvi_token)) | ||
task_response <- content(task_response) | ||
task_response$country_iso3c <- task_name | ||
|
||
return(task_response) | ||
|
||
|
||
} |
Oops, something went wrong.