Skip to content

Commit

Permalink
save progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@emmamendelsohn
emmamendelsohn committed Sep 5, 2023
1 parent 135238e commit 4d3be7a
Show file tree
Hide file tree
Showing 5 changed files with 180 additions and 101 deletions.
39 changes: 21 additions & 18 deletions R/download_modis_ndvi.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,34 @@
#' @author Emma Mendelsohn
#' @export
download_modis_ndvi <- function(modis_ndvi_token,
modis_ndvi_task_request,
file,
modis_ndvi_bundle_request,
download_directory,
overwrite = FALSE) {

existing_files <- list.files(download_directory)
task_id <- unique(modis_ndvi_bundle_request$task_id)
country_iso3c <- unique(modis_ndvi_bundle_request$country_iso3c)
modis_ndvi_bundle_request$filenames <- paste0(country_iso3c, "_", basename(modis_ndvi_bundle_request$file_name))

# Get file ID and file name
file_id <- unlist(file$file_id)
filename <- basename(unlist(file$file_name))

message(paste0("Downloading ", filename))
modis_ndvi_bundle_request = modis_ndvi_bundle_reques

if(filename %in% existing_files & !overwrite) {
message("file already exists, skipping download")
return(file.path(download_directory, filename)) # skip if file exists
for(i in 1:nrow(modis_ndvi_bundle_request)){

file_id <- modis_ndvi_bundle_request$file_id[i]
filename <- modis_ndvi_bundle_request$filenames[i]

message(paste0("Downloading ", filename))

if(filename %in% existing_files & !overwrite) {
message("file already exists, skipping download")
next()
}

# Write the file to disk
response <- GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", task_id, '/', file_id, sep = ""),
write_disk(file.path(download_directory, filename), overwrite = TRUE), progress(), add_headers(Authorization = modis_ndvi_token))
}

# Get the task ID for downloading
modis_ndvi_request_task_id <- fromJSON(modis_ndvi_task_request)$task_id
return(file.path(download_directory, modis_ndvi_bundle_request$filenames))

# Write the file to disk
response <- GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", modis_ndvi_request_task_id, '/', file_id, sep = ""),
write_disk(file.path(download_directory, filename), overwrite = TRUE), progress(), add_headers(Authorization = modis_ndvi_token))

return(file.path(file.path(download_directory, filename)))

}
39 changes: 24 additions & 15 deletions R/submit_modis_ndvi_bundle_request.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,37 @@
#'
#' @title
#' @param modis_ndvi_token
#' @param modis_ndvi_task_request
#' @param modis_ndvi_task_id_list
#' @return
#' @author Emma Mendelsohn
#' @export
submit_modis_ndvi_bundle_request <- function(modis_ndvi_token, modis_ndvi_task_request, timeout = 1000) {
submit_modis_ndvi_bundle_request <- function(modis_ndvi_token, modis_ndvi_task_id_list, timeout = 1000) {

# Get the task ID
modis_ndvi_request_task_id <- fromJSON(modis_ndvi_task_request)$task_id
task_id <- modis_ndvi_task_id_list$task_id
country_iso3c <- modis_ndvi_task_id_list$country_iso3c

message(country_iso3c)

# Get sys time for the loop timeout
sys_start_time <- Sys.time()

# Function to check task status
check_task_status <- function() {
task_response <- GET("https://appeears.earthdatacloud.nasa.gov/api/task", add_headers(Authorization = modis_ndvi_token))
task_response <- fromJSON(toJSON(content(task_response))) |> filter(task_id == modis_ndvi_request_task_id)
task_response <- fromJSON(toJSON(content(task_response))) |> filter(task_id == !!task_id)
task_status <- task_response |> pull(status) |> unlist()
assertthat::assert_that(task_status %in% c("pending", "processing", "done"))
assertthat::assert_that(task_status %in% c("queued", "pending", "processing", "done"))
return(task_status)
}

# Check task status in a loop
task_status <- ""
while (task_status != "done") {
task_status <- check_task_status()

# Print current task status
message(paste("task status:", task_status))

# Check timeout
elapsed_time <- difftime(Sys.time(), sys_start_time, units = "secs")
if (task_status != "done" & elapsed_time >= timeout) {
Expand All @@ -46,10 +48,17 @@ submit_modis_ndvi_bundle_request <- function(modis_ndvi_token, modis_ndvi_task_r
Sys.sleep(60)
}
}

bundle_response <- GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", modis_ndvi_request_task_id, sep = ""), add_headers(Authorization = modis_ndvi_token))
bundle_response <- fromJSON(toJSON(content(bundle_response)))

return(bundle_response)


bundle_response <- GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", task_id, sep = ""), add_headers(Authorization = modis_ndvi_token))
bundle_response <- fromJSON(toJSON(content(bundle_response)))

bundle_response_files <- bundle_response$files |>
mutate(file_type = unlist(file_type)) |>
mutate(file_name = unlist(file_name)) |>
mutate(file_id = unlist(file_id)) |>
filter(file_type == "tif") |>
mutate(country_iso3c = country_iso3c) |>
mutate(task_id = task_id)

return(bundle_response_files)
}
26 changes: 12 additions & 14 deletions R/submit_modis_ndvi_task_request.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,29 @@
#' @return
#' @author Emma Mendelsohn
#' @export
submit_modis_ndvi_task_request <- function(modis_ndvi_start_year, modis_ndvi_end_year, continent_bounding_box, modis_ndvi_token) {
submit_modis_ndvi_task_request <- function(modis_ndvi_start_year,
modis_ndvi_end_year,
modis_ndvi_token,
country_bounding_boxes) {

task_name <- country_bounding_boxes$country_iso3c
bbox_coords <- unlist(country_bounding_boxes$bounding_box)

# create the task list
task <- list(task_type = "area",
task_name = "modis_ndvi_africa",
task_name = task_name,
startDate = paste0("01-01-", modis_ndvi_start_year),
endDate = paste0("12-31-", modis_ndvi_end_year),
layer = "MOD13A2.061,_1_km_16_days_NDVI",
file_type = "geotiff",
projection_name = "native",
bbox = paste(continent_bounding_box, collapse = ","))
bbox = paste(bbox_coords, collapse = ","))

# post the task request
task_response <- POST("https://appeears.earthdatacloud.nasa.gov/api/task", query = task, add_headers(Authorization = modis_ndvi_token))
task_response <- prettify(toJSON(content(task_response), auto_unbox = TRUE))

return(task_response)
task_response <- content(task_response)
task_response$country_iso3c <- task_name

# # get the associated bundle
# task_id <- fromJSON(task_response)$task_id
# bundle_response <- GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", task_id, sep = ""), add_headers(Authorization = modis_ndvi_token))
#
# #TODO in progress
# bundle_response <- prettify(toJSON(content(response), auto_unbox = TRUE))
# bundle_response <- fromJSON(bundle_response)
# bundle_response$files$file_id
return(list(task_response))

}
30 changes: 24 additions & 6 deletions _targets.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,20 +118,39 @@ dynamic_targets <- tar_plan(
tar_target(modis_ndvi_end_year, 2023),

# set parameters and submit request
tar_target(modis_ndvi_task_request, submit_modis_ndvi_task_request(modis_ndvi_start_year, modis_ndvi_end_year, continent_bounding_box, modis_ndvi_token)),
tar_target(modis_ndvi_task_id, submit_modis_ndvi_task_request(modis_ndvi_start_year,
modis_ndvi_end_year,
modis_ndvi_token,
country_bounding_boxes),
pattern = country_bounding_boxes
),
tar_target(modis_ndvi_task_id_list, map_dfr(modis_ndvi_task_id, as_tibble) |> select(-status)), # this extra step could have been avoided if we set iteration = "list" above, which would allow for continued branching

# tanzania ID
# tar_target(modis_ndvi_task_id, "29541500-ed4c-4020-a60c-5d2f74fbc25b"),
# full download ID
# tar_target(modis_ndvi_task_id, "c05ae10b-4170-4192-8cd8-45bbe57a930a"),
# test download ID
# tar_target(modis_ndvi_task_id, "63af3cad-80a1-4322-977c-73d093e678de"),

# check if the request is posted, then get bundle
# this uses a while loop to check every 30 seconds if the request is complete - it takes about 10 minutes
# this function could be refactored to check time of modis_ndvi_task_request and pause for some time before submitting bundle request
tar_target(modis_ndvi_bundle_request, submit_modis_ndvi_bundle_request(modis_ndvi_token, modis_ndvi_task_request, timeout = 1500)),
tar_target(modis_ndvi_bundle_request, submit_modis_ndvi_bundle_request(modis_ndvi_token,
modis_ndvi_task_id_list,
timeout = 1500),
pattern = modis_ndvi_task_id_list,
iteration = "list"),

#TODO why is TCD failing

# download files from source (locally)
# TODO this is failing now
tar_target(modis_ndvi_downloaded, download_modis_ndvi(modis_ndvi_token,
modis_ndvi_task_request,
file = modis_ndvi_bundle_request$files,
modis_ndvi_bundle_request,
download_directory = modis_ndvi_directory_raw,
overwrite = FALSE),
pattern = modis_ndvi_bundle_request$files,
pattern = head(modis_ndvi_bundle_request, 2),
format = "file",
repository = "local",
cue = tar_cue("thorough")),
Expand Down Expand Up @@ -159,7 +178,6 @@ dynamic_targets <- tar_plan(
download_directory = nasa_weather_directory_raw,
overwrite = FALSE),
pattern = crossing(nasa_weather_years, nasa_weather_coordinates),
iteration = "vector",
format = "file",
repository = "local",
cue = tar_cue("thorough")
Expand Down
Loading

0 comments on commit 4d3be7a

Please sign in to comment.