diff --git a/DESCRIPTION b/DESCRIPTION index 6fa3080e..9300cb3f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: dataRetrieval Type: Package Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data -Version: 2.4.5 -Date: 2015-12-09 +Version: 2.4.6 +Date: 2016-01-26 Authors@R: c( person("Robert", "Hirsch", role = c("aut"), email = "rhirsch@usgs.gov"), person("Laura", "DeCicco", role = c("aut","cre"), diff --git a/R/constructNWISURL.r b/R/constructNWISURL.r index 63aedda5..9aecf9a3 100644 --- a/R/constructNWISURL.r +++ b/R/constructNWISURL.r @@ -222,6 +222,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate #' retrieval for the earliest possible record. #' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates #' retrieval for the latest possible record. +#' @param zip logical to request data via downloading zip file. Default set to TRUE. #' @keywords data import WQP web service #' @return url string #' @export @@ -233,7 +234,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate #' url_wqp <- constructWQPURL(paste("USGS",siteNumber,sep="-"), #' c('01075','00029','00453'), #' startDate,endDate) -constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){ +constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate,zip=TRUE){ multipleSites <- length(siteNumber) > 1 multiplePcodes <- length(parameterCd)>1 @@ -267,6 +268,11 @@ constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){ } url <- paste0(url,"&sorted=no&mimeType=tsv") + + if(zip){ + url <- paste0(url,"&zip=yes") + } + return(url) } diff --git a/R/getWebServiceData.R b/R/getWebServiceData.R index c13dc354..ea12f272 100644 --- a/R/getWebServiceData.R +++ b/R/getWebServiceData.R @@ -20,9 +20,12 @@ #' rawData <- getWebServiceData(obs_url) #' } getWebServiceData <- function(obs_url, ...){ + possibleError <- tryCatch({ h <- basicHeaderGatherer() - returnedDoc <- getURI(obs_url, headerfunction = h$update, ...) + + returnedDoc <- getURI(obs_url, headerfunction = h$update, + useragent = paste("dataRetrieval",packageVersion("dataRetrieval"),sep="/"), ...) }, warning = function(w) { warning(w, "with url:", obs_url) }, error = function(e) { diff --git a/R/importRDB1.r b/R/importRDB1.r index fe8df947..08a5f3a1 100644 --- a/R/importRDB1.r +++ b/R/importRDB1.r @@ -185,7 +185,9 @@ importRDB1 <- function(obs_url, asDateTime=TRUE, convertType = TRUE, tz=""){ if("tz_cd" %in% header.names){ date.time.cols <- which(sapply(readr.data, function(x) inherits(x, "POSIXct"))) - readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE) + if(length(date.time.cols) > 0){ + readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE) + } } if("DATE" %in% header.names){ diff --git a/R/importWQP.R b/R/importWQP.R index 996bdb8c..18c9fe6e 100644 --- a/R/importWQP.R +++ b/R/importWQP.R @@ -5,7 +5,7 @@ #' Imports data from the Water Quality Portal based on a specified url. #' #' @param obs_url character URL to Water Quality Portal#' @keywords data import USGS web service -#' @param zip logical used to request the data in a zip format (TRUE) +#' @param zip logical to request data via downloading zip file. Default set to TRUE. #' @param tz character to set timezone attribute of datetime. Default is an empty quote, which converts the #' datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column). #' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles", @@ -34,13 +34,14 @@ #' rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '') #' #' rawSample <- importWQP(rawSampleURL) -#' url2 <- paste0(rawSampleURL,"&zip=yes") -#' rawSample2 <- importWQP(url2, zip=TRUE) +#' +#' rawSampleURL_noZip <- constructWQPURL('USGS-01594440','01075', '', '', FALSE) +#' rawSample2 <- importWQP(rawSampleURL_noZip, zip=FALSE) #' #' STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '') #' STORETdata <- importWQP(STORETex) #' } -importWQP <- function(obs_url, zip=FALSE, tz=""){ +importWQP <- function(obs_url, zip=TRUE, tz=""){ if(tz != ""){ tz <- match.arg(tz, c("America/New_York","America/Chicago", diff --git a/R/readWQPdata.R b/R/readWQPdata.R index 842e2b7e..06b52620 100644 --- a/R/readWQPdata.R +++ b/R/readWQPdata.R @@ -4,8 +4,7 @@ #' because it allows for other agencies rather than the USGS. #' #' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options -#' @param zip logical to request data via downloading zip file. Default set to FALSE. The overhead associated with -#' downloading and un-zipping only improves performance for large data returns. +#' @param zip logical to request data via downloading zip file. Default set to TRUE. #' @keywords data import WQP web service #' @return A data frame with at least the following columns: #' \tabular{lll}{ @@ -98,7 +97,7 @@ #' #' #' } -readWQPdata <- function(..., zip=FALSE){ +readWQPdata <- function(..., zip=TRUE){ matchReturn <- list(...) @@ -158,7 +157,7 @@ readWQPdata <- function(..., zip=FALSE){ retval <- importWQP(urlCall,zip=zip, tz=tz) if(!all(is.na(retval))){ - siteInfo <- whatWQPsites(...) + siteInfo <- whatWQPsites(...,zip=zip) siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName, agency_cd=siteInfo$OrganizationIdentifier, diff --git a/R/whatWQPsites.R b/R/whatWQPsites.R index b7738261..c777e68d 100644 --- a/R/whatWQPsites.R +++ b/R/whatWQPsites.R @@ -4,6 +4,7 @@ #' Arguments to the function should be based on \url{www.waterqualitydata.us/webservices_documentation.jsp} #' #' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options +#' @param zip logical to request data via downloading zip file. Default set to TRUE. #' @keywords data import WQP web service #' @return A data frame with at least the following columns: #' \tabular{lll}{ @@ -55,9 +56,9 @@ #' #' type <- "Stream" #' sites <- whatWQPsites(countycode="US:55:025",siteType=type) -#' data <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55") +#' lakeSites <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55") #' } -whatWQPsites <- function(...){ +whatWQPsites <- function(...,zip=TRUE){ matchReturn <- list(...) @@ -94,12 +95,17 @@ whatWQPsites <- function(...){ baseURL <- "http://www.waterqualitydata.us/Station/search?" - urlCall <- paste(baseURL, + urlCall <- paste0(baseURL, urlCall, - "&mimeType=tsv&sorted=no",sep = "") + "&mimeType=tsv&sorted=no") + if(zip){ + urlCall <- paste0(urlCall,"&zip=yes") + } + + retval <- importWQP(urlCall, zip=zip) - retval <- importWQP(urlCall) - retval$queryTime <- Sys.time() + attr(retval, "queryTime") <- Sys.time() + attr(retval, "url") <- urlCall return(retval) diff --git a/man/constructWQPURL.Rd b/man/constructWQPURL.Rd index 57ae9d55..5c0598eb 100644 --- a/man/constructWQPURL.Rd +++ b/man/constructWQPURL.Rd @@ -4,7 +4,7 @@ \alias{constructWQPURL} \title{Construct WQP url for data retrieval} \usage{ -constructWQPURL(siteNumber, parameterCd, startDate, endDate) +constructWQPURL(siteNumber, parameterCd, startDate, endDate, zip = TRUE) } \arguments{ \item{siteNumber}{string or vector of strings USGS site number. This is usually an 8 digit number} @@ -16,6 +16,8 @@ retrieval for the earliest possible record.} \item{endDate}{character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates retrieval for the latest possible record.} + +\item{zip}{logical to request data via downloading zip file. Default set to TRUE.} } \value{ url string diff --git a/man/importWQP.Rd b/man/importWQP.Rd index 6aee7aab..db89e415 100644 --- a/man/importWQP.Rd +++ b/man/importWQP.Rd @@ -4,12 +4,12 @@ \alias{importWQP} \title{Basic Water Quality Portal Data parser} \usage{ -importWQP(obs_url, zip = FALSE, tz = "") +importWQP(obs_url, zip = TRUE, tz = "") } \arguments{ \item{obs_url}{character URL to Water Quality Portal#' @keywords data import USGS web service} -\item{zip}{logical used to request the data in a zip format (TRUE)} +\item{zip}{logical to request data via downloading zip file. Default set to TRUE.} \item{tz}{character to set timezone attribute of datetime. Default is an empty quote, which converts the datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column). @@ -31,8 +31,9 @@ Imports data from the Water Quality Portal based on a specified url. rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '') rawSample <- importWQP(rawSampleURL) -url2 <- paste0(rawSampleURL,"&zip=yes") -rawSample2 <- importWQP(url2, zip=TRUE) + +rawSampleURL_noZip <- constructWQPURL('USGS-01594440','01075', '', '', FALSE) +rawSample2 <- importWQP(rawSampleURL_noZip, zip=FALSE) STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '') STORETdata <- importWQP(STORETex) diff --git a/man/readWQPdata.Rd b/man/readWQPdata.Rd index b7eeb6e9..8bb72c6a 100644 --- a/man/readWQPdata.Rd +++ b/man/readWQPdata.Rd @@ -4,11 +4,10 @@ \alias{readWQPdata} \title{General Data Import from Water Quality Portal} \usage{ -readWQPdata(..., zip = FALSE) +readWQPdata(..., zip = TRUE) } \arguments{ -\item{zip}{logical to request data via downloading zip file. Default set to FALSE. The overhead associated with -downloading and un-zipping only improves performance for large data returns.} +\item{zip}{logical to request data via downloading zip file. Default set to TRUE.} \item{\dots}{see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options} } diff --git a/man/whatWQPsites.Rd b/man/whatWQPsites.Rd index b4941c54..88986012 100644 --- a/man/whatWQPsites.Rd +++ b/man/whatWQPsites.Rd @@ -4,9 +4,11 @@ \alias{whatWQPsites} \title{Site Data Import from Water Quality Portal} \usage{ -whatWQPsites(...) +whatWQPsites(..., zip = TRUE) } \arguments{ +\item{zip}{logical to request data via downloading zip file. Default set to TRUE.} + \item{\dots}{see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options} } \value{ @@ -62,7 +64,7 @@ site1 <- whatWQPsites(siteid="USGS-01594440") type <- "Stream" sites <- whatWQPsites(countycode="US:55:025",siteType=type) -data <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55") +lakeSites <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55") } } \keyword{WQP} diff --git a/tests/testthat/tests_imports.R b/tests/testthat/tests_imports.R index aef05ac1..9f70c4e6 100644 --- a/tests/testthat/tests_imports.R +++ b/tests/testthat/tests_imports.R @@ -128,8 +128,8 @@ test_that("External WQP tests", { rawSample <- importWQP(rawSampleURL) expect_is(rawSample$ActivityStartDateTime, 'POSIXct') - url2 <- paste0(rawSampleURL,"&zip=yes") - rawSample2 <- suppressWarnings(importWQP(url2, TRUE)) + url2 <- constructWQPURL('USGS-01594440','01075', '', '', zip = FALSE) + rawSample2 <- suppressWarnings(importWQP(url2, FALSE)) expect_is(rawSample2$ActivityStartDateTime, 'POSIXct') STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')