Merge pull request #168 from ldecicco-USGS/master

Change WQP requests to zip as default

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: dataRetrieval
 Type: Package
 Title: Retrieval Functions for USGS and EPA Hydrologic and Water Quality Data
-Version: 2.4.5
-Date: 2015-12-09
+Version: 2.4.6
+Date: 2016-01-26
 Authors@R: c( person("Robert", "Hirsch", role = c("aut"),
     email = "[email protected]"),
     person("Laura", "DeCicco", role = c("aut","cre"),

R/constructNWISURL.r

@@ -222,6 +222,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
 #' retrieval for the earliest possible record.
 #' @param endDate character ending date for data retrieval in the form YYYY-MM-DD. Default is "" which indicates
 #' retrieval for the latest possible record.
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @keywords data import WQP web service
 #' @return url string
 #' @export
@@ -233,7 +234,7 @@ constructNWISURL <- function(siteNumber,parameterCd="00060",startDate="",endDate
 #' url_wqp <- constructWQPURL(paste("USGS",siteNumber,sep="-"),
 #'            c('01075','00029','00453'),
 #'            startDate,endDate)
-constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){
+constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate,zip=TRUE){
 
   multipleSites <- length(siteNumber) > 1
   multiplePcodes <- length(parameterCd)>1
@@ -267,6 +268,11 @@ constructWQPURL <- function(siteNumber,parameterCd,startDate,endDate){
   }
 
   url <- paste0(url,"&sorted=no&mimeType=tsv")
+
+  if(zip){
+    url <- paste0(url,"&zip=yes")
+  }
+
   return(url)
 
 }

R/getWebServiceData.R

@@ -20,9 +20,12 @@
 #' rawData <- getWebServiceData(obs_url)
 #' }
 getWebServiceData <- function(obs_url, ...){
+
   possibleError <- tryCatch({
     h <- basicHeaderGatherer()
-    returnedDoc <- getURI(obs_url, headerfunction = h$update, ...)      
+
+    returnedDoc <- getURI(obs_url, headerfunction = h$update, 
+                          useragent = paste("dataRetrieval",packageVersion("dataRetrieval"),sep="/"), ...)      
   }, warning = function(w) {
     warning(w, "with url:", obs_url)
   }, error = function(e) {

R/importRDB1.r

@@ -185,7 +185,9 @@ importRDB1 <- function(obs_url, asDateTime=TRUE, convertType = TRUE, tz=""){
 
       if("tz_cd" %in% header.names){
         date.time.cols <- which(sapply(readr.data, function(x) inherits(x, "POSIXct")))
-        readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE)
+        if(length(date.time.cols) > 0){
+          readr.data <- convertTZ(readr.data,"tz_cd",date.time.cols,tz, flip.cols=FALSE)
+        }
       }
 
       if("DATE" %in% header.names){

R/importWQP.R

@@ -5,7 +5,7 @@
 #' Imports data from the Water Quality Portal based on a specified url.
 #' 
 #' @param obs_url character URL to Water Quality Portal#' @keywords data import USGS web service
-#' @param zip logical used to request the data in a zip format (TRUE)
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @param tz character to set timezone attribute of datetime. Default is an empty quote, which converts the 
 #' datetimes to UTC (properly accounting for daylight savings times based on the data's provided tz_cd column).
 #' Possible values to provide are "America/New_York","America/Chicago", "America/Denver","America/Los_Angeles",
@@ -34,13 +34,14 @@
 #' rawSampleURL <- constructWQPURL('USGS-01594440','01075', '', '')
 #' 
 #' rawSample <- importWQP(rawSampleURL)
-#' url2 <- paste0(rawSampleURL,"&zip=yes")
-#' rawSample2 <- importWQP(url2, zip=TRUE)
+#' 
+#' rawSampleURL_noZip <- constructWQPURL('USGS-01594440','01075', '', '', FALSE)
+#' rawSample2 <- importWQP(rawSampleURL_noZip, zip=FALSE)
 #' 
 #' STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')
 #' STORETdata <- importWQP(STORETex)
 #' }
-importWQP <- function(obs_url, zip=FALSE, tz=""){
+importWQP <- function(obs_url, zip=TRUE, tz=""){
 
   if(tz != ""){
     tz <- match.arg(tz, c("America/New_York","America/Chicago",

R/readWQPdata.R

@@ -4,8 +4,7 @@
 #' because it allows for other agencies rather than the USGS.  
 #'
 #' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options
-#' @param zip logical to request data via downloading zip file. Default set to FALSE. The overhead associated with 
-#' downloading and un-zipping only improves performance for large data returns.
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @keywords data import WQP web service
 #' @return A data frame with at least the following columns:
 #' \tabular{lll}{ 
@@ -98,7 +97,7 @@
 #' 
 #'                         
 #' }
-readWQPdata <- function(..., zip=FALSE){
+readWQPdata <- function(..., zip=TRUE){
 
   matchReturn <- list(...)
 
@@ -158,7 +157,7 @@ readWQPdata <- function(..., zip=FALSE){
   retval <- importWQP(urlCall,zip=zip, tz=tz)
 
   if(!all(is.na(retval))){
-    siteInfo <- whatWQPsites(...)
+    siteInfo <- whatWQPsites(...,zip=zip)
 
     siteInfoCommon <- data.frame(station_nm=siteInfo$MonitoringLocationName,
                                  agency_cd=siteInfo$OrganizationIdentifier,

R/whatWQPsites.R

@@ -4,6 +4,7 @@
 #' Arguments to the function should be based on \url{www.waterqualitydata.us/webservices_documentation.jsp}
 #'
 #' @param \dots see \url{www.waterqualitydata.us/webservices_documentation.jsp} for a complete list of options
+#' @param zip logical to request data via downloading zip file. Default set to TRUE.
 #' @keywords data import WQP web service
 #' @return A data frame with at least the following columns:
 #' \tabular{lll}{ 
@@ -55,9 +56,9 @@
 #' 
 #' type <- "Stream"
 #' sites <- whatWQPsites(countycode="US:55:025",siteType=type)
-#' data <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
+#' lakeSites <- whatWQPsites(siteType = "Lake, Reservoir, Impoundment", statecode = "US:55")
 #' }
-whatWQPsites <- function(...){
+whatWQPsites <- function(...,zip=TRUE){
 
   matchReturn <- list(...)
 
@@ -94,12 +95,17 @@ whatWQPsites <- function(...){
 
 
   baseURL <- "http://www.waterqualitydata.us/Station/search?"
-  urlCall <- paste(baseURL,
+  urlCall <- paste0(baseURL,
                urlCall,
-               "&mimeType=tsv&sorted=no",sep = "")
+               "&mimeType=tsv&sorted=no")
+  if(zip){
+    urlCall <- paste0(urlCall,"&zip=yes")
+  }
+
+  retval <- importWQP(urlCall, zip=zip)
 
-  retval <- importWQP(urlCall)
-  retval$queryTime <- Sys.time()
+  attr(retval, "queryTime") <- Sys.time()
+  attr(retval, "url") <- urlCall
 
   return(retval)
 

tests/testthat/tests_imports.R

@@ -128,8 +128,8 @@ test_that("External WQP tests", {
   rawSample <- importWQP(rawSampleURL)
   expect_is(rawSample$ActivityStartDateTime, 'POSIXct')
 
-  url2 <- paste0(rawSampleURL,"&zip=yes")
-  rawSample2 <- suppressWarnings(importWQP(url2, TRUE))
+  url2 <- constructWQPURL('USGS-01594440','01075', '', '', zip = FALSE)
+  rawSample2 <- suppressWarnings(importWQP(url2, FALSE))
   expect_is(rawSample2$ActivityStartDateTime, 'POSIXct')
 
   STORETex <- constructWQPURL('WIDNR_WQX-10032762','Specific conductance', '', '')