Fix s3 dirlisting to follow the istruncated/nextmarker in listobject …

…response
cern-fts · Jun 24, 2024 · 38faaea · 38faaea
1 parent f75b736
commit 38faaea
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 3 deletions.
diff --git a/src/fileops/davmeta.cpp b/src/fileops/davmeta.cpp
@@ -1009,7 +1009,7 @@ StatInfo & S3MetaOps::statInfo(IOChainContext & iocontext, StatInfo & st_info){
 
 bool s3_get_next_property(std::unique_ptr<DirHandle> & handle, std::string & name_entry, StatInfo & info){
     DAVIX_SLOG(DAVIX_LOG_DEBUG, DAVIX_LOG_CHAIN, " -> s3_get_next_property");
-    const size_t read_size = 2048;
+    const size_t read_size = 2048*1024;
 
 
     HttpRequest& req = *(handle->request); // setup env again
@@ -1054,6 +1054,13 @@ void s3_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & conte
     }
     else if(params->getS3ListingMode() == S3ListingMode::Hierarchical){
         Uri new_url = S3::s3UriTransformer(url, params, true);
+        if (handle.get() != NULL) {
+            XMLPropParser & parser = *(handle->parser);
+            std::string nextmarker = parser.getNextMarker();
+            if (nextmarker != "") {
+                new_url.addQueryParam("marker", nextmarker);
+            }
+        }
         handle.reset(new DirHandle(new GetRequest(context, new_url, &tmp_err), new S3PropParser(params->getS3ListingMode(), S3::extract_s3_path(url, params->getAwsAlternate()))));
     }
     else if(params->getS3ListingMode() == S3ListingMode::SemiHierarchical){
@@ -1087,10 +1094,10 @@ void s3_start_listing_query(std::unique_ptr<DirHandle> & handle, Context & conte
 
     size_t prop_size = 0;
     do{ // first entry -> bucket information
-       s_resu = incremental_listdir_parsing(&http_req, &parser, 2048, davix_scope_directory_listing_str());
+       s_resu = incremental_listdir_parsing(&http_req, &parser, 2048*1024, davix_scope_directory_listing_str());
 
        prop_size = parser.getProperties().size();
-       if(s_resu < 2048 && prop_size <1){ // verify request status : if req done + no data -> error
+       if(s_resu < 2048*1024 && prop_size <1){ // verify request status : if req done + no data -> error
            throw DavixException(davix_scope_directory_listing_str(), StatusCode::ParsingError, "Invalid server response, not a S3 listing");
        }
        if(timestamp_timeout < time(NULL)){
@@ -1116,6 +1123,10 @@ bool s3_directory_listing(std::unique_ptr<DirHandle> & handle, Context & context
     if(handle.get() == NULL){
         s3_start_listing_query(handle, context, params, uri, body);
     }
+    XMLPropParser& parser = *(handle->parser);
+    if (parser.getProperties().size() == 0 && parser.getNextMarker() != "") {
+        s3_start_listing_query(handle, context, params, uri, body);
+    }
     return s3_get_next_property(handle, name_entry, info);
 }
 

diff --git a/src/xml/davxmlparser.hpp b/src/xml/davxmlparser.hpp
@@ -103,6 +103,7 @@ class ElementsParser {
     virtual ~ElementsParser(){}
 
     virtual std::deque<FileProperties> & getProperties()=0;
+    virtual std::string getNextMarker() { return "";};
 
 };
 

diff --git a/src/xml/s3propparser.cpp b/src/xml/s3propparser.cpp
@@ -37,6 +37,8 @@ const std::string prefix_prop = "Prefix";
 const std::string com_prefix_prop = "CommonPrefixes";
 const std::string listbucketresult_prop = "ListBucketResult";
 const std::string last_modified_prop = "LastModified";
+const std::string istruncated_prop = "IsTruncated";
+const std::string nextmarker_prop = "NextMarker";
 
 struct S3PropParser::Internal{
     std::string current;
@@ -50,6 +52,9 @@ struct S3PropParser::Internal{
     FileProperties property;
     S3ListingMode::S3ListingMode _s3_listing_mode;
 
+    bool istruncated = false;
+    std::string nextmarker = "";
+
     int start_elem(const std::string &elem){
         // new tag, clean content;
         current.clear();
@@ -111,6 +116,9 @@ struct S3PropParser::Internal{
                 property.info.mode &= ~(S_IFREG);
                 props.push_back(property);
                 prop_count++;
+                if (istruncated) {
+                    nextmarker = current;
+                }
             }
         }
 
@@ -190,6 +198,21 @@ struct S3PropParser::Internal{
             throw DavixException(davix_scope_directory_listing_str(), StatusCode::IsNotADirectory, "Not a S3 directory");
         }
 
+        // IsTruncated
+        if( (_s3_listing_mode == S3ListingMode::Hierarchical) && (StrUtil::compare_ncase(istruncated_prop, elem) ==0)){
+            if (current == "True" || current == "true") {
+                istruncated = true;
+            }
+            else {
+                istruncated = false;
+            }
+        }
+
+        // NextMarker
+        if( (_s3_listing_mode == S3ListingMode::Hierarchical) && (StrUtil::compare_ncase(nextmarker_prop, elem) ==0)){
+            nextmarker = current;
+        }
+
         // reduce stack size
         if(stack_status.size() > 0)
             stack_status.pop();
@@ -252,5 +275,9 @@ std::deque<FileProperties> & S3PropParser::getProperties(){
     return d_ptr->props;
 }
 
+std::string S3PropParser::getNextMarker() {
+    return (d_ptr->istruncated? d_ptr->nextmarker : "");
+}
+
 
 }
diff --git a/src/xml/s3propparser.hpp b/src/xml/s3propparser.hpp
@@ -41,6 +41,7 @@ class S3PropParser :  public XMLPropParser
     virtual ~S3PropParser();
 
     virtual std::deque<FileProperties> & getProperties();
+    virtual std::string getNextMarker();
 
 
 protected: