Add ability to not read blobs

hpcc-systems · Oct 24, 2024 · 77e78f4 · 77e78f4
1 parent 31344aa
commit 77e78f4
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 6 deletions.
diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java
@@ -344,6 +344,7 @@ public boolean isBlob()
 
     /**
      * Sets the blob flag.
+     * @param blob is the field a blob?
      */
     public void setIsBlob(boolean blob)
     {

diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
@@ -37,6 +37,7 @@ public class RecordDefinitionTranslator
 
     private static final String ESP_TYPE_NAME_PREFIX = "ty";
 
+    private static final int    BLOB_LENGTH           = 8;
     private static final int    FLAG_UNSIGNED         = 256;
     private static final int    FLAG_UNKNOWN_SIZE     = 1024;
     private static final int    TYPE_ID_MASK          = 0xff;       // 0x7fff & ~FLAG_UNKNOWN_SIZE & ~FLAG_UNSIGNED;
@@ -730,7 +731,7 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
 
             JSONObject typeDef = new JSONObject();
             typeDef.put("fieldType", type_blob);
-            typeDef.put("length", 8);
+            typeDef.put("length", BLOB_LENGTH);
             typeDef.put("child", nonBlobTypeName);
 
             int newTypeIndex = typeDefinitions.size();

diff --git a/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java b/dfsclient/src/main/java/org/hpccsystems/dfs/client/BinaryRecordReader.java
@@ -558,6 +558,36 @@ private Object parseRecord(FieldDef recordDef, IRecordBuilder recordBuilder, boo
         for (int fieldIndex = 0; fieldIndex < recordDef.getNumDefs(); fieldIndex++)
         {
             FieldDef fd = recordDef.getDef(fieldIndex);
+            if (fd.isBlob())
+            {
+                // If we encounter a blob field, we only have access to the blob file location
+                // So read that location and construct a default value field
+                long blobFileLoc = (long) getUnsigned(8, true);
+                try
+                {
+                    switch (fd.getFieldType())
+                    {
+                        case BINARY:
+                            recordBuilder.setFieldValue(fieldIndex, new byte[0]);
+                            continue;
+                        case STRING:
+                        case VAR_STRING:
+                            recordBuilder.setFieldValue(fieldIndex, "");
+                            continue;
+                        case SET:
+                        case DATASET:
+                            recordBuilder.setFieldValue(fieldIndex, new ArrayList<Object>());
+                            continue;
+                        default:
+                            throw new UnparsableContentException("Unexpected blob type: " + fd.getFieldType() + " for field: " + fd.getFieldName());
+                    }
+                }
+                catch (IllegalAccessException e)
+                {
+                    throw new UnparsableContentException("Unable to set field value for field: " + fd.getFieldName() + " with error: " + e.getMessage());
+                }
+            }
+
             Object fieldValue = null;
             switch (fd.getFieldType())
             {

diff --git a/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java b/dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java
@@ -52,6 +52,7 @@ public class HPCCFile implements Serializable
     private DataPartition[]      dataParts;
     private DataPartition        tlkPartition                  = null;
     private boolean              useTLK                        = true;
+    private boolean              readBlobs                     = true;
     private PartitionProcessor   partitionProcessor            = null;
     private long                 dataPartsCreationTimeMS       = -1;
 
@@ -241,7 +242,7 @@ private void updateProjectedRecordDef() throws Exception
     {
         this.projectedRecordDefinition = this.columnPruner.pruneRecordDefinition(this.recordDefinition);
 
-        // By default project all sub-integer types to standard integers
+        // By default project all sub-integer types to standard integers and all blobs to non-blobs
         for (int i = 0; i < this.projectedRecordDefinition.getNumDefs(); i++)
         {
             FieldDef field = this.projectedRecordDefinition.getDef(i);
@@ -250,10 +251,11 @@ private void updateProjectedRecordDef() throws Exception
                 field.setSourceType(HpccSrcType.LITTLE_ENDIAN);
             }
 
-            // if (field.isBlob())
-            // {
-            //     field.setIsBlob(false);
-            // }
+            // Project blobs to non-blobs, otherwise we will only get back the file position of the blob
+            if (readBlobs && field.isBlob())
+            {
+                field.setIsBlob(false);
+            }
         }
     }
 
@@ -367,6 +369,24 @@ public HPCCFile setUseTLK(boolean useTLK)
         return this;
     }
 
+    /**
+     * Sets the read blobs options
+     * Note: Blobs are read by default, on older HPCC systems reading blobs can cause issues reading blobs should be disabled for these systems.
+     *
+     * @param readBlobs should blobs be read
+     *
+     * @return this file
+     */
+    public HPCCFile setReadBlobs(boolean readBlobs)
+    {
+        this.readBlobs = readBlobs;
+
+        // Force the data parts to be re-created
+        this.dataParts = null;
+
+        return this;
+    }
+
     /**
      * Gets the filter.
      *