bulk: do not truncate target paths

Motivation: ========== Bulk truncates path to 256 characters and this seems to be causing problems. Modification: ============= Allow file paths to be 4096 bytes. Result: ======= No issues caused by file path truncation Patch: https://rb.dcache.org/r/14255/ Target: trunk Request: 10.x Request: 9.x
DmitryLitvintsev · May 28, 2024 · 782b274 · 782b274
1 parent d64b99c
commit 782b274
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 3 deletions.
diff --git a/...-bulk/src/main/java/org/dcache/services/bulk/store/jdbc/rtarget/JdbcRequestTargetDao.java b/...-bulk/src/main/java/org/dcache/services/bulk/store/jdbc/rtarget/JdbcRequestTargetDao.java
@@ -98,6 +98,8 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 public final class JdbcRequestTargetDao extends JdbcDaoSupport {
 
     public static final String TABLE_NAME = "request_target";
+    public static final int REQUEST_TARGET_PATH_LENGTH = 4096;
+    public static final int REQUEST_TARGET_ERROR_MESSAGE_LENGTH = 256;
 
     static class TargetPlaceholder {
         Long rid;
@@ -121,6 +123,7 @@ static class TargetPlaceholder {
           = "SELECT r.uid FROM bulk_request r WHERE r.status = 'COMPLETED' AND EXISTS " +
           "(SELECT * FROM request_target t WHERE r.id = t.rid AND t.state = 'FAILED')";
 
+
     static final ParameterizedPreparedStatementSetter<TargetPlaceholder> SETTER = (ps, target) -> {
         Instant now = Instant.now();
         ps.setInt(1, PID.INITIAL.ordinal());
@@ -187,7 +190,7 @@ public void insertInitialTargets(BulkRequest request) {
                 t.path = "invalid (empty) path";
                 t.state = FAILED.name();
             } else {
-                t.path = truncate(target, 256, true);
+                t.path = truncate(target, REQUEST_TARGET_PATH_LENGTH, true);
                 t.state = CREATED.name();
             }
             if (seen.contains(t.path)) {

diff --git a/...lk/src/main/java/org/dcache/services/bulk/store/jdbc/rtarget/JdbcRequestTargetUpdate.java b/...lk/src/main/java/org/dcache/services/bulk/store/jdbc/rtarget/JdbcRequestTargetUpdate.java
@@ -127,7 +127,10 @@ public JdbcRequestTargetUpdate errorType(String errorType) {
     }
 
     public JdbcRequestTargetUpdate errorMessage(String errorMessage) {
-        set("error_message", truncate(errorMessage, 256, false));
+        set("error_message",
+            truncate(errorMessage,
+                     JdbcRequestTargetDao.REQUEST_TARGET_ERROR_MESSAGE_LENGTH,
+                     false));
         return this;
     }
 
@@ -154,7 +157,10 @@ public JdbcRequestTargetUpdate pnfsid(PnfsId pnfsId) {
 
     public JdbcRequestTargetUpdate path(FsPath path) {
         if (path != null) {
-            set("path", truncate(path.toString(), 256,true));
+            set("path",
+                truncate(path.toString(),
+                         JdbcRequestTargetDao.REQUEST_TARGET_PATH_LENGTH,
+                         true));
         }
         return this;
     }

diff --git a/modules/dcache-bulk/src/main/resources/org/dcache/services/bulk/model/db.changelog-9.2.xml b/modules/dcache-bulk/src/main/resources/org/dcache/services/bulk/model/db.changelog-9.2.xml
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--  COPYRIGHT STATUS:-->
+<!--  Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and-->
+<!--  software are sponsored by the U.S. Department of Energy under Contract No.-->
+<!--  DE-AC02-76CH03000. Therefore, the U.S. Government retains a  world-wide-->
+<!--  non-exclusive, royalty-free license to publish or reproduce these documents-->
+<!--  and software for U.S. Government purposes.  All documents and software-->
+<!--  available from this server are protected under the U.S. and Foreign-->
+<!--  Copyright Laws, and FNAL reserves all rights.-->
+
+<!--  Distribution of the software available from this server is free of-->
+<!--  charge subject to the user following the terms of the Fermitools-->
+<!--  Software Legal Information.-->
+
+<!--  Redistribution and/or modification of the software shall be accompanied-->
+<!--  by the Fermitools Software Legal Information  (including the copyright-->
+<!--  notice).-->
+
+<!--  The user is asked to feed back problems, benefits, and/or suggestions-->
+<!--  about the software to the Fermilab Software Providers.-->
+
+<!--  Neither the name of Fermilab, the  URA, nor the names of the contributors-->
+<!--  may be used to endorse or promote products derived from this software-->
+<!--  without specific prior written permission.-->
+
+<!--  DISCLAIMER OF LIABILITY (BSD):-->
+
+<!--  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS-->
+<!--  "AS IS" AND ANY EXPRESS OR IMPLIED  WARRANTIES, INCLUDING, BUT NOT-->
+<!--  LIMITED TO, THE IMPLIED  WARRANTIES OF MERCHANTABILITY AND FITNESS-->
+<!--  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,-->
+<!--  OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE-->
+<!--  FOR  ANY  DIRECT, INDIRECT,  INCIDENTAL, SPECIAL, EXEMPLARY, OR-->
+<!--  CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT-->
+<!--  OF SUBSTITUTE  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR-->
+<!--  BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY  OF-->
+<!--  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING-->
+<!--  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT  OF THE USE OF THIS-->
+<!--  SOFTWARE, EVEN IF ADVISED OF THE  POSSIBILITY OF SUCH DAMAGE.-->
+
+<!--  Liabilities of the Government:-->
+
+<!--  This software is provided by URA, independent from its Prime Contract-->
+<!--  with the U.S. Department of Energy. URA is acting independently from-->
+<!--  the Government and in its own private capacity and is not acting on-->
+<!--  behalf of the U.S. Government, nor as its contractor nor its agent.-->
+<!--  Correspondingly, it is understood and agreed that the U.S. Government-->
+<!--  has no connection to this software and in no manner whatsoever shall-->
+<!--  be liable for nor assume any responsibility or obligation for any claim,-->
+<!--  cost, or damages arising out of or resulting from the use of the software-->
+<!--  available from this server.-->
+
+<!--  Export Control:-->
+
+<!--  All documents and software available from this server are subject to U.S.-->
+<!--  export control laws.  Anyone downloading information from this server is-->
+<!--  obligated to secure any necessary Government licenses before exporting-->
+<!--  documents or software obtained from this server.-->
+
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
+
+    <changeSet author="litvinse" id="7.2" runInTransaction="false">
+        <preConditions onFail="MARK_RAN">
+            <and>
+                <columnExists tableName="request_target" columnName="path"/>
+            </and>
+        </preConditions>
+        <modifyDataType tableName="request_target" columnName="path" newDataType="varchar(4096)"/>
+	<rollback>
+        <modifyDataType tableName="request_target" columnName="path" newDataType="varchar(256)"/>
+        </rollback>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/...les/dcache-bulk/src/main/resources/org/dcache/services/bulk/model/db.changelog-master.xml b/...les/dcache-bulk/src/main/resources/org/dcache/services/bulk/model/db.changelog-master.xml
@@ -5,4 +5,5 @@
     xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
     <include file="org/dcache/services/bulk/model/db.changelog-8.0.xml"/>
     <include file="org/dcache/services/bulk/model/db.changelog-9.0.xml"/>
+    <include file="org/dcache/services/bulk/model/db.changelog-9.2.xml"/>
 </databaseChangeLog>