diff --git a/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserializer.java b/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserializer.java
index 899e002a..1940ae10 100644
--- a/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserializer.java
+++ b/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserializer.java
@@ -156,14 +156,14 @@ && _isValidTimestampString(value)) {
             return _fromTimestamp(ctxt, NumberInput.parseLong(value));
         }
 
-        if (_durationPattern != null) {
-            return _durationPattern.parse(NumberInput.parseLong(value));
-        }
-
         try {
+            if (_durationPattern != null) {
+                return _durationPattern.parse(NumberInput.parseLong(value));
+            }
+
             return Duration.parse(value);
-        } catch (DateTimeException e) {
-            return _handleDateTimeException(ctxt, e, value);
+        } catch (NumberFormatException | DateTimeException e) {
+            return _handleWeirdStringValue(ctxt, e, value);
         }
     }
 
diff --git a/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/JSR310DeserializerBase.java b/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/JSR310DeserializerBase.java
index ec208b72..040a4452 100644
--- a/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/JSR310DeserializerBase.java
+++ b/datetime/src/main/java/com/fasterxml/jackson/datatype/jsr310/deser/JSR310DeserializerBase.java
@@ -141,6 +141,13 @@ protected <BOGUS> BOGUS _reportWrongToken(JsonParser parser, DeserializationCont
     @SuppressWarnings("unchecked")
     protected <R> R _handleDateTimeException(DeserializationContext context,
               DateTimeException e0, String value) throws JsonMappingException
+    {
+        return _handleWeirdStringValue(context, e0, value);
+    }
+
+    @SuppressWarnings("unchecked")
+    protected <R, E extends RuntimeException> R _handleWeirdStringValue(DeserializationContext context,
+              E e0, String value) throws JsonMappingException
     {
         try {
             return (R) context.handleWeirdStringValue(handledType(), value,
diff --git a/datetime/src/test/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserTest.java b/datetime/src/test/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserTest.java
index 6c081423..a33c2ed4 100644
--- a/datetime/src/test/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserTest.java
+++ b/datetime/src/test/java/com/fasterxml/jackson/datatype/jsr310/deser/DurationDeserTest.java
@@ -2,12 +2,10 @@
 
 import java.math.BigInteger;
 import java.time.Duration;
-import java.time.LocalDateTime;
 import java.time.temporal.TemporalAmount;
 import java.util.Map;
 
 import com.fasterxml.jackson.annotation.JsonFormat;
-import com.fasterxml.jackson.annotation.OptBoolean;
 import com.fasterxml.jackson.core.type.TypeReference;
 import org.junit.Test;
 
@@ -21,6 +19,7 @@
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.exc.InvalidFormatException;
 import com.fasterxml.jackson.databind.exc.MismatchedInputException;
 import com.fasterxml.jackson.datatype.jsr310.MockObjectConfiguration;
 import com.fasterxml.jackson.datatype.jsr310.ModuleTestBase;
@@ -433,7 +432,7 @@ public void testStrictDeserializeFromEmptyString() throws Exception {
     }
 
     @Test
-    public void shouldDeserializeInHours_whenValueIsString() throws Exception {
+    public void shouldDeserializeInHours_whenUnitAsPattern_andValueIsString() throws Exception {
         ObjectMapper mapper = newMapper();
         ObjectReader reader = mapper.readerFor(MAP_TYPE_REF);
 
@@ -442,8 +441,18 @@ public void shouldDeserializeInHours_whenValueIsString() throws Exception {
         assertEquals(Duration.ofHours(25),  wrapper.value);
     }
 
+    @Test(expected = InvalidFormatException.class)
+    public void shouldHandleException_whenUsingUnitAsPattern_andValueIsString() throws Exception {
+        ObjectMapper mapper = newMapper();
+        ObjectReader reader = mapper.readerFor(MAP_TYPE_REF);
+
+        Wrapper wrapper = reader.readValue("{\"value\":\"FAIL\"}", Wrapper.class);
+
+        assertEquals(Duration.ofHours(25),  wrapper.value);
+    }
+
     @Test
-    public void shouldDeserializeInHours_whenValueIsInteger() throws Exception {
+    public void shouldDeserializeInHours_whenUnitAsPattern_andValueIsInteger() throws Exception {
         ObjectMapper mapper = newMapper();
         ObjectReader reader = mapper.readerFor(MAP_TYPE_REF);
 
@@ -453,7 +462,7 @@ public void shouldDeserializeInHours_whenValueIsInteger() throws Exception {
     }
 
     @Test
-    public void shouldDeserializeInHours_whenValueIsFloat() throws Exception {
+    public void shouldDeserializeInHours_whenUnitAsPattern_andValueIsFloat() throws Exception {
         ObjectMapper mapper = newMapper();
         ObjectReader reader = mapper.readerFor(MAP_TYPE_REF);