diff --git a/src/reduction/palette.rs b/src/reduction/palette.rs
index e14743fc..7d8204ae 100644
--- a/src/reduction/palette.rs
+++ b/src/reduction/palette.rs
@@ -139,7 +139,10 @@ pub fn sorted_palette(png: &PngImage) -> Option<PngImage> {
     enumerated.sort_by(|a, b| {
         // Sort by ascending alpha and descending luma
         let color_val = |color: &RGBA8| {
-            ((color.a as i32) << 18)
+            let a = i32::from(color.a);
+            // Use a high factor for alpha to ensure they get sorted first,
+            // but allow neighboring values to fall through to the luma sort
+            ((a / 2) << 18) + a
             // These are coefficients for standard sRGB to luma conversion
             - i32::from(color.r) * 299
             - i32::from(color.g) * 587