fix some darktable-org#16534 regressions dragging pickerbox when rota…

…ted 45°

- grab any corner
- change only cursor corner
- right+click in live sample to select

src/common/color_picker.c

@@ -272,30 +272,23 @@ void dt_color_picker_backtransform_box(dt_develop_t *dev,
                                        const float *in,
                                        float *out)
 {
-  const float wd = dev->preview_pipe->iwidth;
-  const float ht = dev->preview_pipe->iheight;
-  const float wdp = dev->preview_pipe->processed_width;
-  const float htp = dev->preview_pipe->processed_height;
-  const gboolean box = num == 2;
-  if(wd < 1.0f || ht < 1.0f || wdp < 1.0f || htp < 1.0f)
-  {
-    for(int i = 0; i < num; i++)
-      out[i] = in[i];
-    return;
-  }
+  const float wd  = MAX(1, dev->preview_pipe->iwidth);
+  const float ht  = MAX(1, dev->preview_pipe->iheight);
+  const float wdp = MAX(1, dev->preview_pipe->processed_width);
+  const float htp = MAX(1, dev->preview_pipe->processed_height);
 
-  dt_boundingbox_t fbox = { wdp * in[0],
-                            htp * in[1],
-                            box ? wdp * in[2] : 0.0f,
-                            box ? htp * in[3] : 0.0f };
-  dt_dev_distort_backtransform(dev, fbox, num);
+  int out_num = num == 2 ? 4 : 1;
 
-  out[0] = fbox[0] / wd;
-  out[1] = fbox[1] / ht;
-  if(box)
+  for(int i = 0; i < out_num; i++)
   {
-    out[2] = fbox[2] / wd;
-    out[3] = fbox[3] / ht;
+    out[i * 2    ] = wdp * in[(i % 3 > 0) * 2];
+    out[i * 2 + 1] = htp * in[(i % 2) * 2 + 1];
+  }
+  dt_dev_distort_backtransform(dev, out, out_num);
+  for(int i = 0; i < out_num; i++)
+  {
+    out[i * 2    ] = CLIP(out[i * 2    ] / wd);
+    out[i * 2 + 1] = CLIP(out[i * 2 + 1] / ht);
   }
 }
 
@@ -320,10 +313,14 @@ static void _sort_coordinates(float *fbox)
 void dt_color_picker_transform_box(dt_develop_t *dev,
                                    const int num,
                                    const float *in,
-                                   float *out)
+                                   float *out,
+                                   gboolean scale)
 {
   const float wd = dev->preview_pipe->iwidth;
   const float ht = dev->preview_pipe->iheight;
+  const float wdp = scale ? dev->preview_pipe->processed_width : 1.0f;
+  const float htp = scale ? dev->preview_pipe->processed_height : 1.0f;
+
   const gboolean box = num == 2;
   if(wd < 1.0f || ht < 1.0f)
   {
@@ -332,26 +329,27 @@ void dt_color_picker_transform_box(dt_develop_t *dev,
     return;
   }
 
-  const float x0 = wd * in[0];
-  const float y0 = ht * in[1];
-  const float x1 = wd * in[2];
-  const float y1 = ht * in[3];
+  dt_pickerbox_t fbox;
+  for(int i = 0; i < 8; i += 2)
+  {
+    fbox[i    ] = wd * in[i    ];
+    fbox[i + 1] = ht * in[i + 1];
+  }
 
-  float fbox[8] = { x0,y0,  x0,y1,  x1,y0,  x1,y1 };
   dt_dev_distort_transform(dev, fbox, box ? 4 : 1);
 
   if(box) // sort the 4 point coordinates
   {
     _sort_coordinates(fbox);
-    out[0] = 0.5f * (fbox[0] + fbox[2]);
-    out[1] = 0.5f * (fbox[1] + fbox[3]);
-    out[2] = 0.5f * (fbox[4] + fbox[6]);
-    out[3] = 0.5f * (fbox[5] + fbox[7]);
+    out[0] = 0.5f * (fbox[0] + fbox[2]) / wdp;
+    out[1] = 0.5f * (fbox[1] + fbox[3]) / htp;
+    out[2] = 0.5f * (fbox[4] + fbox[6]) / wdp;
+    out[3] = 0.5f * (fbox[5] + fbox[7]) / htp;
   }
   else
   {
-    out[0] = fbox[0];
-    out[1] = fbox[1];
+    out[0] = fbox[0] / wdp;
+    out[1] = fbox[1] / htp;
   }
 }
 
@@ -374,23 +372,17 @@ gboolean dt_color_picker_box(dt_iop_module_t *module,
   const int height = roi->height;
   const gboolean isbox = sample->size == DT_LIB_COLORPICKER_SIZE_BOX;
 
-  const float bx0 = wd * sample->box[0];
-  const float by0 = ht * sample->box[1];
-  const float bx1 = wd * sample->box[2];
-  const float by1 = ht * sample->box[3];
-
-  const float sx = sample->point[0] * wd;
-  const float sy = sample->point[1] * ht;
-
   /* get absolute pixel coordinates in final preview image.
      we transform back all 4 corner locations to current module coordinates,
      sort the coordinates, and use average of 2 highest and 2 lowest for the
      resulting rectangle.
   */
-  float fbox[8] = { isbox ? bx0 : sx,   isbox ? by0 : sy,
-                    isbox ? bx0 : sx,   isbox ? by1 : sy,
-                    isbox ? bx1 : sx,   isbox ? by0 : sy,
-                    isbox ? bx1 : sx,   isbox ? by1 : sy };
+  dt_pickerbox_t fbox;
+  for(int i = 0; i < 8; i += 2)
+  {
+    fbox[i    ] = wd * (isbox ? sample->box[i    ] : sample->point[0]);
+    fbox[i + 1] = ht * (isbox ? sample->box[i + 1] : sample->point[1]);
+  }
 
   dt_dev_distort_transform_plus
     (dev, dev->preview_pipe, module->iop_order,

src/common/color_picker.h

@@ -38,7 +38,8 @@ void dt_color_picker_backtransform_box(dt_develop_t *dev,
 void dt_color_picker_transform_box(dt_develop_t *dev,
                               const int num,
                               const float *in,
-                              float *out);
+                              float *out,
+                              gboolean scale);
 gboolean dt_color_picker_box(dt_iop_module_t *module,
                              const dt_iop_roi_t *roi,
                              const dt_colorpicker_sample_t *const sample,

src/common/darktable.h

@@ -284,6 +284,7 @@ struct dt_colorspaces_t;
 struct dt_l10n_t;
 
 typedef float dt_boundingbox_t[4];  //(x,y) of upperleft, then (x,y) of lowerright
+typedef float dt_pickerbox_t[8];
 
 typedef enum dt_debug_thread_t
 {

src/gui/color_picker_proxy.c

@@ -17,7 +17,7 @@
 */
 
 #include "gui/color_picker_proxy.h"
-#include "libs/colorpicker.h"
+#include "common/color_picker.h"
 #include "bauhaus/bauhaus.h"
 #include "libs/lib.h"
 #include "control/control.h"
@@ -90,7 +90,7 @@ static gboolean _record_point_area(dt_iop_color_picker_t *self)
         }
       }
     else if(sample->size == DT_LIB_COLORPICKER_SIZE_BOX)
-      for(int k = 0; k < 4; k++)
+      for(int k = 0; k < 8; k++)
       {
         if(self->pick_box[k] != sample->box[k])
         {
@@ -134,32 +134,6 @@ void dt_iop_color_picker_reset(dt_iop_module_t *module,
   }
 }
 
-static void _backtransform_box(const int num,
-                               const float *in,
-                               float *out)
-{
-  dt_develop_t *dev = darktable.develop;
-  const float wd = MAX(1, dev->preview_pipe->iwidth);
-  const float ht = MAX(1, dev->preview_pipe->iheight);
-  const float wdp = dev->preview_pipe->processed_width;
-  const float htp = dev->preview_pipe->processed_height;
-  const gboolean box = num == 2;
-
-  dt_boundingbox_t fbox = { wdp * in[0],
-                            htp * in[1],
-                            box ? wdp * in[2] : 0.0f,
-                            box ? htp * in[3] : 0.0f };
-  dt_dev_distort_backtransform(dev, fbox, num);
-
-  out[0] = CLIP(fbox[0] / wd);
-  out[1] = CLIP(fbox[1] / ht);
-  if(box)
-  {
-    out[2] = CLIP(fbox[2] / wd);
-    out[3] = CLIP(fbox[3] / ht);
-  }
-}
-
 static void _init_picker(dt_iop_color_picker_t *picker,
                          dt_iop_module_t *module,
                          const dt_iop_color_picker_flags_t flags,
@@ -229,7 +203,7 @@ static gboolean _color_picker_callback_button_press(GtkWidget *button,
          && self->pick_box[2] == 1.0f && self->pick_box[3] == 1.0f)
       {
         dt_boundingbox_t reset = { 0.02f, 0.02f, 0.98f, 0.98f };
-        _backtransform_box(2, reset, self->pick_box);
+        dt_color_picker_backtransform_box(darktable.develop, 2, reset, self->pick_box);
       }
       dt_lib_colorpicker_set_box_area(darktable.lib, self->pick_box);
     }
@@ -238,7 +212,7 @@ static gboolean _color_picker_callback_button_press(GtkWidget *button,
       if(self->pick_pos[0] == 0.0f && self->pick_pos[1] == 0.0f)
       {
         dt_boundingbox_t middle = { 0.5f, 0.5f };
-        _backtransform_box(1, middle, self->pick_pos);
+        dt_color_picker_backtransform_box(darktable.develop, 1, middle, self->pick_pos);
       }
       dt_lib_colorpicker_set_point(darktable.lib, self->pick_pos);
     }

src/gui/color_picker_proxy.h

@@ -63,7 +63,7 @@ typedef struct dt_iop_color_picker_t
   // the picker request for the primary picker when this picker is
   // activated, and will remember the most recent picker position
   float pick_pos[2];
-  dt_boundingbox_t pick_box;
+  dt_pickerbox_t pick_box;
   gboolean changed;
 } dt_iop_color_picker_t;
 

src/libs/colorpicker.c

@@ -245,12 +245,12 @@ static void _update_samples_output(dt_lib_module_t *self)
 /* set sample area proxy impl */
 
 static void _set_sample_box_area(dt_lib_module_t *self,
-                                 const dt_boundingbox_t box)
+                                 const dt_pickerbox_t box)
 {
   dt_lib_colorpicker_t *data = self->data;
 
   // primary sample always follows/represents current picker
-  for(int k = 0; k < 4; k++)
+  for(int k = 0; k < 8; k++)
     data->primary_sample.box[k] = box[k];
 
   _update_size(self, DT_LIB_COLORPICKER_SIZE_BOX);

src/libs/colorpicker.h

@@ -45,7 +45,7 @@ typedef struct dt_colorpicker_sample_t
   // whether from colorpicker lib or an iop. They are used for showing
   // the sample in the center view, and sampling in the pixelpipe.
   float point[2];
-  dt_boundingbox_t box;
+  dt_pickerbox_t box;
   dt_lib_colorpicker_size_t size;
   gboolean denoise;
   gboolean pick_output;

src/libs/lib.c

@@ -1466,7 +1466,7 @@ gchar *dt_lib_get_localized_name(const gchar *plugin_name)
 }
 
 void dt_lib_colorpicker_set_box_area(dt_lib_t *lib,
-                                     const dt_boundingbox_t box)
+                                     const dt_pickerbox_t box)
 {
   if(!lib->proxy.colorpicker.module || !lib->proxy.colorpicker.set_sample_box_area) return;
   lib->proxy.colorpicker.set_sample_box_area(lib->proxy.colorpicker.module, box);

src/libs/lib.h

@@ -213,7 +213,7 @@ gboolean dt_lib_presets_can_autoapply(dt_lib_module_t *mod);
 
 /** set the colorpicker area selection tool and size, box[k] 0.0 - 1.0 */
 void dt_lib_colorpicker_set_box_area(dt_lib_t *lib,
-                                     const dt_boundingbox_t box);
+                                     const dt_pickerbox_t box);
 
 /** set the colorpicker point selection tool and position */
 void dt_lib_colorpicker_set_point(dt_lib_t *lib,