From 4d66e4efc40c4b4284ab451b40c4d5f2613d845a Mon Sep 17 00:00:00 2001
From: Yay295 <Yay295@users.noreply.github.com>
Date: Mon, 30 Sep 2024 19:58:03 -0500
Subject: [PATCH] handle negative intermediate value in rgb to hsv conversion

Python's `%` and C's `fmod` use a slightly different strategy. So to match Python, we need to implement it ourselves. It can also be simplified some due to the second parameter being 1.

The starting hue calculation has also been simplified by inlining some values.
---
 src/libImaging/Convert.c | 54 ++++++++++++++++++++++------------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/src/libImaging/Convert.c b/src/libImaging/Convert.c
index c8f23426105..e308835d085 100644
--- a/src/libImaging/Convert.c
+++ b/src/libImaging/Convert.c
@@ -310,40 +310,46 @@ rgb2bgr24(UINT8 *out, const UINT8 *in, int xsize) {
 }
 
 static void
-rgb2hsv_row(UINT8 *out, const UINT8 *in) {  // following colorsys.py
-    float h, s, rc, gc, bc, cr;
-    UINT8 maxc, minc;
-    UINT8 r, g, b;
-    UINT8 uh, us, uv;
-
-    r = in[0];
-    g = in[1];
-    b = in[2];
-    maxc = MAX(r, MAX(g, b));
-    minc = MIN(r, MIN(g, b));
-    uv = maxc;
+rgb2hsv_row(UINT8 *out, const UINT8 *in) {
+    // based on Python's colorsys module
+
+    const UINT8 r = in[0];
+    const UINT8 g = in[1];
+    const UINT8 b = in[2];
+
+    const UINT8 maxc = MAX(r, MAX(g, b));
+    const UINT8 minc = MIN(r, MIN(g, b));
+
+    UINT8 uh, us;
+    const UINT8 uv = maxc;
+
     if (minc == maxc) {
         uh = 0;
         us = 0;
     } else {
-        cr = (float)(maxc - minc);
-        s = cr / (float)maxc;
-        rc = ((float)(maxc - r)) / cr;
-        gc = ((float)(maxc - g)) / cr;
-        bc = ((float)(maxc - b)) / cr;
+        const UINT8 color_range = maxc - minc;
+        double h;
+
+        const double cr = (double)color_range;
         if (r == maxc) {
-            h = bc - gc;
+            h = (g - b) / cr;
         } else if (g == maxc) {
-            h = 2.0 + rc - bc;
+            h = 2.0 + (b - r) / cr;
         } else {
-            h = 4.0 + gc - rc;
+            h = 4.0 + (r - g) / cr;
         }
-        // incorrect hue happens if h/6 is negative.
-        h = fmod((h / 6.0 + 1.0), 1.0);
 
-        uh = (UINT8)CLIP8((int)(h * 255.0));
-        us = (UINT8)CLIP8((int)(s * 255.0));
+        // the modulus operator in Python does not exactly match fmod in C
+        // https://stackoverflow.com/a/3883019/3878168
+        // "h = (h/6.0) % 1.0" in Python can be computed as:
+        h = h / 6.0;
+        h = (h - floor(h)) + 1.0;
+        h = h - floor(h);
+
+        uh = (UINT8)(255.0 * h);
+        us = 255 * color_range / maxc;
     }
+
     out[0] = uh;
     out[1] = us;
     out[2] = uv;