Implement quad pixels characters for drawing bitmaps (#137)

* QuadPixels using threshold for black and white

* properly determine colors for quad pixel

* lint fixes

* Automated JetBrains cleanup

Co-authored-by:  <[email protected]>

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

src/Consolonia.Core/Drawing/DrawingContextImpl.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using Avalonia;
 using Avalonia.Media;
 using Avalonia.Media.Immutable;
@@ -59,22 +60,37 @@ public void DrawBitmap(IBitmapImpl source, double opacity, Rect sourceRect, Rect
             var targetRect = new Rect(Transform.Transform(new Point(destRect.TopLeft.X, destRect.TopLeft.Y)),
                 Transform.Transform(new Point(destRect.BottomRight.X, destRect.BottomRight.Y)));
             var bmp = (BitmapImpl)source;
-            using var bitmap = new SKBitmap((int)targetRect.Width, (int)targetRect.Height);
+
+            // resize source to be target rect * 2 so we can map to quad pixels
+            using var bitmap = new SKBitmap((int)targetRect.Width * 2, (int)targetRect.Height * 2);
             using var canvas = new SKCanvas(bitmap);
-            canvas.DrawBitmap(bmp.Bitmap, new SKRect(0, 0, (float)targetRect.Width, (float)targetRect.Height),
+            canvas.DrawBitmap(bmp.Bitmap, new SKRect(0, 0, bitmap.Width, bitmap.Height),
                 new SKPaint { FilterQuality = SKFilterQuality.Medium });
 
-            // Rect clip = CurrentClip.Intersect(destRect);
-            int width = bitmap.Info.Width;
-            int height = bitmap.Info.Height;
-            for (int x = 0; x < width; x++)
-            for (int y = 0; y < height; y++)
+            for (int y = 0; y < bitmap.Info.Height; y += 2)
+            for (int x = 0; x < bitmap.Info.Width; x += 2)
             {
-                int px = (int)targetRect.TopLeft.X + x;
-                int py = (int)targetRect.TopLeft.Y + y;
-                SKColor skColor = bitmap.GetPixel(x, y);
-                Color color = Color.FromRgb(skColor.Red, skColor.Green, skColor.Blue);
-                var imagePixel = new Pixel('█', color);
+                // NOTE: we divide by 2 because we are working with quad pixels,
+                // // the bitmap has twice the horizontal and twice the vertical of the target rect.
+                int px = (int)targetRect.TopLeft.X + x / 2;
+                int py = (int)targetRect.TopLeft.Y + y / 2;
+
+                // get the quad pixel the bitmap
+                var quadColors = new[]
+                {
+                    bitmap.GetPixel(x, y), bitmap.GetPixel(x + 1, y),
+                    bitmap.GetPixel(x, y + 1), bitmap.GetPixel(x + 1, y + 1)
+                };
+
+                // map it to a single char to represet the 4 pixels
+                char quadPixel = GetQuadPixelCharacter(quadColors);
+
+                // get the combined colors for the quad pixel
+                Color foreground = GetForegroundColorForQuadPixel(quadColors, quadPixel);
+                Color background = GetBackgroundColorForQuadPixel(quadColors, quadPixel);
+
+                var imagePixel = new Pixel(new PixelForeground(new SimpleSymbol(quadPixel), color: foreground),
+                    new PixelBackground(background));
                 CurrentClip.ExecuteWithClipping(new Point(px, py),
                     () =>
                     {
@@ -495,5 +511,231 @@ private void DrawPixelAndMoveHead(ref Point head, Line line, LineStyle? lineStyl
                 }
             }
         }
+
+        /// <summary>
+        ///     given 4 colors return quadPixel character which is suitable to represent the colors
+        /// </summary>
+        /// <param name="colors"></param>
+        /// <returns></returns>
+        /// <exception cref="NotImplementedException"></exception>
+        private static char GetQuadPixelCharacter(params SKColor[] colors)
+        {
+            char character = GetColorsPattern(colors) switch
+            {
+                "FFFF" => ' ',
+                "TFFF" => '▘',
+                "FTFF" => '▝',
+                "FFTF" => '▖',
+                "FFFT" => '▗',
+                "TFFT" => '▚',
+                "FTTF" => '▞',
+                "TFTF" => '▌',
+                "FTFT" => '▐',
+                "FFTT" => '▄',
+                "TTFF" => '▀',
+                "TTTF" => '▛',
+                "TTFT" => '▜',
+                "TFTT" => '▙',
+                "FTTT" => '▟',
+                "TTTT" => '█',
+                _ => throw new NotImplementedException()
+            };
+            return character;
+        }
+
+
+        /// <summary>
+        ///     Combine the colors for the white part of the quad pixel character.
+        /// </summary>
+        /// <param name="pixelColors">4 colors</param>
+        /// <param name="quadPixel"></param>
+        /// <returns>foreground color</returns>
+        /// <exception cref="NotImplementedException"></exception>
+        private static Color GetForegroundColorForQuadPixel(SKColor[] pixelColors, char quadPixel)
+        {
+            if (pixelColors.Length != 4)
+                throw new ArgumentException($"{nameof(pixelColors)} must have 4 elements.");
+
+            SKColor skColor = quadPixel switch
+            {
+                ' ' => SKColors.Transparent,
+                '▘' => pixelColors[0],
+                '▝' => pixelColors[1],
+                '▖' => pixelColors[2],
+                '▗' => pixelColors[3],
+                '▚' => CombineColors(pixelColors[0], pixelColors[2]),
+                '▞' => CombineColors(pixelColors[1], pixelColors[3]),
+                '▌' => CombineColors(pixelColors[0], pixelColors[2]),
+                '▐' => CombineColors(pixelColors[1], pixelColors[3]),
+                '▄' => CombineColors(pixelColors[2], pixelColors[3]),
+                '▀' => CombineColors(pixelColors[0], pixelColors[1]),
+                '▛' => CombineColors(pixelColors[0], pixelColors[1], pixelColors[2]),
+                '▜' => CombineColors(pixelColors[0], pixelColors[1], pixelColors[3]),
+                '▙' => CombineColors(pixelColors[0], pixelColors[2], pixelColors[3]),
+                '▟' => CombineColors(pixelColors[1], pixelColors[2], pixelColors[3]),
+                '█' => CombineColors(pixelColors.ToArray()),
+                _ => throw new NotImplementedException()
+            };
+
+            return Color.FromRgb(skColor.Red, skColor.Green, skColor.Blue);
+        }
+
+
+        /// <summary>
+        ///     Combine the colors for the black part of the quad pixel character.
+        /// </summary>
+        /// <param name="pixelColors"></param>
+        /// <param name="quadPixel"></param>
+        /// <returns></returns>
+        /// <exception cref="NotImplementedException"></exception>
+        private static Color GetBackgroundColorForQuadPixel(SKColor[] pixelColors, char quadPixel)
+        {
+            SKColor skColor = quadPixel switch
+            {
+                ' ' => CombineColors(pixelColors.ToArray()),
+                '▘' => CombineColors(pixelColors[1], pixelColors[2], pixelColors[3]),
+                '▝' => CombineColors(pixelColors[0], pixelColors[2], pixelColors[3]),
+                '▖' => CombineColors(pixelColors[0], pixelColors[1], pixelColors[3]),
+                '▗' => CombineColors(pixelColors[0], pixelColors[1], pixelColors[2]),
+                '▚' => CombineColors(pixelColors[1], pixelColors[2]),
+                '▞' => CombineColors(pixelColors[0], pixelColors[3]),
+                '▌' => CombineColors(pixelColors[1], pixelColors[3]),
+                '▐' => CombineColors(pixelColors[0], pixelColors[2]),
+                '▄' => CombineColors(pixelColors[0], pixelColors[1]),
+                '▀' => CombineColors(pixelColors[2], pixelColors[3]),
+                '▛' => pixelColors[3],
+                '▜' => pixelColors[2],
+                '▙' => pixelColors[1],
+                '▟' => pixelColors[0],
+                '█' => SKColors.Transparent,
+                _ => throw new NotImplementedException()
+            };
+            return Color.FromArgb(skColor.Alpha, skColor.Red, skColor.Green, skColor.Blue);
+        }
+
+        //private static SKColor CombineColors(params SKColor[] colors)
+        //{
+        //    return new SKColor((byte)colors.Average(c => c.Red),
+        //                       (byte)colors.Average(c => c.Green),
+        //                       (byte)colors.Average(c => c.Blue),
+        //                       (byte)colors.Average(c => c.Alpha));
+        //}
+
+        private static SKColor CombineColors(params SKColor[] colors)
+        {
+            float finalRed = 0;
+            float finalGreen = 0;
+            float finalBlue = 0;
+            float finalAlpha = 0;
+
+            foreach (SKColor color in colors)
+            {
+                float alphaRatio = color.Alpha / 255.0f;
+                finalRed = (finalRed * finalAlpha + color.Red * alphaRatio) / (finalAlpha + alphaRatio);
+                finalGreen = (finalGreen * finalAlpha + color.Green * alphaRatio) / (finalAlpha + alphaRatio);
+                finalBlue = (finalBlue * finalAlpha + color.Blue * alphaRatio) / (finalAlpha + alphaRatio);
+                finalAlpha += alphaRatio * (1 - finalAlpha);
+            }
+
+            byte red = (byte)Math.Clamp(finalRed, 0, 255);
+            byte green = (byte)Math.Clamp(finalGreen, 0, 255);
+            byte blue = (byte)Math.Clamp(finalBlue, 0, 255);
+            byte alpha = (byte)Math.Clamp(finalAlpha * 255, 0, 255);
+
+            return new SKColor(red, green, blue, alpha);
+        }
+
+        /// <summary>
+        ///     Cluster quad colors into a pattern (like: TTFF) based on relative closeness
+        /// </summary>
+        /// <param name="colors"></param>
+        /// <returns>T or F for each color as a string</returns>
+        /// <exception cref="ArgumentException"></exception>
+        private static string GetColorsPattern(SKColor[] colors)
+        {
+            if (colors.Length != 4) throw new ArgumentException("Array must contain exactly 4 colors.");
+
+            // Initial guess: two clusters with the first two colors as centers
+            SKColor[] clusterCenters = { colors[0], colors[1] };
+            int[] clusters = new int[colors.Length];
+
+            for (int iteration = 0; iteration < 10; iteration++) // limit iterations to avoid infinite loop
+            {
+                // Assign colors to the closest cluster center
+                for (int i = 0; i < colors.Length; i++) clusters[i] = GetColorCluster(colors[i], clusterCenters);
+
+                // Recalculate cluster centers
+                var newClusterCenters = new SKColor[2];
+                for (int cluster = 0; cluster < 2; cluster++)
+                {
+                    var clusteredColors = colors.Where((_, i) => clusters[i] == cluster).ToList();
+                    if (clusteredColors.Any())
+                        newClusterCenters[cluster] = GetAverageColor(clusteredColors);
+                    if (clusteredColors.Count == 4)
+                        if (clusteredColors.All(c => c.Alpha == 0))
+                            return "FFFF";
+                    //    return "TTTT";
+                }
+
+                // Check for convergence
+                if (newClusterCenters.SequenceEqual(clusterCenters))
+                    break;
+
+                clusterCenters = newClusterCenters;
+            }
+
+            // Determine which cluster is lower and which is higher
+            int lowerCluster = GetColorBrightness(clusterCenters[0]) < GetColorBrightness(clusterCenters[1]) ? 0 : 1;
+            int higherCluster = 1 - lowerCluster;
+
+            // Replace colors with 0 for lower cluster and 1 for higher cluster
+            var sb = new StringBuilder();
+            for (int i = 0; i < colors.Length; i++) sb.Append(clusters[i] == higherCluster ? 'T' : 'F');
+
+            return sb.ToString();
+        }
+
+        private static int GetColorCluster(SKColor color, SKColor[] clusterCenters)
+        {
+            double minDistance = double.MaxValue;
+            int closestCluster = -1;
+
+            for (int i = 0; i < clusterCenters.Length; i++)
+            {
+                double distance = GetColorDistance(color, clusterCenters[i]);
+                if (distance < minDistance)
+                {
+                    minDistance = distance;
+                    closestCluster = i;
+                }
+            }
+
+            return closestCluster;
+        }
+
+        private static double GetColorDistance(SKColor c1, SKColor c2)
+        {
+            return Math.Sqrt(
+                Math.Pow(c1.Red - c2.Red, 2) +
+                Math.Pow(c1.Green - c2.Green, 2) +
+                Math.Pow(c1.Blue - c2.Blue, 2) +
+                Math.Pow(c1.Alpha - c2.Alpha, 2)
+            );
+        }
+
+        private static SKColor GetAverageColor(List<SKColor> colors)
+        {
+            byte averageRed = (byte)colors.Average(c => c.Red);
+            byte averageGreen = (byte)colors.Average(c => c.Green);
+            byte averageBlue = (byte)colors.Average(c => c.Blue);
+            byte averageAlpha = (byte)colors.Average(c => c.Alpha);
+
+            return new SKColor(averageRed, averageGreen, averageBlue, averageAlpha);
+        }
+
+        private static double GetColorBrightness(SKColor color)
+        {
+            return 0.299 * color.Red + 0.587 * color.Green + 0.114 * color.Blue + color.Alpha;
+        }
     }
 }

src/Consolonia.Core/Drawing/PixelBufferImplementation/Pixel.cs

@@ -66,7 +66,11 @@ public Pixel Blend(Pixel pixelAbove)
             switch (pixelAbove.Background.Mode)
             {
                 case PixelBackgroundMode.Colored:
-                    return pixelAbove;
+                    // merge pixelAbove into this pixel using alpha channel.
+                    Color mergedColors = MergeColors(Background.Color, pixelAbove.Background.Color);
+                    return new Pixel(pixelAbove.Foreground,
+                        new PixelBackground(mergedColors));
+
                 case PixelBackgroundMode.Transparent:
                     // when a textdecoration of underline happens a DrawLine() is called over the top of the a pixel with non-zero symbol.
                     // this detects this situation and eats the draw line, turning it into a textdecoration
@@ -97,5 +101,23 @@ Foreground.Symbol is SimpleSymbol simpleSymbol &&
         {
             return (Foreground.Shade(), Background.Shade());
         }
+
+        /// <summary>
+        ///     merge colors with alpha blending
+        /// </summary>
+        /// <param name="target"></param>
+        /// <param name="source"></param>
+        /// <returns>source blended into target</returns>
+        private static Color MergeColors(Color target, Color source)
+        {
+            float alphaB = source.A / 255.0f;
+            float inverseAlphaB = 1.0f - alphaB;
+
+            byte red = (byte)(target.R * inverseAlphaB + source.R * alphaB);
+            byte green = (byte)(target.G * inverseAlphaB + source.G * alphaB);
+            byte blue = (byte)(target.B * inverseAlphaB + source.B * alphaB);
+
+            return new Color(0xFF, red, green, blue);
+        }
     }
 }

src/Consolonia.Core/Drawing/PixelBufferImplementation/PixelBackground.cs

@@ -5,6 +5,12 @@ namespace Consolonia.Core.Drawing.PixelBufferImplementation
 {
     public readonly struct PixelBackground
     {
+        public PixelBackground(Color color)
+        {
+            Mode = color.A == 0 ? PixelBackgroundMode.Transparent : PixelBackgroundMode.Colored;
+            Color = color;
+        }
+
         public PixelBackground(PixelBackgroundMode mode, Color? color = null)
         {
             Color = color ?? Colors.Black;

src/Consolonia.Gallery/Gallery/GalleryViews/GalleryImage.axaml

@@ -9,7 +9,7 @@
              x:Class="Consolonia.Gallery.Gallery.GalleryViews.GalleryImage">
 
     <Grid RowDefinitions="20 *">
-        <Image Source="avares://Consolonia.Gallery/Resources/happy.jpg" Margin="0 1 1 0" />
+        <Image Source="avares://Consolonia.Gallery/Resources/happy.png" Margin="0 1 1 0" />
 
         <Border BorderBrush="Gray" BorderThickness="1" Grid.Row="1">
             <ScrollViewer >