Added Sharpen block

README.md

@@ -27,7 +27,7 @@ This model uses lookup tables for complex functions like Gaussian and Sigmoid, a
 
 In its current state, the model implements simple algorithm per module, with plans to incorporate RTL-friendly complex algorithms in future versions.
 
-![](docs/assets/infinite-isp-architecture-initial.png)
+![](docs/assets/infinite-isp-architecture.png)
 
 ISP pipeline for `InfiniteISP_ReferenceModel v1.0`
 
@@ -66,6 +66,7 @@ The table below provides a feature list of the model. The version `1.0` of the m
 | Gamma Correction             |Implements a LUT from config |
 | Auto Exposure                                 | [Auto Exposure](https://www.atlantis-press.com/article/25875811.pdf) <br> - AE stats calculations based on skewness |
 | Color Space Conversion                        | YCbCr digital <br> - BT 601 <br> - Bt 709  <br>   |YCbCr digital <br> - BT 601 <br> - Bt 709  <br> |
+| Sharpening                                    | Simple unsharp masking with strength control      |
 | Noise Reduction                               | [Non-local means filter](https://www.ipol.im/pub/art/2011/bcm_nlm/article.pdf) <br> - Implements intensity level difference  through a LUT|
 | RGB Conversion                               | Converts YCbCr digital image to RGB|
 | Invalid Region Crop    | Crops image to a fixed size|

config/configs.yml

@@ -15,7 +15,7 @@ sensor_info:
   height: 1536
 
 crop:
-  is_enable: false
+  is_enable: true
   is_debug: false
   new_width: 2592
   new_height: 1536
@@ -76,7 +76,7 @@ bayer_noise_reduction:
 auto_white_balance:
   is_enable: true
   is_debug: true
-  stats_window_offset: [16, 16, 16, 16]
+  stats_window_offset: [8, 8, 8, 8]
   underexposed_percentage: 5
   overexposed_percentage: 5
 
@@ -112,14 +112,20 @@ gamma_correction:
 auto_exposure:
   is_enable: true
   is_debug: true
-  stats_window_offset: [4, 4, 4, 4]
+  stats_window_offset: [12, 12, 12, 12]
   center_illuminance: 90
   histogram_skewness: 0.9
 
 color_space_conversion:
   conv_standard: 2
   is_save: false
 
+sharpen:
+  is_enable: true
+  sharpen_sigma: 5
+  sharpen_strength: 0.9
+  is_save: false
+
 2d_noise_reduction:
   is_enable: true
   window_size: 9
@@ -132,10 +138,10 @@ rgb_conversion:
 
 invalid_region_crop:
   # 1: 1920x1080, 2: 1920x1440
-  is_enable: false
-  crop_to_size: 2
-  height_start_idx: 40
-  width_start_idx: 40
+  is_enable: true
+  crop_to_size: 1
+  height_start_idx: 210
+  width_start_idx: 336
   is_debug: true
   is_save: false
 

infinite_isp.py

@@ -23,6 +23,7 @@
 from modules.demosaic import Demosaic
 from modules.color_correction_matrix import ColorCorrectionMatrix as CCM
 from modules.color_space_conversion import ColorSpaceConversion as CSC
+from modules.sharpen import Sharpening as SHARP
 from modules.yuv_conv_format import YUVConvFormat as YUV_C
 from modules.noise_reduction_2d import NoiseReduction2d as NR2D
 from modules.rgb_conversion import RGBConversion as RGBC
@@ -77,6 +78,7 @@ def load_config(self, config_path):
         self.parm_gmc = c_yaml["gamma_correction"]
         self.parm_ae = c_yaml["auto_exposure"]
         self.parm_csc = c_yaml["color_space_conversion"]
+        self.parm_sha = c_yaml["sharpen"]
         self.parm_2dn = c_yaml["2d_noise_reduction"]
         self.parm_rgb = c_yaml["rgb_conversion"]
         self.parm_irc = c_yaml["invalid_region_crop"]
@@ -258,10 +260,21 @@ def run_pipeline(self, visualize_output=True):
         )
         csc_img = csc.execute()
 
+        # =====================================================================
+        # Sharpening
+        sha = SHARP(
+            csc_img,
+            self.platform,
+            self.sensor_info,
+            self.parm_sha,
+            self.save_output_obj,
+        )
+        sha_img = sha.execute()
+
         # =====================================================================
         # 2d noise reduction
         nr2d = NR2D(
-            csc_img,
+            sha_img,
             self.sensor_info,
             self.parm_2dn,
             self.platform,

modules/invalid_region_crop.py

@@ -48,6 +48,8 @@ def get_idx_for_rtl(self):
             offset += 6
         if "2d_noise_reduction" in dut:
             offset += 4
+        if "sharpen" in dut:
+            offset += 4
 
         # indices for RTL
         self.h_idx_rtl = self.h_strat_idx + offset

modules/sharpen.py

@@ -0,0 +1,126 @@
+"""
+File: sharpen.py
+Description: Simple unsharp masking with frequency and strength control.
+Code / Paper  Reference:
+Author: xx-isp ([email protected])
+"""
+
+import time
+import os
+import numpy as np
+from scipy import ndimage
+
+from util.utils import create_coeff_file
+
+class Sharpening:
+    """
+    Sharpening
+    """
+
+    def __init__(self, img, platform, sensor_info, parm_sha, save_out_obj):
+        self.img = img.copy()
+        self.enable = parm_sha["is_enable"]
+        self.sensor_info = sensor_info
+        self.parm_sha = parm_sha
+        self.is_save = parm_sha["is_save"]
+        self.platform = platform
+        self.save_out_obj = save_out_obj
+
+    def gaussian_kernel(self, size_x, size_y=None, sigma_x=5, sigma_y=None):
+        """
+        Generate a Gaussian kernel for convolutions for Sharpening Algorithm
+        """
+
+        if size_y is None:
+            size_y = size_x
+        if sigma_y is None:
+            sigma_y = sigma_x
+
+        assert isinstance(size_x, int)
+        assert isinstance(size_y, int)
+
+        x_0 = size_x // 2
+        y_0 = size_y // 2
+
+        x_axis = np.arange(0, size_x, dtype=float)  # x_axis range [0:size_x]
+        y_axis = np.arange(0, size_y, dtype=float)[:, np.newaxis]
+
+        x_axis -= x_0
+        y_axis -= y_0
+
+        exp_part = (x_axis**2 / (2 * sigma_x**2)) + (y_axis**2 / (2 * sigma_y**2))
+        return 1 / ((2 * np.pi * sigma_x * sigma_y) * np.exp(-exp_part))
+
+    def apply_sharpen(self):
+        """Sharpens an image using the unsharp mask algorithm.
+
+        Args:
+            image: A numpy array of shape (height, width) representing the image to be sharpened.
+            kernel_size: The size of the Gaussian kernel to use for blurring the image.
+            sigma: The standard deviation of the Gaussian kernel.
+
+        Returns:
+            A numpy array of shape (height, width) representing the sharpened image.
+        """
+
+        sigma = self.parm_sha["sharpen_sigma"]
+        kernel_size = 9
+
+        kernel = self.gaussian_kernel(kernel_size, kernel_size, sigma, sigma)
+
+        kernel = (kernel * (2**20)).astype(np.int32)
+
+        folder_name = "coefficients/SHARP"
+        if not os.path.exists(folder_name):
+            os.makedirs(folder_name)
+        create_coeff_file(kernel, "coefficients/SHARP/luma_kernel", 20)
+
+        luma = (self.img[:, :, 0]).astype(np.int32)
+
+        # Filter the luma component of the image with a Gaussian LPF
+        # Smoothing magnitude can be controlled with the sharpen_sigma parameter
+        correlation = ndimage.correlate(luma, kernel, mode='constant', cval=0.0, origin=0)
+
+        smoothened = correlation >> 20
+
+        # Sharpen the image with upsharp mask
+        # Strength is tuneable with the sharpen_strength parameter]
+        sh_str = self.parm_sha["sharpen_strength"]
+        print("   - Sharpen  - strength = ", sh_str)
+        strength = int(sh_str * (2**10))
+
+        edge = luma - smoothened
+
+        sharpen = strength * edge
+        sharpened = sharpen >> 10
+
+        out_y = luma + sharpened
+        self.img[:, :, 0] = np.uint8(np.clip(out_y, 0, 255))
+        return self.img
+
+    def save(self):
+        """
+        Function to save module output
+        """
+        if self.is_save:
+            self.save_out_obj.save_output_array_yuv(
+                self.platform["in_file"],
+                self.img,
+                "Out_sharpen_",
+                self.platform,
+            )
+
+    def execute(self):
+        """
+        Applying sharpening to input image
+        """
+        print("Sharpen = " + str(self.enable))
+
+        if self.enable is True:
+            start = time.time()
+            s_out = self.apply_sharpen()
+            print(f"  Execution time: {time.time() - start:.3f}s")
+            self.img = s_out
+
+        self.save()
+        return self.img