Skip to content

Commit

Permalink
Added Sharpen block (#21)
Browse files Browse the repository at this point in the history
* Added Sharpen block

* Resolved linting issues

* linting issues resolved

* updated diagram and README

* Added print statement in sharpen to print sharpen strength value for isp logs

* Added print statement in sharpen to print sharpen strength value for isp logs
  • Loading branch information
mfaizan-10xe authored Aug 30, 2024
1 parent dd8dd8b commit b269caa
Show file tree
Hide file tree
Showing 6 changed files with 157 additions and 9 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ This model uses lookup tables for complex functions like Gaussian and Sigmoid, a

In its current state, the model implements simple algorithm per module, with plans to incorporate RTL-friendly complex algorithms in future versions.

![](docs/assets/infinite-isp-architecture-initial.png)
![](docs/assets/infinite-isp-architecture.png)

ISP pipeline for `InfiniteISP_ReferenceModel v1.0`

Expand Down Expand Up @@ -66,6 +66,7 @@ The table below provides a feature list of the model. The version `1.0` of the m
| Gamma Correction |Implements a LUT from config |
| Auto Exposure | [Auto Exposure](https://www.atlantis-press.com/article/25875811.pdf) <br> - AE stats calculations based on skewness |
| Color Space Conversion | YCbCr digital <br> - BT 601 <br> - Bt 709 <br> |YCbCr digital <br> - BT 601 <br> - Bt 709 <br> |
| Sharpening | Simple unsharp masking with strength control |
| Noise Reduction | [Non-local means filter](https://www.ipol.im/pub/art/2011/bcm_nlm/article.pdf) <br> - Implements intensity level difference through a LUT|
| RGB Conversion | Converts YCbCr digital image to RGB|
| Invalid Region Crop | Crops image to a fixed size|
Expand Down
20 changes: 13 additions & 7 deletions config/configs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ sensor_info:
height: 1536

crop:
is_enable: false
is_enable: true
is_debug: false
new_width: 2592
new_height: 1536
Expand Down Expand Up @@ -76,7 +76,7 @@ bayer_noise_reduction:
auto_white_balance:
is_enable: true
is_debug: true
stats_window_offset: [16, 16, 16, 16]
stats_window_offset: [8, 8, 8, 8]
underexposed_percentage: 5
overexposed_percentage: 5

Expand Down Expand Up @@ -112,14 +112,20 @@ gamma_correction:
auto_exposure:
is_enable: true
is_debug: true
stats_window_offset: [4, 4, 4, 4]
stats_window_offset: [12, 12, 12, 12]
center_illuminance: 90
histogram_skewness: 0.9

color_space_conversion:
conv_standard: 2
is_save: false

sharpen:
is_enable: true
sharpen_sigma: 5
sharpen_strength: 0.9
is_save: false

2d_noise_reduction:
is_enable: true
window_size: 9
Expand All @@ -132,10 +138,10 @@ rgb_conversion:

invalid_region_crop:
# 1: 1920x1080, 2: 1920x1440
is_enable: false
crop_to_size: 2
height_start_idx: 40
width_start_idx: 40
is_enable: true
crop_to_size: 1
height_start_idx: 210
width_start_idx: 336
is_debug: true
is_save: false

Expand Down
Binary file added docs/assets/infinite-isp-architecture.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
15 changes: 14 additions & 1 deletion infinite_isp.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from modules.demosaic import Demosaic
from modules.color_correction_matrix import ColorCorrectionMatrix as CCM
from modules.color_space_conversion import ColorSpaceConversion as CSC
from modules.sharpen import Sharpening as SHARP
from modules.yuv_conv_format import YUVConvFormat as YUV_C
from modules.noise_reduction_2d import NoiseReduction2d as NR2D
from modules.rgb_conversion import RGBConversion as RGBC
Expand Down Expand Up @@ -77,6 +78,7 @@ def load_config(self, config_path):
self.parm_gmc = c_yaml["gamma_correction"]
self.parm_ae = c_yaml["auto_exposure"]
self.parm_csc = c_yaml["color_space_conversion"]
self.parm_sha = c_yaml["sharpen"]
self.parm_2dn = c_yaml["2d_noise_reduction"]
self.parm_rgb = c_yaml["rgb_conversion"]
self.parm_irc = c_yaml["invalid_region_crop"]
Expand Down Expand Up @@ -258,10 +260,21 @@ def run_pipeline(self, visualize_output=True):
)
csc_img = csc.execute()

# =====================================================================
# Sharpening
sha = SHARP(
csc_img,
self.platform,
self.sensor_info,
self.parm_sha,
self.save_output_obj,
)
sha_img = sha.execute()

# =====================================================================
# 2d noise reduction
nr2d = NR2D(
csc_img,
sha_img,
self.sensor_info,
self.parm_2dn,
self.platform,
Expand Down
2 changes: 2 additions & 0 deletions modules/invalid_region_crop.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ def get_idx_for_rtl(self):
offset += 6
if "2d_noise_reduction" in dut:
offset += 4
if "sharpen" in dut:
offset += 4

# indices for RTL
self.h_idx_rtl = self.h_strat_idx + offset
Expand Down
126 changes: 126 additions & 0 deletions modules/sharpen.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
"""
File: sharpen.py
Description: Simple unsharp masking with frequency and strength control.
Code / Paper Reference:
Author: xx-isp ([email protected])
"""

import time
import os
import numpy as np
from scipy import ndimage

from util.utils import create_coeff_file

class Sharpening:
"""
Sharpening
"""

def __init__(self, img, platform, sensor_info, parm_sha, save_out_obj):
self.img = img.copy()
self.enable = parm_sha["is_enable"]
self.sensor_info = sensor_info
self.parm_sha = parm_sha
self.is_save = parm_sha["is_save"]
self.platform = platform
self.save_out_obj = save_out_obj

def gaussian_kernel(self, size_x, size_y=None, sigma_x=5, sigma_y=None):
"""
Generate a Gaussian kernel for convolutions for Sharpening Algorithm
"""

if size_y is None:
size_y = size_x
if sigma_y is None:
sigma_y = sigma_x

assert isinstance(size_x, int)
assert isinstance(size_y, int)

x_0 = size_x // 2
y_0 = size_y // 2

x_axis = np.arange(0, size_x, dtype=float) # x_axis range [0:size_x]
y_axis = np.arange(0, size_y, dtype=float)[:, np.newaxis]

x_axis -= x_0
y_axis -= y_0

exp_part = (x_axis**2 / (2 * sigma_x**2)) + (y_axis**2 / (2 * sigma_y**2))
return 1 / ((2 * np.pi * sigma_x * sigma_y) * np.exp(-exp_part))

def apply_sharpen(self):
"""Sharpens an image using the unsharp mask algorithm.
Args:
image: A numpy array of shape (height, width) representing the image to be sharpened.
kernel_size: The size of the Gaussian kernel to use for blurring the image.
sigma: The standard deviation of the Gaussian kernel.
Returns:
A numpy array of shape (height, width) representing the sharpened image.
"""

sigma = self.parm_sha["sharpen_sigma"]
kernel_size = 9

kernel = self.gaussian_kernel(kernel_size, kernel_size, sigma, sigma)

kernel = (kernel * (2**20)).astype(np.int32)

folder_name = "coefficients/SHARP"
if not os.path.exists(folder_name):
os.makedirs(folder_name)
create_coeff_file(kernel, "coefficients/SHARP/luma_kernel", 20)

luma = (self.img[:, :, 0]).astype(np.int32)

# Filter the luma component of the image with a Gaussian LPF
# Smoothing magnitude can be controlled with the sharpen_sigma parameter
correlation = ndimage.correlate(luma, kernel, mode='constant', cval=0.0, origin=0)

smoothened = correlation >> 20

# Sharpen the image with upsharp mask
# Strength is tuneable with the sharpen_strength parameter]
sh_str = self.parm_sha["sharpen_strength"]
print(" - Sharpen - strength = ", sh_str)
strength = int(sh_str * (2**10))

edge = luma - smoothened

sharpen = strength * edge
sharpened = sharpen >> 10

out_y = luma + sharpened
self.img[:, :, 0] = np.uint8(np.clip(out_y, 0, 255))
return self.img

def save(self):
"""
Function to save module output
"""
if self.is_save:
self.save_out_obj.save_output_array_yuv(
self.platform["in_file"],
self.img,
"Out_sharpen_",
self.platform,
)

def execute(self):
"""
Applying sharpening to input image
"""
print("Sharpen = " + str(self.enable))

if self.enable is True:
start = time.time()
s_out = self.apply_sharpen()
print(f" Execution time: {time.time() - start:.3f}s")
self.img = s_out

self.save()
return self.img

0 comments on commit b269caa

Please sign in to comment.