GS/HW: Adjust DATE precision.

PCSX2 · Feb 27, 2024 · 414beb1 · 414beb1
1 parent 1d0ff59
commit 414beb1
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 20 deletions.
diff --git a/bin/resources/shaders/dx11/convert.fx b/bin/resources/shaders/dx11/convert.fx
@@ -117,7 +117,7 @@ PS_OUTPUT ps_datm1_rta_correction(PS_INPUT input)
 {
 	PS_OUTPUT output;
 
-	clip(sample_c(input.t).a - 255 / 255); // >= 0x80 pass
+	clip(sample_c(input.t).a - 254.5f / 255); // >= 0x80 pass
 
 	output.c = 0;
 
@@ -128,7 +128,7 @@ PS_OUTPUT ps_datm0_rta_correction(PS_INPUT input)
 {
 	PS_OUTPUT output;
 
-	clip(255 / 255 - sample_c(input.t).a); // < 0x80 pass (== 0x80 should not pass)
+	clip(254.5f / 255 - sample_c(input.t).a); // < 0x80 pass (== 0x80 should not pass)
 
 	output.c = 0;
 
@@ -441,7 +441,7 @@ float ps_stencil_image_init_2(PS_INPUT input)
 	: SV_Target
 {
 	float c;
-	if ((255.0f / 255.0f) < sample_c(input.t).a) // < 0x80 pass (== 0x80 should not pass)
+	if ((254.5f / 255.0f) < sample_c(input.t).a) // < 0x80 pass (== 0x80 should not pass)
 		c = float(-1);
 	else
 		c = float(0x7FFFFFFF);
@@ -452,7 +452,7 @@ float ps_stencil_image_init_3(PS_INPUT input)
 	: SV_Target
 {
 	float c;
-	if (sample_c(input.t).a < (255.0f / 255.0f)) // >= 0x80 pass
+	if (sample_c(input.t).a < (254.5f / 255.0f)) // >= 0x80 pass
 		c = float(-1);
 	else
 		c = float(0x7FFFFFFF);

diff --git a/bin/resources/shaders/opengl/convert.glsl b/bin/resources/shaders/opengl/convert.glsl
@@ -318,7 +318,7 @@ void ps_datm0()
 #ifdef ps_datm1_rta_correction
 void ps_datm1_rta_correction()
 {
-	if(sample_c().a < (255.0f / 255.0f)) // >= 0x80 pass
+	if(sample_c().a < (254.5f / 255.0f)) // >= 0x80 pass
 		discard;
 }
 #endif
@@ -328,7 +328,7 @@ void ps_datm1_rta_correction()
 #ifdef ps_datm0_rta_correction
 void ps_datm0_rta_correction()
 {
-	if((255.0f / 255.0f) < sample_c().a) // < 0x80 pass (== 0x80 should not pass)
+	if((254.5f / 255.0f) < sample_c().a) // < 0x80 pass (== 0x80 should not pass)
 		discard;
 }
 #endif
@@ -470,11 +470,11 @@ void main()
 			SV_Target0 = vec4(-1);
 	#endif
 	#ifdef ps_stencil_image_init_2
-		if((255.0f / 255.0f) < sample_c().a) // < 0x80 pass (== 0x80 should not pass)
+		if((254.5f / 255.0f) < sample_c().a) // < 0x80 pass (== 0x80 should not pass)
 			SV_Target0 = vec4(-1);
 	#endif
 	#ifdef ps_stencil_image_init_3
-		if(sample_c().a < (255.0f / 255.0f)) // >= 0x80 pass
+		if(sample_c().a < (254.5f / 255.0f)) // >= 0x80 pass
 			SV_Target0 = vec4(-1);
 	#endif
 }

diff --git a/bin/resources/shaders/opengl/tfx_fs.glsl b/bin/resources/shaders/opengl/tfx_fs.glsl
@@ -932,14 +932,14 @@ void ps_main()
 #if (PS_DATE & 3) == 1
 	// DATM == 0: Pixel with alpha equal to 1 will failed
 	#if PS_RTA_CORRECTION
-		bool bad = (255.0f / 255.0f) < rt_a;
+		bool bad = (254.5f / 255.0f) < rt_a;
 	#else
 		bool bad = (127.5f / 255.0f) < rt_a;
 	#endif
 #elif (PS_DATE & 3) == 2
 	// DATM == 1: Pixel with alpha equal to 0 will failed
 	#if PS_RTA_CORRECTION
-		bool bad = rt_a < (255.0f / 255.0f);
+		bool bad = rt_a < (254.5f / 255.0f);
 	#else
 		bool bad = rt_a < (127.5f / 255.0f);
 	#endif

diff --git a/bin/resources/shaders/vulkan/convert.glsl b/bin/resources/shaders/vulkan/convert.glsl
@@ -97,7 +97,7 @@ void ps_datm0()
 #ifdef ps_datm1_rta_correction
 void ps_datm1_rta_correction()
 {
-	if(sample_c(v_tex).a < (255.0f / 255.0f)) // >= 0x80 pass
+	if(sample_c(v_tex).a < (254.5f / 255.0f)) // >= 0x80 pass
 		discard;
 
 }
@@ -106,7 +106,7 @@ void ps_datm1_rta_correction()
 #ifdef ps_datm0_rta_correction
 void ps_datm0_rta_correction()
 {
-	if((255.0f / 255.0f) < sample_c(v_tex).a) // < 0x80 pass (== 0x80 should not pass)
+	if((254.5f / 255.0f) < sample_c(v_tex).a) // < 0x80 pass (== 0x80 should not pass)
 		discard;
 }
 #endif
@@ -450,11 +450,11 @@ void main()
 			o_col0 = vec4(-1);
 	#endif
 	#ifdef ps_stencil_image_init_2
-		if((255.0f / 255.0f) < sample_c(v_tex).a) // < 0x80 pass (== 0x80 should not pass)
+		if((254.5f / 255.0f) < sample_c(v_tex).a) // < 0x80 pass (== 0x80 should not pass)
 			o_col0 = vec4(-1);
 	#endif
 	#ifdef ps_stencil_image_init_3
-		if(sample_c(v_tex).a < (255.0f / 255.0f)) // >= 0x80 pass
+		if(sample_c(v_tex).a < (254.5f / 255.0f)) // >= 0x80 pass
 			o_col0 = vec4(-1);
 	#endif
 }

diff --git a/bin/resources/shaders/vulkan/tfx.glsl b/bin/resources/shaders/vulkan/tfx.glsl
@@ -1182,14 +1182,14 @@ void main()
 #if (PS_DATE & 3) == 1
 	// DATM == 0: Pixel with alpha equal to 1 will failed
 	#if PS_RTA_CORRECTION
-		bool bad = (255.0f / 255.0f) < rt_a;
+		bool bad = (254.5f / 255.0f) < rt_a;
 	#else
 		bool bad = (127.5f / 255.0f) < rt_a;
 	#endif
 #elif (PS_DATE & 3) == 2
 	// DATM == 1: Pixel with alpha equal to 0 will failed
 	#if PS_RTA_CORRECTION
-		bool bad = rt_a < (255.0f / 255.0f);
+		bool bad = rt_a < (254.5f / 255.0f);
 	#else
 		bool bad = rt_a < (127.5f / 255.0f);
 	#endif

diff --git a/pcsx2/GS/Renderers/Metal/convert.metal b/pcsx2/GS/Renderers/Metal/convert.metal
@@ -90,13 +90,13 @@ fragment void ps_datm1(float4 p [[position]], DirectReadTextureIn<float> tex)
 
 fragment void ps_datm0_rta_correction(float4 p [[position]], DirectReadTextureIn<float> tex)
 {
-	if (tex.read(p).a > (255.f / 255.f))
+	if (tex.read(p).a > (254.5f / 255.f))
 		discard_fragment();
 }
 
 fragment void ps_datm1_rta_correction(float4 p [[position]], DirectReadTextureIn<float> tex)
 {
-	if (tex.read(p).a < (255.f / 255.f))
+	if (tex.read(p).a < (254.5f/ 255.f))
 		discard_fragment();
 }
 
@@ -118,12 +118,12 @@ fragment float4 ps_primid_init_datm0(float4 p [[position]], DirectReadTextureIn<
 
 fragment float4 ps_primid_rta_init_datm1(float4 p [[position]], DirectReadTextureIn<float> tex)
 {
-	return tex.read(p).a < (255.f / 255.f) ? -1 : FLT_MAX;
+	return tex.read(p).a < (254.5f / 255.f) ? -1 : FLT_MAX;
 }
 
 fragment float4 ps_primid_rta_init_datm0(float4 p [[position]], DirectReadTextureIn<float> tex)
 {
-	return tex.read(p).a > (255.f / 255.f) ? -1 : FLT_MAX;
+	return tex.read(p).a > (254.5f / 255.f) ? -1 : FLT_MAX;
 }
 
 fragment float4 ps_rta_correction(float4 p [[position]], DirectReadTextureIn<float> tex)