GS/HW: RTA Correction, implement on DATE.

Less copies is better.

bin/resources/shaders/dx11/convert.fx

@@ -113,6 +113,28 @@ PS_OUTPUT ps_datm0(PS_INPUT input)
 	return output;
 }
 
+PS_OUTPUT ps_datm1_rta_correction(PS_INPUT input)
+{
+	PS_OUTPUT output;
+
+	clip(sample_c(input.t).a - 255 / 255); // >= 0x80 pass
+
+	output.c = 0;
+
+	return output;
+}
+
+PS_OUTPUT ps_datm0_rta_correction(PS_INPUT input)
+{
+	PS_OUTPUT output;
+
+	clip(255 / 255 - sample_c(input.t).a); // < 0x80 pass (== 0x80 should not pass)
+
+	output.c = 0;
+
+	return output;
+}
+
 PS_OUTPUT ps_rta_correction(PS_INPUT input)
 {
 	PS_OUTPUT output;
@@ -414,3 +436,25 @@ float ps_stencil_image_init_1(PS_INPUT input) : SV_Target
 		c = float(0x7FFFFFFF);
 	return c;
 }
+
+float ps_stencil_image_init_2(PS_INPUT input)
+	: SV_Target
+{
+	float c;
+	if ((255.0f / 255.0f) < sample_c(input.t).a) // < 0x80 pass (== 0x80 should not pass)
+		c = float(-1);
+	else
+		c = float(0x7FFFFFFF);
+	return c;
+}
+
+float ps_stencil_image_init_3(PS_INPUT input)
+	: SV_Target
+{
+	float c;
+	if (sample_c(input.t).a < (255.0f / 255.0f)) // >= 0x80 pass
+		c = float(-1);
+	else
+		c = float(0x7FFFFFFF);
+	return c;
+}

bin/resources/shaders/opengl/convert.glsl

@@ -313,6 +313,26 @@ void ps_datm0()
 }
 #endif
 
+// Used for DATE (stencil)
+// DATM == 1
+#ifdef ps_datm1_rta_correction
+void ps_datm1_rta_correction()
+{
+	if(sample_c().a < (255.0f / 255.0f)) // >= 0x80 pass
+		discard;
+}
+#endif
+
+// Used for DATE (stencil)
+// DATM == 0
+#ifdef ps_datm0_rta_correction
+void ps_datm0_rta_correction()
+{
+	if((255.0f / 255.0f) < sample_c().a) // < 0x80 pass (== 0x80 should not pass)
+		discard;
+}
+#endif
+
 #ifdef ps_rta_correction
 void ps_rta_correction()
 {
@@ -435,7 +455,7 @@ void ps_yuv()
 }
 #endif
 
-#if defined(ps_stencil_image_init_0) || defined(ps_stencil_image_init_1)
+#if defined(ps_stencil_image_init_0) || defined(ps_stencil_image_init_1) || defined(ps_stencil_image_init_2) || defined(ps_stencil_image_init_3)
 
 void main()
 {
@@ -449,6 +469,14 @@ void main()
 		if(sample_c().a < (127.5f / 255.0f)) // >= 0x80 pass
 			SV_Target0 = vec4(-1);
 	#endif
+	#ifdef ps_stencil_image_init_2
+		if((255.0f / 255.0f) < sample_c().a) // < 0x80 pass (== 0x80 should not pass)
+			SV_Target0 = vec4(-1);
+	#endif
+	#ifdef ps_stencil_image_init_3
+		if(sample_c().a < (255.0f / 255.0f)) // >= 0x80 pass
+			SV_Target0 = vec4(-1);
+	#endif
 }
 #endif
 

bin/resources/shaders/opengl/tfx_fs.glsl

@@ -931,10 +931,18 @@ void ps_main()
 
 #if (PS_DATE & 3) == 1
 	// DATM == 0: Pixel with alpha equal to 1 will failed
-	bool bad = (127.5f / 255.0f) < rt_a;
+	#if PS_RTA_CORRECTION
+		bool bad = (255.0f / 255.0f) < rt_a;
+	#else
+		bool bad = (127.5f / 255.0f) < rt_a;
+	#endif
 #elif (PS_DATE & 3) == 2
 	// DATM == 1: Pixel with alpha equal to 0 will failed
-	bool bad = rt_a < (127.5f / 255.0f);
+	#if PS_RTA_CORRECTION
+		bool bad = rt_a < (255.0f / 255.0f);
+	#else
+		bool bad = rt_a < (127.5f / 255.0f);
+	#endif
 #endif
 
 	if (bad) {

bin/resources/shaders/vulkan/convert.glsl

@@ -24,6 +24,8 @@ layout(location = 0) in vec2 v_tex;
 layout(location = 0) out uint o_col0;
 #elif !defined(ps_datm1) && \
 	!defined(ps_datm0) && \
+	!defined(ps_datm1_rta_correction) && \
+	!defined(ps_datm0_rta_correction) && \
 	!defined(ps_convert_rgba8_float32) && \
 	!defined(ps_convert_rgba8_float24) && \
 	!defined(ps_convert_rgba8_float16) && \
@@ -92,6 +94,23 @@ void ps_datm0()
 }
 #endif
 
+#ifdef ps_datm1_rta_correction
+void ps_datm1_rta_correction()
+{
+	if(sample_c(v_tex).a < (255.0f / 255.0f)) // >= 0x80 pass
+		discard;
+
+}
+#endif
+
+#ifdef ps_datm0_rta_correction
+void ps_datm0_rta_correction()
+{
+	if((255.0f / 255.0f) < sample_c(v_tex).a) // < 0x80 pass (== 0x80 should not pass)
+		discard;
+}
+#endif
+
 #ifdef ps_rta_correction
 void ps_rta_correction()
 {
@@ -416,7 +435,7 @@ void ps_yuv()
 }
 #endif
 
-#if defined(ps_stencil_image_init_0) || defined(ps_stencil_image_init_1)
+#if defined(ps_stencil_image_init_0) || defined(ps_stencil_image_init_1) || defined(ps_stencil_image_init_2) || defined(ps_stencil_image_init_3)
 
 void main()
 {
@@ -430,6 +449,14 @@ void main()
 		if(sample_c(v_tex).a < (127.5f / 255.0f)) // >= 0x80 pass
 			o_col0 = vec4(-1);
 	#endif
+	#ifdef ps_stencil_image_init_2
+		if((255.0f / 255.0f) < sample_c(v_tex).a) // < 0x80 pass (== 0x80 should not pass)
+			o_col0 = vec4(-1);
+	#endif
+	#ifdef ps_stencil_image_init_3
+		if(sample_c(v_tex).a < (255.0f / 255.0f)) // >= 0x80 pass
+			o_col0 = vec4(-1);
+	#endif
 }
 #endif
 

bin/resources/shaders/vulkan/tfx.glsl

@@ -1181,10 +1181,18 @@ void main()
 
 #if (PS_DATE & 3) == 1
 	// DATM == 0: Pixel with alpha equal to 1 will failed
-	bool bad = (127.5f / 255.0f) < rt_a;
+	#if PS_RTA_CORRECTION
+		bool bad = (255.0f / 255.0f) < rt_a;
+	#else
+		bool bad = (127.5f / 255.0f) < rt_a;
+	#endif
 #elif (PS_DATE & 3) == 2
 	// DATM == 1: Pixel with alpha equal to 0 will failed
-	bool bad = rt_a < (127.5f / 255.0f);
+	#if PS_RTA_CORRECTION
+		bool bad = rt_a < (255.0f / 255.0f);
+	#else
+		bool bad = rt_a < (127.5f / 255.0f);
+	#endif
 #endif
 
 	if (bad) {

pcsx2/GS/Renderers/Common/GSDevice.cpp

@@ -26,6 +26,8 @@ const char* shaderName(ShaderConvert value)
 		case ShaderConvert::RGBA8_TO_16_BITS:       return "ps_convert_rgba8_16bits";
 		case ShaderConvert::DATM_1:                 return "ps_datm1";
 		case ShaderConvert::DATM_0:                 return "ps_datm0";
+		case ShaderConvert::DATM_1_RTA_CORRECTION:  return "ps_datm1_rta_correction";
+		case ShaderConvert::DATM_0_RTA_CORRECTION:  return "ps_datm0_rta_correction";
 		case ShaderConvert::HDR_INIT:               return "ps_hdr_init";
 		case ShaderConvert::HDR_RESOLVE:            return "ps_hdr_resolve";
 		case ShaderConvert::RTA_CORRECTION:         return "ps_rta_correction";

pcsx2/GS/Renderers/Common/GSDevice.h

@@ -19,6 +19,8 @@ enum class ShaderConvert
 	RGBA8_TO_16_BITS,
 	DATM_1,
 	DATM_0,
+	DATM_1_RTA_CORRECTION,
+	DATM_0_RTA_CORRECTION,
 	HDR_INIT,
 	HDR_RESOLVE,
 	RTA_CORRECTION,
@@ -80,6 +82,8 @@ static inline bool HasStencilOutput(ShaderConvert shader)
 	{
 		case ShaderConvert::DATM_0:
 		case ShaderConvert::DATM_1:
+		case ShaderConvert::DATM_0_RTA_CORRECTION:
+		case ShaderConvert::DATM_1_RTA_CORRECTION:
 			return true;
 		default:
 			return false;
@@ -652,7 +656,7 @@ struct alignas(16) GSHWDrawConfig
 	bool require_full_barrier; ///< Require texture barrier between all prims
 
 	DestinationAlphaMode destination_alpha;
-	bool datm : 1;
+	u8 datm : 2;
 	bool line_expand : 1;
 	bool separate_alpha_pass : 1;
 	bool second_separate_alpha_pass : 1;

pcsx2/GS/Renderers/DX11/GSDevice11.cpp

@@ -2088,7 +2088,7 @@ void GSDevice11::RenderImGui()
 	m_ctx->IASetVertexBuffers(0, 1, m_vb.addressof(), &m_state.vb_stride, &vb_offset);
 }
 
-void GSDevice11::SetupDATE(GSTexture* rt, GSTexture* ds, const GSVertexPT1* vertices, bool datm)
+void GSDevice11::SetupDATE(GSTexture* rt, GSTexture* ds, const GSVertexPT1* vertices, u8 datm)
 {
 	// sfex3 (after the capcom logo), vf4 (first menu fading in), ffxii shadows, rumble roses shadows, persona4 shadows
 
@@ -2116,7 +2116,26 @@ void GSDevice11::SetupDATE(GSTexture* rt, GSTexture* ds, const GSVertexPT1* vert
 	// ps
 	PSSetShaderResource(0, rt);
 	PSSetSamplerState(m_convert.pt.get());
-	PSSetShader(m_convert.ps[static_cast<int>(datm ? ShaderConvert::DATM_1 : ShaderConvert::DATM_0)].get(), nullptr);
+	{
+		int shader = 0;
+		switch (datm)
+		{
+			case 3:
+				shader = static_cast<int>(ShaderConvert::DATM_1_RTA_CORRECTION);
+				break;
+			case 2:
+				shader = static_cast<int>(ShaderConvert::DATM_0_RTA_CORRECTION);
+				break;
+			case 1:
+				shader = static_cast<int>(ShaderConvert::DATM_1);
+				break;
+			case 0:
+			default:
+				shader = static_cast<int>(ShaderConvert::DATM_0);
+				break;
+		}
+		PSSetShader(m_convert.ps[shader].get(), nullptr);
+	}
 
 	//
 

pcsx2/GS/Renderers/DX11/GSDevice11.h

@@ -227,7 +227,7 @@ class GSDevice11 final : public GSDevice
 	{
 		wil::com_ptr_nothrow<ID3D11DepthStencilState> dss;
 		wil::com_ptr_nothrow<ID3D11BlendState> bs;
-		wil::com_ptr_nothrow<ID3D11PixelShader> primid_init_ps[2];
+		wil::com_ptr_nothrow<ID3D11PixelShader> primid_init_ps[4];
 	} m_date;
 
 	struct
@@ -322,7 +322,7 @@ class GSDevice11 final : public GSDevice
 	void DrawMultiStretchRects(const MultiStretchRect* rects, u32 num_rects, GSTexture* dTex, ShaderConvert shader) override;
 	void DoMultiStretchRects(const MultiStretchRect* rects, u32 num_rects, const GSVector2& ds);
 
-	void SetupDATE(GSTexture* rt, GSTexture* ds, const GSVertexPT1* vertices, bool datm);
+	void SetupDATE(GSTexture* rt, GSTexture* ds, const GSVertexPT1* vertices, u8 datm);
 
 	void* IAMapVertexBuffer(u32 stride, u32 count);
 	void IAUnmapVertexBuffer(u32 stride, u32 count);

pcsx2/GS/Renderers/DX12/GSDevice12.cpp

@@ -34,7 +34,7 @@ static u32 s_debug_scope_depth = 0;
 
 static bool IsDATMConvertShader(ShaderConvert i)
 {
-	return (i == ShaderConvert::DATM_0 || i == ShaderConvert::DATM_1);
+	return (i == ShaderConvert::DATM_0 || i == ShaderConvert::DATM_1 || i == ShaderConvert::DATM_0_RTA_CORRECTION || i == ShaderConvert::DATM_1_RTA_CORRECTION);
 }
 static bool IsDATEModePrimIDInit(u32 flag)
 {
@@ -2401,6 +2401,8 @@ bool GSDevice12::CompileConvertPipelines()
 			break;
 			case ShaderConvert::DATM_0:
 			case ShaderConvert::DATM_1:
+			case ShaderConvert::DATM_0_RTA_CORRECTION:
+			case ShaderConvert::DATM_1_RTA_CORRECTION:
 			{
 				gpb.ClearRenderTargets();
 				gpb.SetDepthStencilFormat(DXGI_FORMAT_D32_FLOAT_S8X24_UINT);
@@ -2478,10 +2480,10 @@ bool GSDevice12::CompileConvertPipelines()
 		}
 	}
 
-	for (u32 datm = 0; datm < 2; datm++)
+	for (u32 datm = 0; datm < 4; datm++)
 	{
-		ComPtr<ID3DBlob> ps(
-			GetUtilityPixelShader(*shader, datm ? "ps_stencil_image_init_1" : "ps_stencil_image_init_0"));
+		const std::string entry_point(StringUtil::StdStringFromFormat("ps_stencil_image_init_%d", datm));
+		ComPtr<ID3DBlob> ps(GetUtilityPixelShader(*shader, entry_point.c_str()));
 		if (!ps)
 			return false;
 
@@ -2501,7 +2503,7 @@ bool GSDevice12::CompileConvertPipelines()
 				return false;
 
 			D3D12::SetObjectName(m_date_image_setup_pipelines[ds][datm].get(),
-				TinyString::from_fmt("DATE image clear pipeline (ds={}, datm={})", ds, datm));
+				TinyString::from_fmt("DATE image clear pipeline (ds={}, datm={})", ds, (datm == 1 || datm == 3)));
 		}
 	}
 
@@ -3620,7 +3622,7 @@ void GSDevice12::SetPSConstantBuffer(const GSHWDrawConfig::PSConstantBuffer& cb)
 		m_dirty_flags |= DIRTY_FLAG_PS_CONSTANT_BUFFER;
 }
 
-void GSDevice12::SetupDATE(GSTexture* rt, GSTexture* ds, bool datm, const GSVector4i& bbox)
+void GSDevice12::SetupDATE(GSTexture* rt, GSTexture* ds, u8 datm, const GSVector4i& bbox)
 {
 	GL_PUSH("SetupDATE {%d,%d} %dx%d", bbox.left, bbox.top, bbox.width(), bbox.height());
 
@@ -3640,7 +3642,26 @@ void GSDevice12::SetupDATE(GSTexture* rt, GSTexture* ds, bool datm, const GSVect
 	OMSetRenderTargets(nullptr, ds, bbox);
 	IASetVertexBuffer(vertices, sizeof(vertices[0]), 4);
 	SetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
-	SetPipeline(m_convert[static_cast<int>(datm ? ShaderConvert::DATM_1 : ShaderConvert::DATM_0)].get());
+	{
+		int shader = 0;
+		switch (datm)
+		{
+			case 3:
+				shader = static_cast<int>(ShaderConvert::DATM_1_RTA_CORRECTION);
+				break;
+			case 2:
+				shader = static_cast<int>(ShaderConvert::DATM_0_RTA_CORRECTION);
+				break;
+			case 1:
+				shader = static_cast<int>(ShaderConvert::DATM_1);
+				break;
+			case 0:
+			default:
+				shader = static_cast<int>(ShaderConvert::DATM_0);
+				break;
+		}
+		SetPipeline(m_convert[shader].get());
+	}
 	SetStencilRef(1);
 	BeginRenderPass(D3D12_RENDER_PASS_BEGINNING_ACCESS_TYPE_NO_ACCESS, D3D12_RENDER_PASS_ENDING_ACCESS_TYPE_NO_ACCESS,
 		D3D12_RENDER_PASS_BEGINNING_ACCESS_TYPE_PRESERVE, D3D12_RENDER_PASS_ENDING_ACCESS_TYPE_PRESERVE,

pcsx2/GS/Renderers/DX12/GSDevice12.h

@@ -317,7 +317,7 @@ class GSDevice12 final : public GSDevice
 	std::array<ComPtr<ID3D12PipelineState>, NUM_INTERLACE_SHADERS> m_interlace{};
 	std::array<ComPtr<ID3D12PipelineState>, 2> m_hdr_setup_pipelines{}; // [depth]
 	std::array<ComPtr<ID3D12PipelineState>, 2> m_hdr_finish_pipelines{}; // [depth]
-	std::array<std::array<ComPtr<ID3D12PipelineState>, 2>, 2> m_date_image_setup_pipelines{}; // [depth][datm]
+	std::array<std::array<ComPtr<ID3D12PipelineState>, 2>, 4> m_date_image_setup_pipelines{}; // [depth][datm]
 	ComPtr<ID3D12PipelineState> m_fxaa_pipeline;
 	ComPtr<ID3D12PipelineState> m_shadeboost_pipeline;
 	ComPtr<ID3D12PipelineState> m_imgui_pipeline;
@@ -450,7 +450,7 @@ class GSDevice12 final : public GSDevice
 		const ID3D12PipelineState* pipeline, bool linear, bool allow_discard);
 	void DrawStretchRect(const GSVector4& sRect, const GSVector4& dRect, const GSVector2i& ds);
 
-	void SetupDATE(GSTexture* rt, GSTexture* ds, bool datm, const GSVector4i& bbox);
+	void SetupDATE(GSTexture* rt, GSTexture* ds, u8 datm, const GSVector4i& bbox);
 	GSTexture12* SetupPrimitiveTrackingDATE(GSHWDrawConfig& config, PipelineSelector& pipe);
 
 	void IASetVertexBuffer(const void* vertex, size_t stride, size_t count);