d3d12: Implement stencil states

Unfortunatly it doesnt solve stencil reflect test atm

rpcs3/Emu/RSX/D3D12/D3D12GSRender.cpp

@@ -597,6 +597,48 @@ static D3D12_LOGIC_OP getLogicOp(u32 op)
 	}
 }
 
+static D3D12_STENCIL_OP getStencilOp(u32 op)
+{
+	switch (op)
+	{
+	case GL_KEEP:
+		return D3D12_STENCIL_OP_KEEP;
+	case GL_ZERO:
+		return D3D12_STENCIL_OP_ZERO;
+	case GL_REPLACE:
+		return D3D12_STENCIL_OP_REPLACE;
+	case GL_INCR:
+		return D3D12_STENCIL_OP_INCR;
+	case GL_DECR:
+		return D3D12_STENCIL_OP_DECR;
+	case GL_INVERT:
+		return D3D12_STENCIL_OP_INVERT;
+	}
+}
+
+static D3D12_COMPARISON_FUNC getStencilFunc(u32 op)
+{
+	switch (op)
+	{
+	case GL_NEVER:
+		return D3D12_COMPARISON_FUNC_NEVER;
+	case GL_LESS:
+		return D3D12_COMPARISON_FUNC_LESS;
+	case GL_LEQUAL:
+		return D3D12_COMPARISON_FUNC_LESS_EQUAL;
+	case GL_GREATER:
+		return D3D12_COMPARISON_FUNC_GREATER;
+	case GL_GEQUAL:
+		return D3D12_COMPARISON_FUNC_GREATER_EQUAL;
+	case GL_EQUAL:
+		return D3D12_COMPARISON_FUNC_EQUAL;
+	case GL_NOTEQUAL:
+		return D3D12_COMPARISON_FUNC_NOT_EQUAL;
+	case GL_ALWAYS:
+		return D3D12_COMPARISON_FUNC_ALWAYS;
+	}
+}
+
 bool D3D12GSRender::LoadProgram()
 {
 	if (!m_cur_fragment_prog)
@@ -717,7 +759,32 @@ bool D3D12GSRender::LoadProgram()
 		LOG_ERROR(RSX, "Bad surface color target: %d", m_surface_color_target);
 	}
 
-	prop.depthEnabled = m_set_depth_test;
+	prop.DepthStencil.DepthEnable = m_set_depth_test;
+	prop.DepthStencil.DepthWriteMask = D3D12_DEPTH_WRITE_MASK_ALL;
+	prop.DepthStencil.DepthFunc = D3D12_COMPARISON_FUNC_LESS_EQUAL;
+	prop.DepthStencil.StencilEnable = m_set_stencil_test;
+	prop.DepthStencil.StencilReadMask = m_stencil_func_mask;
+	prop.DepthStencil.StencilWriteMask = m_set_stencil_mask;
+	prop.DepthStencil.FrontFace.StencilPassOp = getStencilOp(m_stencil_zpass);
+	prop.DepthStencil.FrontFace.StencilDepthFailOp = getStencilOp(m_stencil_zfail);
+	prop.DepthStencil.FrontFace.StencilFailOp = getStencilOp(m_stencil_fail);
+	prop.DepthStencil.FrontFace.StencilFunc = getStencilFunc(m_stencil_func);
+
+	if (m_set_two_sided_stencil_test_enable)
+	{
+		prop.DepthStencil.BackFace.StencilFailOp = getStencilOp(m_stencil_fail);
+		prop.DepthStencil.BackFace.StencilFunc = getStencilFunc(m_stencil_func);
+		prop.DepthStencil.BackFace.StencilPassOp = getStencilOp(m_stencil_zpass);
+		prop.DepthStencil.BackFace.StencilDepthFailOp = getStencilOp(m_stencil_zfail);
+	}
+	else
+	{
+		prop.DepthStencil.BackFace.StencilFunc = D3D12_COMPARISON_FUNC_NEVER;
+		prop.DepthStencil.BackFace.StencilFailOp = D3D12_STENCIL_OP_KEEP;
+		prop.DepthStencil.BackFace.StencilPassOp = D3D12_STENCIL_OP_KEEP;
+		prop.DepthStencil.BackFace.StencilDepthFailOp = D3D12_STENCIL_OP_KEEP;
+	}
+
 
 	prop.IASet = m_IASet;
 
@@ -774,6 +841,7 @@ void D3D12GSRender::ExecCMD()
 	}
 
 	commandList->SetGraphicsRootSignature(m_rootSignatures[m_PSO->second]);
+	commandList->OMSetStencilRef(m_stencil_func_ref);
 
 	// Constants
 	setScaleOffset();

rpcs3/Emu/RSX/D3D12/D3D12PipelineState.h

@@ -17,7 +17,7 @@ struct D3D12PipelineProperties
 	std::vector<D3D12_INPUT_ELEMENT_DESC> IASet;
 	D3D12_BLEND_DESC Blend;
 	unsigned numMRT : 3;
-	bool depthEnabled : 1;
+	D3D12_DEPTH_STENCIL_DESC DepthStencil;
 
 	bool operator==(const D3D12PipelineProperties &in) const
 	{
@@ -37,8 +37,8 @@ struct D3D12PipelineProperties
 			if (a.SemanticIndex != b.SemanticIndex)
 				return false;
 		}
-		// TODO: blend
+		// TODO: blend and depth stencil
-		return Topology == in.Topology && DepthStencilFormat == in.DepthStencilFormat && numMRT == in.numMRT && depthEnabled == in.depthEnabled;
+		return Topology == in.Topology && DepthStencilFormat == in.DepthStencilFormat && numMRT == in.numMRT;
 	}
 };
 
@@ -156,23 +156,11 @@ struct D3D12Traits
 			D3D12_CONSERVATIVE_RASTERIZATION_MODE_OFF,
 		};
 
-		static D3D12_DEPTH_STENCIL_DESC CD3D12_DEPTH_STENCIL_DESC =
-		{
-			TRUE,
-			D3D12_DEPTH_WRITE_MASK_ALL,
-			D3D12_COMPARISON_FUNC_LESS_EQUAL,
-			FALSE,
-			D3D12_DEFAULT_STENCIL_READ_MASK,
-			D3D12_DEFAULT_STENCIL_WRITE_MASK,
-		};
-
 		graphicPipelineStateDesc.BlendState = pipelineProperties.Blend;
-		graphicPipelineStateDesc.DepthStencilState = CD3D12_DEPTH_STENCIL_DESC;
+		graphicPipelineStateDesc.DepthStencilState = pipelineProperties.DepthStencil;
 		graphicPipelineStateDesc.RasterizerState = CD3D12_RASTERIZER_DESC;
 		graphicPipelineStateDesc.PrimitiveTopologyType = pipelineProperties.Topology;
 
-		graphicPipelineStateDesc.DepthStencilState.DepthEnable = pipelineProperties.depthEnabled;
-
 		graphicPipelineStateDesc.NumRenderTargets = pipelineProperties.numMRT;
 		for (unsigned i = 0; i < pipelineProperties.numMRT; i++)
 			graphicPipelineStateDesc.RTVFormats[i] = DXGI_FORMAT_R8G8B8A8_UNORM;