UE5でエンジン改造してセルシェーディングする

https://forums.unrealengine.com/t/creating-a-custom-shading-model-in-unreal-engine-5/243623
https://medium.com/@solaslin/learning-unreal-engine-4-implement-cel-shading-w-outline-using-custom-shading-model-in-ue4-22-1-775bccdb9ffb
この記事を参考にさせていただきやってみた。

• バージョンはUnrealEngine5.1
  • gitのSHA: 19f3db469c9ed2cff7f554ddcc99a44eb06e8bd4
• コード変更した箇所には「//@Begin NakCustom」「//@End NakCustom」でマークをつけた。
• 元記事の影響で２つのMaterial出力ピンが使えるようにしてあるが、実際には使ってない。元記事ではこれでシェーディング階調を制御するようにしていたようだ。
• この変更を入れた状態で全然運用していないので何か不具合が出る可能性もあるのであしからず。
• 変更箇所はgitのPatchを書き出したものをコピペした、@@ のところは行番号に対応している（はず）
• CustomBxDFで引数のNoLではなくContext.NoLを使っている理由は、引数のものでShadingすると影がガタつくからだ。
  • https://cedec.cesa.or.jp/2021/session/detail/s605da5b7ddab8.html
  • こちらの資料でもNoLをBasePassで求める理由としてG-Bufferの精度問題というのが取り上げられていたので、その辺に原因があるのかもしれない（未確認）

DynamicGIとReflectionをOFFにしてあるので余計にパキッとした感じになっている。左は普通のLitMaterial、右が今回のカスタム版。

左がカスタム版。真ん中は普通のLitMaterial。右の緑Sphereは比較用にUnlitMaterialで同じようにCelShadingしているもの。

裏側から見た感じ。右がカスタム版のSphereだが陰の部分の色にグラデーションがかかっている。Diffuse_Lambertの結果がこのようになっているのだと思うのだが、どういう要因なんだろう…。

Engine/Shaders/Private/BasePassCommon.ush

@@ -39,7 +39,9 @@
 #define USES_GBUFFER						(FEATURE_LEVEL >= FEATURE_LEVEL_SM4 && (MATERIALBLENDING_SOLID || MATERIALBLENDING_MASKED) && !FORWARD_SHADING)
 
 // Only some shader models actually need custom data.
-#define WRITES_CUSTOMDATA_TO_GBUFFER		(USES_GBUFFER && (MATERIAL_SHADINGMODEL_SUBSURFACE || MATERIAL_SHADINGMODEL_PREINTEGRATED_SKIN || MATERIAL_SHADINGMODEL_SUBSURFACE_PROFILE || MATERIAL_SHADINGMODEL_CLEAR_COAT || MATERIAL_SHADINGMODEL_TWOSIDED_FOLIAGE || MATERIAL_SHADINGMODEL_HAIR || MATERIAL_SHADINGMODEL_CLOTH || MATERIAL_SHADINGMODEL_EYE))
+//@Begin NakCustom
+#define WRITES_CUSTOMDATA_TO_GBUFFER		(USES_GBUFFER && (MATERIAL_SHADINGMODEL_SUBSURFACE || MATERIAL_SHADINGMODEL_PREINTEGRATED_SKIN || MATERIAL_SHADINGMODEL_SUBSURFACE_PROFILE || MATERIAL_SHADINGMODEL_CLEAR_COAT || MATERIAL_SHADINGMODEL_TWOSIDED_FOLIAGE || MATERIAL_SHADINGMODEL_HAIR || MATERIAL_SHADINGMODEL_CLOTH || MATERIAL_SHADINGMODEL_EYE || MATERIAL_SHADINGMODEL_CUSTOM))
+//@End NakCustom
 
 // Based on GetPrecomputedShadowMasks()
 // Note: WRITES_PRECSHADOWFACTOR_TO_GBUFFER is currently disabled because we use the precomputed shadow factor GBuffer outside of STATICLIGHTING_TEXTUREMASK to store UseSingleSampleShadowFromStationaryLights

Engine/Shaders/Private/ShadingCommon.ush

@@ -30,7 +30,10 @@
 #define SHADINGMODELID_SINGLELAYERWATER		10
 #define SHADINGMODELID_THIN_TRANSLUCENT		11
 #define SHADINGMODELID_STRATA				12		// Temporary while we convert everything to Strata
-#define SHADINGMODELID_NUM					13
+//@Begin NakCustom
+#define SHADINGMODELID_CUSTOM				13
+//@End NakCustom
+#define SHADINGMODELID_NUM					14
 #define SHADINGMODELID_MASK					0xF		// 4 bits reserved for ShadingModelID			
 
 // The flags are defined so that 0 value has no effect!
@@ -71,6 +74,9 @@ float3 GetShadingModelColor(uint ShadingModelID)
 	else if (ShadingModelID == SHADINGMODELID_SINGLELAYERWATER) return float3(0.5f, 0.5f, 1.0f);
 	else if (ShadingModelID == SHADINGMODELID_THIN_TRANSLUCENT) return float3(1.0f, 0.8f, 0.3f);
 	else if (ShadingModelID == SHADINGMODELID_STRATA) return float3(1.0f, 1.0f, 0.0f);
+	//@Begin NakCustom
+	else if (ShadingModelID == SHADINGMODELID_CUSTOM) return float3(0.4f, 0.0f, 0.8f);
+	//@End NakCustom
 	else return float3(1.0f, 1.0f, 1.0f); // White
 #else
 	switch(ShadingModelID)
@@ -88,6 +94,9 @@ float3 GetShadingModelColor(uint ShadingModelID)
 		case SHADINGMODELID_SINGLELAYERWATER: return float3(0.5f, 0.5f, 1.0f);
 		case SHADINGMODELID_THIN_TRANSLUCENT: return float3(1.0f, 0.8f, 0.3f);
 		case SHADINGMODELID_STRATA: return float3(1.0f, 1.0f, 0.0f);
+		//@Begin NakCustom
+		case SHADINGMODELID_CUSTOM: return float3(0.4f, 0.0f, 0.8f);
+		//@End NakCustom
 		default: return float3(1.0f, 1.0f, 1.0f); // White
 	}
 #endif

Engine/Shaders/Private/ShadingModels.ush

@@ -340,6 +340,90 @@ FDirectLighting DefaultLitBxDF( FGBufferData GBuffer, half3 N, half3 V, half3 L,
 	return Lighting;
 }
 
+//@Begin NakCustom
+FDirectLighting CustomBxDF( FGBufferData GBuffer, half3 N, half3 V, half3 L, float Falloff, half NoL, FAreaLight AreaLight, FShadowTerms Shadow )
+{
+	BxDFContext Context;
+	FDirectLighting Lighting;
+
+#if SUPPORTS_ANISOTROPIC_MATERIALS
+	bool bHasAnisotropy = HasAnisotropy(GBuffer.SelectiveOutputMask);
+#else
+	bool bHasAnisotropy = false;
+#endif
+
+	float NoV, VoH, NoH;
+	BRANCH
+	if (bHasAnisotropy)
+	{
+		half3 X = GBuffer.WorldTangent;
+		half3 Y = normalize(cross(N, X));
+		Init(Context, N, X, Y, V, L);
+
+		NoV = Context.NoV;
+		VoH = Context.VoH;
+		NoH = Context.NoH;
+	}
+	else
+	{
+#if SHADING_PATH_MOBILE
+		InitMobile(Context, N, V, L, NoL);
+#else
+		Init(Context, N, V, L);
+#endif
+
+		NoV = Context.NoV;
+		VoH = Context.VoH;
+		NoH = Context.NoH;
+
+		SphereMaxNoH(Context, AreaLight.SphereSinAlpha, true);
+	}
+
+	Context.NoV = saturate(abs( Context.NoV ) + 1e-5);
+
+	NoL = Context.NoL;
+	NoL = floor(NoL + 1);
+
+#if MATERIAL_ROUGHDIFFUSE
+	// Chan diffuse model with roughness == specular roughness. This is not necessarily a good modelisation of reality because when the mean free path is super small, the diffuse can in fact looks rougher. But this is a start.
+	// Also we cannot use the morphed context maximising NoH as this is causing visual artefact when interpolating rough/smooth diffuse response. 
+	Lighting.Diffuse = Diffuse_Chan(GBuffer.DiffuseColor, Pow4(GBuffer.Roughness), NoV, NoL, VoH, NoH, GetAreaLightDiffuseMicroReflWeight(AreaLight));
+#else
+	Lighting.Diffuse = Diffuse_Lambert(GBuffer.DiffuseColor);
+#endif
+	Lighting.Diffuse *= AreaLight.FalloffColor * (Falloff * NoL);
+
+	BRANCH
+	if (bHasAnisotropy)
+	{
+		//Lighting.Specular = GBuffer.WorldTangent * .5f + .5f;
+		Lighting.Specular = AreaLight.FalloffColor * (Falloff * NoL) * SpecularGGX(GBuffer.Roughness, GBuffer.Anisotropy, GBuffer.SpecularColor, Context, NoL, AreaLight);
+	}
+	else
+	{
+		if( IsRectLight(AreaLight) )
+		{
+			Lighting.Specular = RectGGXApproxLTC(GBuffer.Roughness, GBuffer.SpecularColor, N, V, AreaLight.Rect, AreaLight.Texture);
+		}
+		else
+		{
+			Lighting.Specular = AreaLight.FalloffColor * (Falloff * NoL) * SpecularGGX(GBuffer.Roughness, GBuffer.SpecularColor, Context, NoL, AreaLight);
+		}
+	}
+	
+	FBxDFEnergyTerms EnergyTerms = ComputeGGXSpecEnergyTerms(GBuffer.Roughness, Context.NoV, GBuffer.SpecularColor);
+	
+	// Add energy presevation (i.e. attenuation of the specular layer onto the diffuse component
+	Lighting.Diffuse *= ComputeEnergyPreservation(EnergyTerms);
+	
+	// Add specular microfacet multiple scattering term (energy-conservation)
+	Lighting.Specular *= ComputeEnergyConservation(EnergyTerms);
+
+	Lighting.Transmission = 0;
+	return Lighting;
+}
+//@End NakCustom
+
 // Simple default lit model used by Lumen
 float3 SimpleShading( float3 DiffuseColor, float3 SpecularColor, float Roughness, float3 L, float3 V, half3 N )
 {
@@ -976,6 +1060,10 @@ FDirectLighting IntegrateBxDF( FGBufferData GBuffer, half3 N, half3 V, half3 L,
 			return ClothBxDF( GBuffer, N, V, L, Falloff, NoL, AreaLight, Shadow );
 		case SHADINGMODELID_EYE:
 			return EyeBxDF( GBuffer, N, V, L, Falloff, NoL, AreaLight, Shadow );
+		//@Begin NakCustom
+		case SHADINGMODELID_CUSTOM:
+			return CustomBxDF( GBuffer, N, V, L, Falloff, NoL, AreaLight, Shadow );
+		//@End NakCustom
 		default:
 			return (FDirectLighting)0;
 	}

Engine/Shaders/Private/ShadingModelsMaterial.ush

@@ -38,6 +38,14 @@ void SetGBufferForShadingModel(
 	if (false)
 	{
 	}
+	//@Begin NakCustom
+#if MATERIAL_SHADINGMODEL_CUSTOM
+	else if (ShadingModel == SHADINGMODELID_CUSTOM)
+	{
+		GBuffer.CustomData.xy = float2(GetMaterialCustomData0(MaterialParameters), GetMaterialCustomData1(MaterialParameters));
+	}
+#endif
+	//@End NakCustom
 #if MATERIAL_SHADINGMODEL_SUBSURFACE
 	else if (ShadingModel == SHADINGMODELID_SUBSURFACE)
 	{

Engine/Source/Runtime/Engine/Private/Materials/HLSLMaterialTranslator.cpp

@@ -1842,6 +1842,13 @@ void FHLSLMaterialTranslator::GetMaterialEnvironment(EShaderPlatform InPlatform,
 
 			bMaterialRequestsDualSourceBlending = true;
 		}
+		//@Begin NakCustom
+		if (ShadingModels.HasShadingModel(MSM_Custom))
+		{
+			OutEnvironment.SetDefine(TEXT("MATERIAL_SHADINGMODEL_CUSTOM"), TEXT("1"));
+			NumSetMaterials++;
+		}
+		//@End NakCustom
 
 		if (ShadingModels.HasShadingModel(MSM_SingleLayerWater) && FDataDrivenShaderPlatformInfo::GetRequiresDisableForwardLocalLights(Platform))
 		{

Engine/Source/Runtime/Engine/Private/Materials/MaterialHLSLEmitter.cpp

@@ -604,6 +604,13 @@ static void GetMaterialEnvironment(EShaderPlatform InPlatform,
 
 			bMaterialRequestsDualSourceBlending = true;
 		}
+		//@Begin NakCustom
+		if (ShadingModels.HasShadingModel(MSM_Custom))
+		{
+			OutEnvironment.SetDefine(TEXT("MATERIAL_SHADINGMODEL_CUSTOM"), TEXT("1"));
+			NumSetMaterials++;
+		}
+		//@End NakCustom
 
 		if (ShadingModels.HasShadingModel(MSM_SingleLayerWater) && FDataDrivenShaderPlatformInfo::GetRequiresDisableForwardLocalLights(InPlatform))
 		{

Engine/Source/Runtime/Engine/Classes/Engine/EngineTypes.h

@@ -610,6 +610,9 @@ enum EMaterialShadingModel
 	MSM_SingleLayerWater		UMETA(DisplayName="SingleLayerWater"),
 	MSM_ThinTranslucent			UMETA(DisplayName="Thin Translucent"),
 	MSM_Strata					UMETA(DisplayName="Strata", Hidden),
+	//@Begin NakCustom
+	MSM_Custom					UMETA(DisplayName="Custom"),
+	//@End NakCustom
 	/** Number of unique shading models. */
 	MSM_NUM						UMETA(Hidden),
 	/** Shading model will be determined by the Material Expression Graph,

Engine/Source/Runtime/Engine/Private/ShaderCompiler/ShaderGenerationUtil.cpp

@@ -155,6 +155,9 @@ void FShaderCompileUtilities::ApplyFetchEnvironment(FShaderMaterialPropertyDefin
 	FETCH_COMPILE_BOOL(MATERIAL_SHADINGMODEL_EYE);
 	FETCH_COMPILE_BOOL(MATERIAL_SHADINGMODEL_SINGLELAYERWATER);
 	FETCH_COMPILE_BOOL(MATERIAL_SHADINGMODEL_THIN_TRANSLUCENT);
+	//@Begin NakCustom
+	FETCH_COMPILE_BOOL(MATERIAL_SHADINGMODEL_CUSTOM);
+	//@End NakCustom
 
 	FETCH_COMPILE_BOOL(SINGLE_LAYER_WATER_DF_SHADOW_ENABLED);
 
@@ -1586,6 +1589,11 @@ static void SetSlotsForShadingModelType(bool Slots[], EMaterialShadingModel Shad
 	case MSM_DefaultLit:
 		SetSharedGBufferSlots(Slots);
 		break;
+	//@Begin NakCustom
+	case MSM_Custom:
+		SetSharedGBufferSlots(Slots);
+		break;
+	//@End NakCustom
 	case MSM_Subsurface:
 		SetSharedGBufferSlots(Slots);
 		if (bMergeCustom)
@@ -1744,6 +1752,13 @@ static void DetermineUsedMaterialSlots(
 	{
 		SetStandardGBufferSlots(Slots, bWriteEmissive, bHasTangent, bHasVelocity, bHasStaticLighting, bIsStrataMaterial);
 	}
+	
+	//@Begin NakCustom
+	if (Mat.MATERIAL_SHADINGMODEL_CUSTOM)
+	{
+		SetStandardGBufferSlots(Slots, bWriteEmissive, bHasTangent, bHasVelocity, bHasStaticLighting, bIsStrataMaterial);
+	}
+	//@End NakCustom
 
 	if (Mat.MATERIAL_SHADINGMODEL_SUBSURFACE)
 	{

Engine/Source/Runtime/Engine/Private/Materials/MaterialShader.cpp

@@ -88,6 +88,9 @@ FString GetShadingModelString(EMaterialShadingModel ShadingModel)
 		case MSM_Eye:				ShadingModelName = TEXT("MSM_Eye"); break;
 		case MSM_SingleLayerWater:	ShadingModelName = TEXT("MSM_SingleLayerWater"); break;
 		case MSM_ThinTranslucent:	ShadingModelName = TEXT("MSM_ThinTranslucent"); break;
+		//@Begin NakCustom
+		case MSM_Custom:			ShadingModelName = TEXT("MSM_Custom"); break;
+		//@End NakCustom
 		default: ShadingModelName = TEXT("Unknown"); break;
 	}
 	return ShadingModelName;
@@ -246,6 +249,12 @@ void UpdateMaterialShaderCompilingStats(const FMaterial* Material)
 	{
 		INC_DWORD_STAT_BY(STAT_ShaderCompiling_NumLitMaterialShaders, 1);
 	}
+	//@Begin NakCustom
+	else if (ShadingModels.HasAnyShadingModel({ MSM_Custom }))
+	{
+		INC_DWORD_STAT_BY(STAT_ShaderCompiling_NumLitMaterialShaders, 1);
+	}
+	//@End NakCustom
 
 
 	if (Material->IsSpecialEngineMaterial())

Engine/Source/Runtime/RenderCore/Public/ShaderMaterial.h

@@ -103,6 +103,9 @@ struct FShaderMaterialPropertyDefines
 	uint8 MATERIAL_SHADINGMODEL_SINGLELAYERWATER : 1;
 	uint8 SINGLE_LAYER_WATER_DF_SHADOW_ENABLED : 1;
 	uint8 MATERIAL_SHADINGMODEL_THIN_TRANSLUCENT : 1;
+	//@Begin NakCustom
+	uint8 MATERIAL_SHADINGMODEL_CUSTOM : 1;
+	//@End NakCustom
 
 	uint8 TRANSLUCENCY_LIGHTING_VOLUMETRIC_NONDIRECTIONAL : 1;
 	uint8 TRANSLUCENCY_LIGHTING_VOLUMETRIC_DIRECTIONAL : 1;