Toon Multitexturing

From GameStudio Wiki

This multitexturing toon shader works with DirectX9. Your entities skins should be made of solid colors(no gradients or photorealistic stuff) to get the cartoonish look.

You will need one picture file:

The four textures that are mixed are entSkin1-4. Also you need a blendmap which is mtlSkin1. Use the r/g/b channels to blend. The shadowmap uses the alpha channel

In order to add a shadow map to the shader, code has been inserted in green. If you do not wish to use a shadow map, please remove the code as it will speed up render times without the additional shader code.

The shadowmap will use the alpha layer of the blend map (mtlSkin1). So any area that is black will be shadow.

In the fx file there's a line "dword mtlSkill1 = 16;". Change the value to adjust tiling.

example material:

bmap blend_map = <blend_texture.tga>; // use the channels of this bitmap for blending
bmap lookup_map = <toonlookup.tga>; // used for toon shading

material mat_multitoon {
  skin1 = blend_map;  // assign the blend texture to mtlSkin1
  skin2 = lookup_map;
  effect = "the_fx_file.fx";
}

fx file:

texture entSkin1;
texture entSkin2;
texture entSkin3;
texture entSkin4;

texture mtlSkin1;
texture mtlSkin2;

matrix matWorldViewProj;
matrix matWorld;
vector vecSunDir;
vector vecFog;
vector vecViewPos;

dword mtlskill1 = 16;

sampler multiTex1 = sampler_state
{
    Texture = <entSkin1>;
    MinFilter = Linear;
    MagFilter = Linear;
    MipFilter = Linear;   
    AddressU  = Wrap;
    AddressV  = Wrap;
};

sampler multiTex2 = sampler_state
{
    Texture = <entSkin2>;
    MinFilter = Linear;
    MagFilter = Linear;
    MipFilter = Linear;   
    AddressU  = Wrap;
    AddressV  = Wrap;
};

sampler multiTex3 = sampler_state
{
    Texture = <entSkin3>;
    MinFilter = Linear;
    MagFilter = Linear;
    MipFilter = Linear;   
    AddressU  = Wrap;
    AddressV  = Wrap;
};

sampler multiTex4 = sampler_state
{
    Texture = <entSkin4>;
    MinFilter = Linear;
    MagFilter = Linear;
    MipFilter = Linear;   
    AddressU  = Wrap;
    AddressV  = Wrap;
};

sampler multiRes1 = sampler_state
{
    Texture = <mtlSkin1>;
    MinFilter = Linear;
    MagFilter = Linear;
    MipFilter = Linear;   
    AddressU  = Mirror;
    AddressV  = Mirror;
};

sampler toon_lookup = sampler_state
{
    Texture = <mtlSkin2>;
    MinFilter = Linear;
    MagFilter = Linear;
    MipFilter = Linear;    

    // clamp is important!
    AddressU  = Clamp;
    AddressV  = Clamp;
};

void mtoonVS( 
	in float4 inPosition : POSITION0,
	in float4 inNormal : NORMAL0, 
	in float4 inTexcoord : TEXCOORD0,

	out float4 outPosition : POSITION0,
	out float4 outNormal : TEXCOORD0,
	out float4 outLight : TEXCOORD1,
	out float4 outTexcoord : TEXCOORD2,
	out float outFog : FOG0)
{
	outPosition = mul(inPosition, matWorldViewProj); 
	outNormal = mul(inNormal, matWorld);
	outLight = -vecSunDir;
	outTexcoord = inTexcoord;
	outFog = 1 - (distance(inPosition, vecViewPos) - vecFog.x) * vecFog.z;
}

void mtoonPS(
     in float4 inNormal : TEXCOORD0, 
     in float4 inLight : TEXCOORD1, 
     in float4 inTexcoord : TEXCOORD2, 
     out float4 outColor : COLOR0)
{
	float4 Color2;
	float lerpFactor;
	inTexcoord *= mtlskill1;
	outColor = tex2D(multiTex1, inTexcoord.xy);
	Color2 = tex2D(multiTex2, inTexcoord.xy);
	lerpFactor = tex2D(multiRes1, inTexcoord.xy/mtlskill1).r;
	outColor.rgb = lerp(outColor.rgb,Color2.rgb, lerpFactor);
	Color2 = tex2D(multiTex3, inTexcoord.xy);
	lerpFactor = tex2D(multiRes1, inTexcoord.xy/mtlskill1).g;
	outColor.rgb = lerp(outColor.rgb,Color2.rgb, lerpFactor);
	Color2 = tex2D(multiTex4, inTexcoord.xy);
	lerpFactor = tex2D(multiRes1, inTexcoord.xy/mtlskill1).b;
	outColor.rgb = lerp(outColor.rgb,Color2.rgb, lerpFactor);
 
 
 	// toon shade
 	float4 ToonShade = tex2D(toon_lookup, dot(inLight.xyz, inNormal.xyz));
        lerpFactor = tex2D(multiRes1, inTexcoord.xy/mtlskill1).a;  
	  outColor.rgb *= ToonShade.rgb  * lerpFactor  ;  
 }

technique toon
{
 	pass p0
 	{
 		VertexShader = compile vs_1_1 mtoonVS();
		PixelShader = compile ps_2_0 mtoonPS();

	}
}