Simple Toon Shading

From GameStudio Wiki

the entity should use a texture with solid colors only. no gradients, photorealistic stuff and so on should be used but maybe this would lead to some nice effects. :)

normalize+6.tga is this normalization cube map:

toonlookup.tga is a 1d look-up table and should be similar to this example:

you can experiment with this one to achieve different shading styles.

the outline thickness can be set in the entity action, the outline color can be set as pixelshader constant. the entity will get lit from the sun direction. DirectX 9 version :

bmap bmp_normalizationcube=<normalize+6.tga>;
bmap bmp_toonlookup=<toonlookup.tga>; 

function mtl_toon_init()
{
	bmap_to_cubemap(mtl.skin1);
}

material mtl_toon
{
	skin1=bmp_normalizationcube;
	skin2=bmp_toonlookup;
	event=mtl_toon_init;
	effect
	"
	texture entSkin1;
	texture mtlSkin1;
	texture mtlSkin2;
	matrix matWorldViewProj;
	matrix matWorld;
	vector vecSkill41;
	vector vecSunDir; 

	technique toon
	{
		//-------------------------------paint
		pass p0
		{
			texture[0]=<mtlSkin1>;
			texture[1]=<mtlSkin2>;
			texture[2]=<entSkin1>; 

			addressU[1]=clamp; 

			magFilter[2]=linear;
			minFilter[2]=linear;
			mipFilter[2]=linear;

			vertexShaderConstant[0]=<matWorldViewProj>;
			vertexShaderConstant[4]=<matWorld>; 

			vertexShaderConstant[16]=<vecSunDir>; 

			vertexShader=
			asm
			{
				vs.1.1
 		                dcl_position v0
				dcl_normal v3
				dcl_texcoord0 v7
				dcl_texcoord1 v8

				m4x4 oPos,v0,c0	  // transform position to clip space
				m3x3 r0,v3,c4	  // transform normal to world space
				dp3 r0.w,r0,r0	  // renormalize it
				rsq r0.w,r0.w
				mul r0,r0,r0.w
				mov oT0.xyz,r0.xyz  // output normal to oT0
				mov oT1.xyz,-c16    // output light direction to oT1
				mov oT2.xy,v7.xy    // output uvs to oT2
			};
			pixelShader=
			asm
			{
				ps.1.3
				tex t0		    // fetch normalized normal
				texdp3tex t1,t0_bx2 // light.normal=u -> use u to lookup in t1 
                                                   // _bx2 -> 0..1 of normcube will be   -1..1 
	 			tex t2		    // sample color map
				mul r0,t2,t1	    // modulate with shading
			};
		}
		//--------------------------------------------ink
		pass p1
		{
			cullMode=cw;
			vertexShaderConstant[0]=<matWorldViewProj>;
			vertexShaderConstant[16]=<vecSkill41>; // outline_thickness, 0, 0, 0
			vertexShader=
			asm
			{
				vs.1.1
				dcl_position v0
				dcl_normal v3
				dcl_texcoord0 v7
				dcl_texcoord1 v8

				mov r0,v0
				mul r1,c16.x,v3		  // scale normal
				add r0.xyz,r0.xyz,r1.xyz  // shell offset 
                                                         //(vertex position + scaled normal)
				m4x4 oPos,r0,c0		  // transform position to clip space
			};
			pixelShader=
			asm
			{
				ps.1.3
				def c0,0,0,0,1		  // outline rgba
 				mov r0,c0
 			};
 		}
 	}
	";
}

action toon
{
	my.skill41=float(0.8); // outline_thickness
	my.material=mtl_toon;
}

DirectX 8 version :

bmap bmp_normalizationcube=<normalize+6.tga>;
bmap bmp_toonlookup=<toonlookup.tga>; 

function mtl_toon_init()
{
	bmap_to_cubemap(mtl.skin1);
}

material mtl_toon
{
	skin1=bmp_normalizationcube;
	skin2=bmp_toonlookup;
	event=mtl_toon_init;
	effect
	"
	texture entSkin1;
	texture mtlSkin1;
	texture mtlSkin2;
	matrix matWorldViewProj;
	matrix matWorld;
	vector vecSkill41;
	vector vecSunDir; 

	technique toon
	{
		//---------------------------------------------------------------------------paint
		pass p0
		{
			texture[0]=<mtlSkin1>;
			texture[1]=<mtlSkin2>;
			texture[2]=<entSkin1>; 

			addressU[1]=clamp; 

			// magFilter[1]=point;
			// minFilter[1]=point;
			// mipFilter[1]=point;
			magFilter[2]=linear;
			minFilter[2]=linear;
			mipFilter[2]=linear;

			vertexShaderConstant[0]=<matWorldViewProj>;
			vertexShaderConstant[4]=<matWorld>; 

			vertexShaderConstant[16]=<vecSunDir>; 

			vertexShader=
			decl
			{
				stream 0;
				float v0[3]; // position
				float v3[3]; // normal
				float v7[3]; // uv
			}
			asm
			{
				vs.1.1
				m4x4 oPos,v0,c0		// transform position to clip space
				m3x3 r0,v3,c4		// transform normal to world space
				dp3 r0.w,r0,r0		// renormalize it
				rsq r0.w,r0.w
				mul r0,r0,r0.w
				mov oT0.xyz,r0.xyz	// output normal to oT0
				mov oT1.xyz,-c16		// output light direction to oT1
				mov oT2.xy,v7.xy	// output uvs to oT2
			};
			pixelShader=
			asm
			{
				ps.1.3
				tex t0			// fetch normalized normal
				texdp3tex t1,t0_bx2	// light.normal=u -> use u to lookup in t1 // _bx2 -> 0..1 of normcube will be   -1..1 
	 			tex t2			// sample color map
				mul r0,t2,t1		// modulate with shading
			};
		}
		//---------------------------------------------------------------------------ink
		pass p1
		{
			cullMode=cw;
			vertexShaderConstant[0]=<matWorldViewProj>;
			vertexShaderConstant[16]=<vecSkill41>; // outline_thickness, 0, 0, 0
			vertexShader=
			decl
			{
				stream 0;
				float v0[3];
				float v3[3];
				float v7[3];
			}
			asm
			{
				vs.1.1
				mov r0,v0
				mul r1,c16.x,v3			// scale normal
				add r0.xyz,r0.xyz,r1.xyz	// shell offset (vertex position + scaled normal)
				m4x4 oPos,r0,c0			// transform position to clip space
			};
			pixelShader=
			asm
			{
				ps.1.3
				def c0,0,0,0,1			// outline rgba
 				mov r0,c0
 			};
 		}
 	}
	";
}

action toon
{
	my.skill41=float(0.8); // outline_thickness
	my.material=mtl_toon;
}