Toonshading with Outline

From GameStudio Wiki

Thanks to ello for sharing this shader!

Lite-C:

---

Material and function to adjust the effect:

//Toon Shading with Outline
MATERIAL* Toon_mat =
{
	effect = "Toon_.fx";
}

function Toon_set_Value(r,g,b,threshold)
{
	Toon_mat.skill1 = floatv(r);	//4 looks okay
	Toon_mat.skill2 = floatv(g);	//same as above
	Toon_mat.skill3 = floatv(b);	//same as above
	Toon_mat.skill4 = floatv(threshold);	//0.01
}

The Toon_.fx file:

////Makes Details invisible and the colors a bit more intensive,
////which gives the Level a toon like look this is combined with a sobel edge filter
////which causes some kind of outlining

float4 vecSkill1;	//"Color Details" in xyz and threshold in w
float4 vecViewPort;

Texture TargetMap;
sampler2D smpSource = sampler_state { texture = <TargetMap>; };

float4 dirtyToonPS( float2 Tex : TEXCOORD0 ) : COLOR0 
{		
	half4 color = tex2D(smpSource,Tex.xy);
	color.r = round(color.r*vecSkill1.r)/vecSkill1.r;
	color.g = round(color.g*vecSkill1.g)/vecSkill1.g;
	color.b = round(color.b*vecSkill1.b)/vecSkill1.b;
	
	const float threshold = vecSkill1.w;

	const int NUM = 9;
	const float2 c[NUM] =
	{
		float2(-0.0078125, 0.0078125), 
		float2( 0.00 ,     0.0078125),
		float2( 0.0078125, 0.0078125),
		float2(-0.0078125, 0.00 ),
		float2( 0.0,       0.0),
		float2( 0.0078125, 0.007 ),
		float2(-0.0078125,-0.0078125),
		float2( 0.00 ,    -0.0078125),
		float2( 0.0078125,-0.0078125),
	};	

	int i;
	float3 col[NUM];
	for (i=0; i < NUM; i++)
	{
		col[i] = tex2D(smpSource, Tex.xy + 0.2*c[i]);
	}
	
	float3 rgb2lum = float3(0.30, 0.59, 0.11);
	float lum[NUM];
	for (i = 0; i < NUM; i++)
	{
		lum[i] = dot(col[i].xyz, rgb2lum);
	}
	float x = lum[2]+  lum[8]+2*lum[5]-lum[0]-2*lum[3]-lum[6];
	float y = lum[6]+2*lum[7]+  lum[8]-lum[0]-2*lum[1]-lum[2];
	float edge =(x*x + y*y < threshold)? 1.0:0.0;
	
	color.rgb *= edge;
	return color;
}

technique postFX 
{ 
	pass p1 
	{ 
		PixelShader = compile ps_2_0 dirtyToonPS(); 
	} 
}

C-Script

---

Material and function to adjust the effect:

//Toon Shading with Outline
material Toon_mat
{
	effect = "Toon_.fx";
}

function Toon_set_Value(r,g,b,threshold)
{
	Toon_mat.skill1 = floatv(r);	//4 looks okay
	Toon_mat.skill2 = floatv(g);	//same as above
	Toon_mat.skill3 = floatv(b);	//same as above
	Toon_mat.skill4 = floatv(threshold);	//0.01
}

The Toon_.fx file:

////Makes Details invisible and the colors a bit more intensive,
////which gives the Level a toon like look this is combined with a sobel edge filter
////which causes some kind of outlining

float4 vecSkill1;	//"Color Details" in xyz and threshold in w
float4 vecViewPort;

Texture entSkin1;
sampler2D smpSource = sampler_state { texture = <entSkin1>; };

float4 dirtyToonPS( float2 Tex : TEXCOORD0 ) : COLOR0 
{		
	half4 color = tex2D(smpSource,Tex.xy);
	color.r = round(color.r*vecSkill1.r)/vecSkill1.r;
	color.g = round(color.g*vecSkill1.g)/vecSkill1.g;
	color.b = round(color.b*vecSkill1.b)/vecSkill1.b;
	
	const float threshold = vecSkill1.w;

	const int NUM = 9;
	const float2 c[NUM] =
	{
		float2(-0.0078125, 0.0078125), 
		float2( 0.00 ,     0.0078125),
		float2( 0.0078125, 0.0078125),
		float2(-0.0078125, 0.00 ),
		float2( 0.0,       0.0),
		float2( 0.0078125, 0.007 ),
		float2(-0.0078125,-0.0078125),
		float2( 0.00 ,    -0.0078125),
		float2( 0.0078125,-0.0078125),
	};	

	int i;
	float3 col[NUM];
	for (i=0; i < NUM; i++)
	{
		col[i] = tex2D(smpSource, Tex.xy + 0.2*c[i]);
	}
	
	float3 rgb2lum = float3(0.30, 0.59, 0.11);
	float lum[NUM];
	for (i = 0; i < NUM; i++)
	{
		lum[i] = dot(col[i].xyz, rgb2lum);
	}
	float x = lum[2]+  lum[8]+2*lum[5]-lum[0]-2*lum[3]-lum[6];
	float y = lum[6]+2*lum[7]+  lum[8]-lum[0]-2*lum[1]-lum[2];
	float edge =(x*x + y*y < threshold)? 1.0:0.0;
	
	color.rgb *= edge;
	return color;
}

technique postFX 
{ 
	pass p1 
	{ 
		PixelShader = compile ps_2_0 dirtyToonPS(); 
	} 
}

--Slin 15:54, 17 November 2007 (CET)