Shader hints

From GameStudio Wiki

Tips & Tricks for HLSL shaders

The engine passes non-transposed matrices to shaders (Microsoft Style). Some other shader tools, like Rendermonkey, use transposed matrices (ATI style). For transforming a vector with a nontransposed matrix, put the vector at the first place in the multiplication, and the matrix at the second. ATI does this the opposite way, so keep this in mind when converting Rendermonkey shaders to HLSL. For the examples below, version 6.31.4 is required:

// transform the vector position to screen coordinates
	Out.Pos = mul(inPos,matWorldViewProj);

For applying Sunlight, use the following function:

	float4 vecSunDir;
	float4 vecSunDiffuse = float4(200.f/255.f, 200.f/255.f, 200.f/255.f, 1.f);
	...
	float4 DoSunLight(float3 N)
	{
// modulate the sunlight by the surface angle
		return vecSunDiffuse * dot(N,-vecSunDir);
	}
	...

For applying dynamic lights, use the following function:

	float4 vecLightPos[8];
	float4 vecLightColor[8];
	float3 vecFalloff = float3(0.f, 0.f, 2.f);
	...
	float4 DoPointLight(float3 P, float3 N, int i)
	{
// calculate the light ray pointing from the light to the surface
		float3 D = (float3)vecLightPos[i]-P;
// calculate the angle between surface and light ray
		float NdotL = dot(N,normalize(D));
// modulate the light by the surface angle
		float4 Color = vecLightColor[i] * NdotL; 
// calculate the light attenuation factor, DX uses a really strange formula here
		float fac = 0.f;
		if (NdotL >= 0.f && vecLightPos[i].w > 0.f)
		{
// get the distance factor
			float LD = length(D)/vecLightPos[i].w;
#ifdef DXLIGHTING
			if (LD < 1.3f)
				fac = 1.f/(vecFalloff.x + vecFalloff.y*LD + vecFalloff.z*LD*LD);
#else  // linear Lighting
			if (LD < 1.f) 
				fac = 1.f - LD;
#endif
		}
		return Color * fac;
	}

For applying fog, use the following function:

	float4 vecFog;
	...
	float DoFog(float3 Pos)
	{
// convert the vector position to view space to get it's depth (.z)
		float3 P = mul(Pos,matWorldView);
// apply the linear fog formula
   		return saturate((vecFog.y-P.z) * vecFog.z);
	}

The light and fog functions are then called from the vertex shader:

struct VS_OUT // Output to the pixelshader fragment { float4 Pos : POSITION; float4 Color: COLOR0; float Fog: FOG; ... };

	VS_OUT Test_VS(
		...
		float4 inPos : POSITION,
		float3 inNormal : NORMAL)
	{
		VS_OUT Out;

// transform the vector position to screen coordinates
		Out.Pos = mul(inPos,matWorldViewProj); 

// transform the normal and the position
		float3 N = normalize(mul(inNormal,matWorld));
		float3 P = mul(inPos,matWorld);
// Add ambient and sun light
		Out.Color = vecSkill41.w + DoSunLight(N);
// Add 6 dynamic lights (maximum for vs 1.1)
		for (int i=0; i<6; i++)
			Out.Color += DoPointLight(P,N,i);
// Add fog
		Out.Fog = DoFog(inPos);

// convert texture coordinates or do other stuff
		... 

		return Out;
	}