@Roel
I think you are right, the fsaa works when toon ppe is not active & it would be logical to assume that it also works when toon ppe is active. My programming skills are not really that impressive so I wouldn't know how to tweak the shader in order to fix the edges, but I tried to add blur (both classic & Gaussian) & that definitely didn't produce desired visual effect.

..but the toon shader from GS folder gets anti-aliased (edges are nice & smooth), so I think this is the card to play at the moment. As I mentioned, it seems to be even better solution since outlines don't change during camera movement:)

@Hummel
Yes, I use two rooms, one in wmb & one in mdl format. The wmb room has one texture & shadowmap and the mdl room has transparent texture & toon shader applied.
I modified the fx file as you suggested (I think I did:) so this is what I have now:

Click to reveal..
#include <transform>
#include <fog>
#include <pos>
#include <normal>
#include <tangent>
#include <light>

float4 vecLight;

float4 vecSkill41;
//vecSkill41.x: float facAmbient = 1.0;
//vecSkill41.y: float facEdgeWidth = 1.0;

texture entSkin1; // texture
sampler sBaseTex = sampler_state { Texture = <entSkin1>; MipFilter = Linear; };

struct toonOut
{
float4 Pos: POSITION;
float Fog: FOG;
float4 Color: COLOR;
float4 Ambient:COLOR1;
float2 Tex: TEXCOORD0;
float3 Normal: TEXCOORD1;
float3 Light: TEXCOORD2;
};

toonOut toonVS(
in float4 inPos: POSITION,
in float3 inNormal: NORMAL,
in float2 inTex: TEXCOORD0,
in float3 inTangent: TEXCOORD2)
{
toonOut Out;
Out.Pos = DoTransform(inPos);
Out.Tex = inTex;
Out.Fog = DoFog(inPos);

float facAmbient = vecSkill41.x / 100.;
Out.Ambient = facAmbient*vecLight;

CreateTangents(inNormal,inTangent);
float3 P = DoPos(inPos);

float3 lightVec;
lightVec = normalize(vecLightPos[0].xyz - P);
Out.Color = vecLightColor[0];

Out.Light = DoTangent(lightVec) * 0.5 + 0.5;
Out.Normal = DoTangent(DoPos(inNormal)) * 0.5 + 0.5;

return Out;
}

float4 toonPS(toonOut In): COLOR
{
float4 base = tex2D(sBaseTex,In.Tex);

float intensity = dot(In.Light*2 - 1,In.Normal*2 - 1);

float light;
if (intensity > 0.85 )
light = 1.;
else if (intensity > 0.35 )
light = 0.55;
else
light = 0.05;

return base * (light*In.Color + In.Ambient);
}

technique toon
{
//pass one
//{
// VertexShader = compile vs_1_1 toonVS();
// PixelShader = compile ps_1_4 toonPS();
//}

pass two
{
CULLMODE=CW;
vertexShaderConstant[0] = <matWorldViewProj>;
vertexShaderConstant[16] = <vecSkill41.y/20.>;

VertexShader = asm
{
vs_1_0
dcl_position v0
dcl_normal v3
dcl_texcoord v7
mov r0,v0
mul r1,c16.x,v3
// Scale the normal
add r0.xyz,r0.xyz,r1.xyz
m4x4 oPos,r0,c0
// Transorm position to clip space
mov oD0, c0
mov r0,c0
};
}
}

//technique fallback { pass one { } }

..is there anything else that I can exclude from the fx without damaging it's functionality?


>>Demos free3DModels Tutorials<<
>>>>>>> by Pavle Nikolic <<<<<<<