another small step:

Code:
//----------------------------------------------------------------------------- ent_getmatrix_rb
// return transformation matrix of entity (without scale -> for rigid bodies)
//-----------------------------------------------------------------------------
void ent_getmatrix_rb(ENTITY *entity, D3DXMATRIX *mout)
{
// ignores entity scale -> not needed for rigid bodies -> collision shape size gets defined by bounding box

D3DXMATRIX mpan;
D3DXMatrixRotationZ(&mpan, entity->pan * DEG2RAD);

D3DXMATRIX mtilt;
D3DXMatrixRotationY(&mtilt, -entity->tilt * DEG2RAD); // -tilt?

D3DXMATRIX mroll;
D3DXMatrixRotationX(&mroll, entity->roll * DEG2RAD);

D3DXMATRIX mtranslation;
D3DXMatrixTranslation(&mtranslation,
entity->x * QUANTTOMETER,
entity->y * QUANTTOMETER,
entity->z * QUANTTOMETER);

D3DXMATRIX mtemp1;
D3DXMATRIX mtemp2;

D3DXMatrixMultiply(&mtemp1, &mroll, &mtilt);
D3DXMatrixMultiply(&mtemp2, &mtemp1, &mpan);
D3DXMatrixMultiply(mout, &mtemp2, &mtranslation);
}

//----------------------------------------------------------------------------- ent_getmatrix_rb
// return transformation matrix of entity (without scale -> for rigid bodies)
// (same as above but done in a different way)
//-----------------------------------------------------------------------------
//void ent_getmatrix_rb(ENTITY *entity, float *mout)
//{
// VECTOR xaxis; vec_set(&xaxis, vector(1, 0, 0));
// VECTOR yaxis; vec_set(&yaxis, vector(0, 1, 0));
// VECTOR zaxis; vec_set(&zaxis, vector(0, 0, 1));
//
// // ignores entity scale -> not needed for rigid bodies -> collision shape size gets defined by bounding box
//
// vec_rotate(&xaxis, &entity->pan);
// vec_rotate(&yaxis, &entity->pan);
// vec_rotate(&zaxis, &entity->pan);
//
// mout[0] = xaxis.x; mout[1] = xaxis.y; mout[2] = xaxis.z; mout[3] = 0;
// mout[4] = yaxis.x; mout[5] = yaxis.y; mout[6] = yaxis.z; mout[7] = 0;
// mout[8] = zaxis.x; mout[9] = zaxis.y; mout[10] = zaxis.z; mout[11] = 0;
// mout[12] = entity->x * QUANTTOMETER;
// mout[13] = entity->y * QUANTTOMETER;
// mout[14] = entity->z * QUANTTOMETER;
// mout[15] = 1;
//}

//----------------------------------------------------------------------------- ent_setmatrix_rb
//
// i think a 3dgs rotation matrix gets constructed like that:
//
// pan matrix (rotation about z-axis)
// | cos(p) -sin(p) 0 |
// | sin(p) cos(p) 0 |
// | 0 0 1 |
//
// tilt matrix (rotation about y-axis)
// | cos(t) 0 sin(t) |
// | 0 1 0 |
// | -sin(t) 0 cos(t) |
//
// roll matrix (rotation about x-axis)
// | 1 0 0 |
// | 0 cos(r) -sin(r) |
// | 0 sin(r) cos(r) |
//
// roll * tilt * pan =
//
// | cos(t)*cos(p) cos(t)*-sin(p) sin(t) |
// | -sin(r)*-sin(t)*cos(p) + cos(r)*sin(p) -sin(r)*-sin(t)*-sin(p) + cos(r)*cos(p) -sin(r)*cos(t) |
// | cos(r)*-sin(t)*cos(p) + sin(r)*sin(p) cos(r)*-sin(t)*-sin(p) + sin(r)*cos(p) cos(r)*cos(t) |
//
// in case of gimbal lock (cos(t) = 0, sin(r) = 0, cos(r) = 1):
//
// | 0 0 sin(t) |
// | sin(p) cos(p) 0 |
// | -sin(t)*cos(p) -sin(t)*-sin(p) 0 |

void ent_setmatrix_rb(ENTITY *entity, float *m)
{
float pan, tilt, roll;

if(fabs(m[2]) > 0.9999) // looking straight up or down -> gimbal lock
{
draw_text("gimbal lock", 10, 10, vector(255,255,255));
pan = atan2f(-m[4], m[5]); // -m[4]?
tilt = 1.570796 * sign(m[2]);
roll = 0;
}
else
{
pan = atan2f(m[1], m[0]); // swap?
tilt = asinf(m[2]);
roll = atan2f(m[6], m[10]);
}

entity->pan = pan * RAD2DEG;
entity->tilt = tilt * RAD2DEG;
entity->roll = roll * RAD2DEG;

entity->x = m[12] * METERTOQUANT;
entity->y = m[13] * METERTOQUANT;
entity->z = m[14] * METERTOQUANT;
}



the conversion from 3dgs eulers to matrices (needed by newton) and back again. previously i had some problems with this which caused wrong orientation and jitters occasionally. now it seems to work flawlessly.

there isn't much missing now for posting the first simple working example.