This is really ugly, since my matrix class is kind of messed up, but hope it helps.
Here's my 2D coordinate test method (tests a 2d coordinate against a polygon).
The matrix passed to it is the full transform matrix of the polygon's object multiplied by the inverse of the current camera matrix.
bool OpenGLRenderer::test2DCoordinate(float x, float y, Tau::Polygon *poly, const Matrix4 &matrix) {
GLdouble nearPlane[3],farPlane[3];
GLdouble mv[16];
Matrix4 camInverse = cameraMatrix.inverse();
Matrix4 cmv;
cmv.identity();
cmv = cmv * camInverse;
for(int i=0; i < 16; i++) {
mv[i] = cmv.ml[i];
}
GLint vp[4];
glGetIntegerv( GL_VIEWPORT, vp );
gluUnProject(x, yRes - y, 0.0, mv, sceneProjectionMatrix, vp, &nearPlane[0], &nearPlane[1], &nearPlane[2]);
gluUnProject(x, yRes - y, 1.0, mv, sceneProjectionMatrix, vp, &farPlane[0], &farPlane[1], &farPlane[2]);
Vector3 nearVec(nearPlane[0], nearPlane[1], nearPlane[2]);
Vector3 farVec(farPlane[0], farPlane[1], farPlane[2]);
Vector3 dirVec = farVec - nearVec;
dirVec.Normalize();
Vector3 hitPoint;
Matrix4 fullMatrix = matrix;
if(poly->getVertexCount() == 3) {
return rayTriangleIntersect(Vector3(0,0,0), dirVec, fullMatrix * (*poly->getVertex(0)), fullMatrix * (*poly->getVertex(1)), fullMatrix * (*poly->getVertex(2)), &hitPoint);
} else if(poly->getVertexCount() == 4) {
return (rayTriangleIntersect(Vector3(0,0,0), dirVec, fullMatrix * (*poly->getVertex(2)), fullMatrix * (*poly->getVertex(1)), fullMatrix * (*poly->getVertex(0)), &hitPoint) ||
rayTriangleIntersect(Vector3(0,0,0), dirVec, fullMatrix * (*poly->getVertex(0)), fullMatrix * (*poly->getVertex(3)), fullMatrix * (*poly->getVertex(2)), &hitPoint));
} else {
return false;
}
}
Here's the ray triangle intersect method for good measure:
bool Renderer::rayTriangleIntersect(Vector3 ray_origin, Vector3 ray_direction, Vector3 vert0, Vector3 vert1, Vector3 vert2, Vector3 *hitPoint)
{
float t,u,v;
t = 0; u = 0; v = 0;
Vector3 edge1 = vert1 - vert0;
Vector3 edge2 = vert2 - vert0;
Vector3 tvec, pvec, qvec;
float det, inv_det;
pvec = ray_direction.crossProduct(edge2);
det = edge1.dot(pvec);
if (det > -0.00001f)
return false;
inv_det = 1.0f / det;
tvec = ray_origin - vert0;
u = tvec.dot(pvec) * inv_det;
if (u < -0.001f || u > 1.001f)
return false;
qvec = tvec.crossProduct(edge1);
v = ray_direction.dot(qvec) * inv_det;
if (v < -0.001f || u + v > 1.001f)
return false;
t = edge2.dot(qvec) * inv_det;
if (t <= 0)
return false;
hitPoint->x = ray_origin.x+t*ray_direction.x;
hitPoint->y = ray_origin.y+t*ray_direction.y;
hitPoint->z = ray_origin.z+t*ray_direction.z;
return true;
}