matrix.c

OOKVec3 matrix_unproject(GLuint fbo,OOKVec3 camera,int x,int y,Z_DEPTH z)
{
    int viewport[4];
    float winZ;
    
    switch(z)
    {
        case Z_NEAR:
            winZ=0.0f;
            break;
        case Z_FAR:
            winZ=1.0f;
        case Z_DBUF:
            glGetIntegerv(GL_VIEWPORT,viewport);
            y=viewport[3]-y;
            glBindFramebuffer(GL_FRAMEBUFFER,fbo);
            glReadPixels(x,y,1,1,GL_DEPTH_COMPONENT,GL_FLOAT,&winZ);
            glBindFramebuffer(GL_FRAMEBUFFER,0);
            break;
    }
    
    glm::vec3 screen=glm::vec3(x,y,winZ);
    glm::mat4 view=glm::mat4();
    glm::mat4 projection=glm::perspective(20.0f,(float)viewport[2]/(float)viewport[3],500.0f,20000.0f);
    
    glm::vec3 pos=glm::unProject(screen,view,projection,glm::vec4(0,0,viewport[2],viewport[3]));
    
    return {pos.x,pos.y,pos.z};
}

xy_intercept.c

OOKVec3 p1=matrix_unproject(ook->fbo,ook->camera_pos,ook->cursor_x,ook->cursor_y,Z_NEAR);
OOKVec3 p2=matrix_unproject(ook->fbo,ook->camera_pos,ook->cursor_x,ook->cursor_y,Z_FAR);
OOKVec3 v={p2.x-p1.x,p2.y-p1.y,p2.z-p1.z};

float t=-p1.z/v.z;
float x=p1.x+v.x*t;
float y=p1.y+v.y*t;

//Whoa huge numbers like this:
//-5612056576.000000 108973.578125 0.025641
printf("%f %f %f\n",x,y,t);