Occluder classification from camera region.cpp

// This assumes the occluder AABB and camera position are in the same space.
// You can either then transform the planes into world space, or transform the
// box verts and camera pos into world space before building the planes, but
// after finding the silhouette.
int makeOccluderPlanes(Vec4 planes[9], Vec3 bmin, Vec3 bmax, Vec3 camPos)
{
   static const uint8_t s_silhouettes[43][6] = {
      { },                       //  0
      { 0, 4, 6, 2 },            //  1: -x
      { 1, 3, 7, 5 },            //  2: +x
      { },                       //  3
      { 0, 1, 5, 4 },            //  4: -y
      { 0, 1, 5, 4, 6, 2 },      //  5: -y-x
      { 0, 1, 3, 7, 5, 4 },      //  6: -y+x
      { },                       //  7
      { 2, 6, 7, 3 },            //  8: +y
      { 4, 6, 7, 3, 2, 0 },      //  9: +y-x
      { 2, 6, 7, 5, 1, 3 },      // 10: +y+x
      { },                       // 11
      { },                       // 12
      { },                       // 13
      { },                       // 14
      { },                       // 15
      { 0, 2, 3, 1 },            // 16: -z
      { 6, 2, 3, 1, 0, 4 },      // 17: -z-x
      { 0, 2, 3, 7, 5, 1 },      // 18: -z+x
      { },                       // 19
      { 0, 2, 3, 1, 5, 4 },      // 20: -z-y
      { 6, 2, 3, 1, 5, 4 },      // 21: -z-y-x
      { 0, 2, 3, 7, 5, 4 },      // 22: -z-y+x
      { },                       // 23
      { 0, 2, 6, 7, 3, 1 },      // 24: -z+y
      { 0, 4, 6, 7, 3, 1 },      // 25: -z+y-x
      { 0, 2, 6, 7, 5, 1 },      // 26: -z+y+x
      { },                       // 27
      { },                       // 28
      { },                       // 29
      { },                       // 30
      { },                       // 31
      { 4, 5, 7, 6 },            // 32: +z
      { 4, 5, 7, 6, 2, 0 },      // 33: +z-x
      { 1, 3, 7, 6, 4, 5 },      // 34: +z+x
      { },                       // 35
      { 0, 1, 5, 7, 6, 4 },      // 36: +z-y
      { 0, 1, 5, 7, 6, 2 },      // 37: +z-y-x
      { 0, 1, 3, 7, 6, 4 },      // 38: +z-y+x
      { },                       // 39
      { 6, 4, 5, 7, 3, 2 },      // 40: +z+y
      { 0, 4, 5, 7, 3, 2 },      // 41: +z+y-x
      { 6, 4, 5, 1, 3, 2 },      // 42: +z+y+x
   };

   int faces = 0;
   
   if (camPos.x < bmin.x) {
      faces |= (1 << 0);
   } else if (camPos.x > bmax.x) {
      faces |= (1 << 1);
   }

   if (camPos.y < bmin.y) {
      faces |= (1 << 2);
   } else if (camPos.y > bmax.y) {
      faces |= (1 << 3);
   }

   if (camPos.z < bmin.z) {
      faces |= (1 << 4);
   } else if (camPos.z > bmax.z) {
      faces |= (1 << 5);
   }
   
   if (faces == 0) {
      // camera inside occluder
      return 0;
   }

   Vec3 verts[8];
   verts[0] = { bmin.x, bmin.y, bmin.z };
   verts[1] = { bmax.x, bmin.y, bmin.z };
   verts[2] = { bmin.x, bmax.y, bmin.z };
   verts[3] = { bmax.x, bmax.y, bmin.z };
   verts[4] = { bmin.x, bmin.y, bmax.z };
   verts[5] = { bmax.x, bmin.y, bmax.z };
   verts[6] = { bmin.x, bmax.y, bmax.z };
   verts[7] = { bmax.x, bmax.y, bmax.z };

   int nplanes = 0;

   // build edge planes
   const uint8_t *edges = s_silhouettes[faces];
   if (faces & (faces - 1)) {
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[0]], verts[edges[1]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[1]], verts[edges[2]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[2]], verts[edges[3]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[3]], verts[edges[4]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[4]], verts[edges[5]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[5]], verts[edges[0]]);
   } else {
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[0]], verts[edges[1]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[1]], verts[edges[2]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[2]], verts[edges[3]]);
      planeFromPoints(&planes[nplanes++], camPos, verts[edges[3]], verts[edges[0]]);
   }

   // build face planes
   if (faces & 0x01) {
      planeFromPoints(&planes[nplanes++], verts[4], verts[2], verts[0]);
   } else if (faces & 0x02) {
      planeFromPoints(&planes[nplanes++], verts[1], verts[3], verts[5]);
   }

   if (faces & 0x04) {
      planeFromPoints(&planes[nplanes++], verts[0], verts[1], verts[4]);
   } else if (faces & 0x08) {
      planeFromPoints(&planes[nplanes++], verts[6], verts[3], verts[2]);
   }

   if (faces & 0x10) {
      planeFromPoints(&planes[nplanes++], verts[1], verts[2], verts[3]);
   } else if (faces & 0x20) {
      planeFromPoints(&planes[nplanes++], verts[4], verts[5], verts[6]);
   }

   return nplanes;
}