Artifical Lag Inducer Effect

PicDelVid.py

# Title = Picture-Delta-Video
# Author =  Pierce Brooks
# Usage = `python3 ./PicDelVid.py ./MyMP4.mp4`
# Reference = https://towardsdatascience.com/head-pose-estimation-using-python-d165d3541600

import cv2
import sys
import os
import numpy as np
import mediapipe as mp
from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates

def dist(left, right):
  d = 0.0
  for i in range(len(left)):
    d += (left[i]-right[i])**2.0
  return d**0.5

def diff(prev, next, threshold, basis):
  if not len(prev) == len(next):
    return False
  for i in range(len(prev)):
    if not len(prev[i]) == len(next[i]):
      return False
    for j in range(len(prev[i])):
      if (dist(prev[i][j], next[i][j]) > basis/threshold):
        return True
  return False

def diff_angles(prev, next, threshold, basis):
  if not len(prev) == len(next):
    return False
  for i in range(len(prev)):
    if (dist(prev[i], next[i]) > basis/threshold):
      return True
  return False

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_face_mesh = mp.solutions.face_mesh

# For webcam input:
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
capture = 0
if len(sys.argv) > 1:
  capture = sys.argv[1]
capture = cv2.VideoCapture(capture)
prev = [[], []]
last = -1
counter = -1
fps = 30
width = 1920
height = 1080
frame = None
path = os.path.join(os.getcwd(), os.path.basename(sys.argv[0]))+'.mp4'
output = cv2.VideoWriter(path, cv2.VideoWriter_fourcc(*'MPEG'), fps, (width, height))
with mp_face_mesh.FaceMesh(
    max_num_faces=2,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5) as face_mesh:
  while capture.isOpened():
    if cv2.waitKey(5) & 0xFF == 27:
      break
    success, image = capture.read()
    if not success:
      continue
    image = cv2.resize(image, (width, height))
    image.flags.writeable = False
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    results = face_mesh.process(image)

    image.flags.writeable = True
    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
    image_rows, image_cols, _ = image.shape
    next = []
    faces_3d = []
    faces_2d = []
    faces = []
    if results.multi_face_landmarks:
      if not len(results.multi_face_landmarks) == len(prev[0]):
        prev = None
      for face_landmarks in results.multi_face_landmarks:
        face = []
        face_3d = []
        face_2d = []
        for index, face_landmark in enumerate(face_landmarks.landmark):
          if index == 33 or index == 263 or index == 1 or index == 61 or index == 291 or index == 199:
            x, y = int(face_landmark.x * image_cols), int(face_landmark.y * image_rows)
            face_2d.append([x, y])
            face_3d.append([x, y, face_landmark.z])
          landmark = _normalized_to_pixel_coordinates(face_landmark.x, face_landmark.y, image_cols, image_rows)
          if (landmark == None):
            continue
          face.append(landmark)
        faces_3d.append(face_3d)
        faces_2d.append(face_2d)
        next.append(face)
        """
        mp_drawing.draw_landmarks(
            image=image,
            landmark_list=face_landmarks,
            connections=mp_face_mesh.FACEMESH_TESSELATION,
            landmark_drawing_spec=None,
            connection_drawing_spec=mp_drawing_styles
            .get_default_face_mesh_tesselation_style())
        mp_drawing.draw_landmarks(
            image=image,
            landmark_list=face_landmarks,
            connections=mp_face_mesh.FACEMESH_CONTOURS,
            landmark_drawing_spec=None,
            connection_drawing_spec=mp_drawing_styles
            .get_default_face_mesh_contours_style())
        mp_drawing.draw_landmarks(
            image=image,
            landmark_list=face_landmarks,
            connections=mp_face_mesh.FACEMESH_IRISES,
            landmark_drawing_spec=None,
            connection_drawing_spec=mp_drawing_styles
            .get_default_face_mesh_iris_connections_style())
        """
      for i in range(len(faces_3d)):
        face_2d = np.array(faces_2d[i], dtype=np.float64)
        face_3d = np.array(faces_3d[i], dtype=np.float64)
        focal_length = 1*image_cols
        cam_matrix = np.array([ [focal_length, 0, image_rows/2],
                                [0, focal_length, image_cols/2],
                                [0, 0, 1] ])
        dist_matrix = np.zeros((4, 1), dtype=np.float64)
        success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix, dist_matrix)
        rmat, jac = cv2.Rodrigues(rot_vec)
        angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)
        x = angles[0]*360
        y = angles[1]*360
        z = angles[2]*360
        faces.append([x, y, z])
      next = [next, faces]
    else:
      prev = None
    if len(next) < 2:
      next = [next, []]
    counter += 1
    threshold = max(1.0, float(counter-last)**0.5)
    if prev == None or diff(prev[0], next[0], threshold, 35.0)^diff_angles(prev[1], next[1], threshold, 35.0):
      cv2.imshow('MediaPipe Face Mesh', cv2.flip(image, 1))
      prev = next
      last = counter
      frame = image.copy()
    output.write(frame)
capture.release()
output.release()
cv2.destroyAllWindows()