Iris detection and color change

irisColor.py

import cv2
import dlib
import faceBlendCommon as face
import numpy as np

# Load Image
im = cv2.imread("cv2/images/girl.jpg")

# Detect face landmarks
PREDICTOR_PATH = r"C:\Users\felipe.cunha\Documents\venv\cv2\week1-pyton\data\models\shape_predictor_68_face_landmarks.dat"
faceDetector = dlib.get_frontal_face_detector()
landmarkDetector = dlib.shape_predictor(PREDICTOR_PATH)
landmarks = face.getLandmarks(faceDetector, landmarkDetector, im)

def createEyeMask(eyeLandmarks, im):
    leftEyePoints = eyeLandmarks
    eyeMask = np.zeros_like(im)
    cv2.fillConvexPoly(eyeMask, np.int32(leftEyePoints), (255, 255, 255))
    eyeMask = np.uint8(eyeMask)
    return eyeMask

def findIris(eyeMask, im, thresh):
    r = im[:,:,2]
    _, binaryIm = cv2.threshold(r, thresh, 255, cv2.THRESH_BINARY_INV)
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (4,4))
    morph = cv2.dilate(binaryIm, kernel, 1)
    morph = cv2.merge((morph, morph, morph))
    morph = morph.astype(float)/255
    eyeMask = eyeMask.astype(float)/255
    iris = cv2.multiply(eyeMask, morph)
    return iris

def findCentroid(iris):
    M = cv2.moments(iris[:,:,0])
    cX = int(M["m10"] / M["m00"])
    cY = int(M["m01"] / M["m00"])
    centroid = (cX,cY)
    return centroid

def createIrisMask(iris, centroid):
    cnts, _ = cv2.findContours(np.uint8(iris[:,:,0]), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    flag = 10000
    final_cnt = None
    for cnt in cnts:
        (x,y),radius = cv2.minEnclosingCircle(cnt)
        distance = abs(centroid[0]-x)+abs(centroid[1]-y)
        if distance < flag :
            flag = distance
            final_cnt = cnt
        else:
            continue
    (x,y),radius = cv2.minEnclosingCircle(final_cnt)
    center = (int(x),int(y))
    radius = int(radius) - 2

    irisMask = np.zeros_like(iris)
    inverseIrisMask = np.ones_like(iris)*255
    cv2.circle(irisMask,center,radius,(255, 255, 255),-1)
    cv2.circle(inverseIrisMask,center,radius,(0, 0, 0),-1)
    irisMask = cv2.GaussianBlur(irisMask, (5,5), cv2.BORDER_DEFAULT)
    inverseIrisMask = cv2.GaussianBlur(inverseIrisMask, (5,5), cv2.BORDER_DEFAULT)

    return irisMask, inverseIrisMask

def changeEyeColor(im, irisMask, inverseIrisMask):
    imCopy = cv2.applyColorMap(im, cv2.COLORMAP_TWILIGHT_SHIFTED)
    imCopy = imCopy.astype(float)/255
    irisMask = irisMask.astype(float)/255
    inverseIrisMask = inverseIrisMask.astype(float)/255
    im = im.astype(float)/255
    faceWithoutEye = cv2.multiply(inverseIrisMask, im)
    newIris = cv2.multiply(irisMask, imCopy)
    result = faceWithoutEye + newIris
    return result

def float642Uint8(im):
    im2Convert = im.astype(np.float64) / np.amax(im)
    im2Convert = 255 * im2Convert 
    convertedIm = im2Convert.astype(np.uint8)
    return convertedIm

# Create eye mask using eye landmarks from facial landmark detection
leftEyeMask = createEyeMask(landmarks[36:42], im)
rightEyeMask = createEyeMask(landmarks[43:49], im)

# Find the iris by thresholding the red channel of the image within the boundaries of the eye mask
leftIris = findIris(leftEyeMask, im, 40)
rightIris = findIris(rightEyeMask, im, 50)

# Find the centroid of the binary image of the eye
leftIrisCentroid = findCentroid(leftIris)
rightIrisCentroid = findCentroid(rightIris)

# Generate the iris mask and its inverse mask
leftIrisMask, leftInverseIrisMask = createIrisMask(leftIris, leftIrisCentroid)
rightIrisMask, rightInverseIrisMask = createIrisMask(rightIris, rightIrisCentroid)

# Change the eye color and merge it to the original image
coloredEyesLady = changeEyeColor(im, rightIrisMask, rightInverseIrisMask)
coloredEyesLady = float642Uint8(coloredEyesLady)
coloredEyesLady = changeEyeColor(coloredEyesLady, leftIrisMask, leftInverseIrisMask)

# Present results
cv2.imshow("", coloredEyesLady)
cv2.waitKey(0)

hey felipe, can i know what is faceBlendcommon in line 3, i cant find any references for this library

faceBlendCommon is a file
import cv2
import dlib
import numpy as np
import math

Returns 8 points on the boundary of a rectangle

def getEightBoundaryPoints(h, w):
boundaryPts = []
boundaryPts.append((0,0))
boundaryPts.append((w/2, 0))
boundaryPts.append((w-1,0))
boundaryPts.append((w-1, h/2))
boundaryPts.append((w-1, h-1))
boundaryPts.append((w/2, h-1))
boundaryPts.append((0, h-1))
boundaryPts.append((0, h/2))
return np.array(boundaryPts, dtype=np.float)

Constrains points to be inside boundary

def constrainPoint(p, w, h):
p = (min(max(p[0], 0), w - 1), min(max(p[1], 0), h - 1))
return p

convert Dlib shape detector object to list of tuples

def dlibLandmarksToPoints(shape):
points = []
for p in shape.parts():
pt = (p.x, p.y)
points.append(pt)
return points

Compute similarity transform given two sets of two points.

OpenCV requires 3 pairs of corresponding points.

We are faking the third one.

def similarityTransform(inPoints, outPoints):
s60 = math.sin(60math.pi/180)
c60 = math.cos(60math.pi/180)

inPts = np.copy(inPoints).tolist()
outPts = np.copy(outPoints).tolist()

The third point is calculated so that the three points make an equilateral triangle

xin = c60*(inPts[0][0] - inPts[1][0]) - s60*(inPts[0][1] - inPts[1][1]) + inPts[1][0]
yin = s60*(inPts[0][0] - inPts[1][0]) + c60*(inPts[0][1] - inPts[1][1]) + inPts[1][1]

inPts.append([np.int(xin), np.int(yin)])

xout = c60*(outPts[0][0] - outPts[1][0]) - s60*(outPts[0][1] - outPts[1][1]) + outPts[1][0]
yout = s60*(outPts[0][0] - outPts[1][0]) + c60*(outPts[0][1] - outPts[1][1]) + outPts[1][1]

outPts.append([np.int(xout), np.int(yout)])

Now we can use estimateRigidTransform for calculating the similarity transform.

tform = cv2.estimateRigidTransform(np.array([inPts]), np.array([outPts]), False)
return tform

Normalizes a facial image to a standard size given by outSize.

Normalization is done based on Dlib's landmark points passed as pointsIn

After normalization, left corner of the left eye is at (0.3 * w, h/3 )

and right corner of the right eye is at ( 0.7 * w, h / 3) where w and h

are the width and height of outSize.

def normalizeImagesAndLandmarks(outSize, imIn, pointsIn):
h, w = outSize

Corners of the eye in input image

eyecornerSrc = [pointsIn[36], pointsIn[45]]

Corners of the eye in normalized image

eyecornerDst = [(np.int(0.3 * w), np.int(h/3)),
(np.int(0.7 * w), np.int(h/3))]

Calculate similarity transform

tform = similarityTransform(eyecornerSrc, eyecornerDst)
imOut = np.zeros(imIn.shape, dtype=imIn.dtype)

Apply similarity transform to input image

imOut = cv2.warpAffine(imIn, tform, (w, h))

reshape pointsIn from numLandmarks x 2 to numLandmarks x 1 x 2

points2 = np.reshape(pointsIn, (pointsIn.shape[0], 1, pointsIn.shape[1]))

Apply similarity transform to landmarks

pointsOut = cv2.transform(points2, tform)

reshape pointsOut to numLandmarks x 2

pointsOut = np.reshape(pointsOut, (pointsIn.shape[0], pointsIn.shape[1]))

return imOut, pointsOut

find the point closest to an array of points

pointsArray is a Nx2 and point is 1x2 ndarray

def findIndex(pointsArray, point):
dist = np.linalg.norm(pointsArray-point, axis=1)
minIndex = np.argmin(dist)
return minIndex

Check if a point is inside a rectangle

def rectContains(rect, point):
if point[0] < rect[0]:
return False
elif point[1] < rect[1]:
return False
elif point[0] > rect[2]:
return False
elif point[1] > rect[3]:
return False
return True

Calculate Delaunay triangles for set of points

Returns the vector of indices of 3 points for each triangle

def calculateDelaunayTriangles(rect, points):

Create an instance of Subdiv2D

subdiv = cv2.Subdiv2D(rect)

Insert points into subdiv

for p in points:
subdiv.insert((p[0], p[1]))

Get Delaunay triangulation

triangleList = subdiv.getTriangleList()

Find the indices of triangles in the points array

delaunayTri = []

for t in triangleList:
# The triangle returned by getTriangleList is
# a list of 6 coordinates of the 3 points in
# x1, y1, x2, y2, x3, y3 format.
# Store triangle as a list of three points
pt = []
pt.append((t[0], t[1]))
pt.append((t[2], t[3]))
pt.append((t[4], t[5]))

pt1 = (t[0], t[1])
pt2 = (t[2], t[3])
pt3 = (t[4], t[5])

if rectContains(rect, pt1) and rectContains(rect, pt2) and rectContains(rect, pt3):
  # Variable to store a triangle as indices from list of points
  ind = []
  # Find the index of each vertex in the points list
  for j in range(0, 3):
    for k in range(0, len(points)):
      if(abs(pt[j][0] - points[k][0]) < 1.0 and abs(pt[j][1] - points[k][1]) < 1.0):
        ind.append(k)
    # Store triangulation as a list of indices
  if len(ind) == 3:
    delaunayTri.append((ind[0], ind[1], ind[2]))

return delaunayTri

Apply affine transform calculated using srcTri and dstTri to src and

output an image of size.

def applyAffineTransform(src, srcTri, dstTri, size):

Given a pair of triangles, find the affine transform.

warpMat = cv2.getAffineTransform(np.float32(srcTri), np.float32(dstTri))

Apply the Affine Transform just found to the src image

dst = cv2.warpAffine(src, warpMat, (size[0], size[1]), None,
flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT_101)

return dst

Warps and alpha blends triangular regions from img1 and img2 to img

def warpTriangle(img1, img2, t1, t2):

Find bounding rectangle for each triangle

r1 = cv2.boundingRect(np.float32([t1]))
r2 = cv2.boundingRect(np.float32([t2]))

Offset points by left top corner of the respective rectangles

t1Rect = []
t2Rect = []
t2RectInt = []

for i in range(0, 3):
t1Rect.append(((t1[i][0] - r1[0]), (t1[i][1] - r1[1])))
t2Rect.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1])))
t2RectInt.append(((t2[i][0] - r2[0]), (t2[i][1] - r2[1])))

Get mask by filling triangle

mask = np.zeros((r2[3], r2[2], 3), dtype=np.float32)
cv2.fillConvexPoly(mask, np.int32(t2RectInt), (1.0, 1.0, 1.0), 16, 0)

Apply warpImage to small rectangular patches

img1Rect = img1[r1[1]:r1[1] + r1[3], r1[0]:r1[0] + r1[2]]

size = (r2[2], r2[3])

img2Rect = applyAffineTransform(img1Rect, t1Rect, t2Rect, size)

img2Rect = img2Rect * mask

Copy triangular region of the rectangular patch to the output image

img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] * ((1.0, 1.0, 1.0) - mask)
img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] = img2[r2[1]:r2[1]+r2[3], r2[0]:r2[0]+r2[2]] + img2Rect

detect facial landmarks in image

def getLandmarks(faceDetector, landmarkDetector, im, FACE_DOWNSAMPLE_RATIO = 1):
points = []
imSmall = cv2.resize(im,None,
fx=1.0/FACE_DOWNSAMPLE_RATIO,
fy=1.0/FACE_DOWNSAMPLE_RATIO,
interpolation = cv2.INTER_LINEAR)

faceRects = faceDetector(imSmall, 0)

if len(faceRects) > 0:
maxArea = 0
maxRect = None
# TODO: test on images with multiple faces
for face in faceRects:
if face.area() > maxArea:
maxArea = face.area()
maxRect = [face.left(),
face.top(),
face.right(),
face.bottom()
]

rect = dlib.rectangle(*maxRect)
scaledRect = dlib.rectangle(int(rect.left()*FACE_DOWNSAMPLE_RATIO),
                         int(rect.top()*FACE_DOWNSAMPLE_RATIO),
                         int(rect.right()*FACE_DOWNSAMPLE_RATIO),
                         int(rect.bottom()*FACE_DOWNSAMPLE_RATIO))

landmarks = landmarkDetector(im, scaledRect)
points = dlibLandmarksToPoints(landmarks)

return points

Warps an image in a piecewise affine manner.

The warp is defined by the movement of landmark points specified by pointsIn

to a new location specified by pointsOut. The triangulation beween points is specified

by their indices in delaunayTri.

def warpImage(imIn, pointsIn, pointsOut, delaunayTri):
h, w, ch = imIn.shape

Output image

imOut = np.zeros(imIn.shape, dtype=imIn.dtype)

Warp each input triangle to output triangle.

The triangulation is specified by delaunayTri

for j in range(0, len(delaunayTri)):
# Input and output points corresponding to jth triangle
tin = []
tout = []

for k in range(0, 3):
  # Extract a vertex of input triangle
  pIn = pointsIn[delaunayTri[j][k]]
  # Make sure the vertex is inside the image.
  pIn = constrainPoint(pIn, w, h)

  # Extract a vertex of the output triangle
  pOut = pointsOut[delaunayTri[j][k]]
  # Make sure the vertex is inside the image.
  pOut = constrainPoint(pOut, w, h)

  # Push the input vertex into input triangle
  tin.append(pIn)
  # Push the output vertex into output triangle
  tout.append(pOut)

# Warp pixels inside input triangle to output triangle.
warpTriangle(imIn, imOut, tin, tout)

return imOut