Remove artifacts from mediapipe rendering program - image

Question

I am trying to get a simple face mask, but I cannot pin point the cause and get rid of tiny artifacts in the textured output
Example image has a few artifacts marked:
Code:
import cv2
import mediapipe as mp
import triangulation_media_pipe as tmp
import numpy as np
mp_drawing = mp.solutions.drawing_utils
mp_face_mesh = mp.solutions.face_mesh
face = "face_textures/yash.jpg"
def load_base_img(face_mesh, image_file_name, ):
image = cv2.imread(image_file_name)
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
return {"img": image, "landmarks": results}
def transform_landmarks_from_tf_to_ocv(keypoints, face_width, face_height):
landmark_list = []
if (keypoints.multi_face_landmarks != None):
for face_landmarks in keypoints.multi_face_landmarks:
for l in face_landmarks.landmark:
pt = mp_drawing._normalized_to_pixel_coordinates(l.x, l.y,
face_width, face_height)
landmark_list.append(pt)
return landmark_list
def main():
# For webcam input:
face_mesh = mp_face_mesh.FaceMesh()
base_face_handler, landmark_base_ocv, base_input_image = process_base_face_mesh(face_mesh, face)
cap = cv2.VideoCapture(2)
while cap.isOpened():
_, webcam_img = cap.read()
image_rows, image_cols, _ = webcam_img.shape
results = face_mesh.process(webcam_img)
landmark_target_ocv = transform_landmarks_from_tf_to_ocv(results, image_cols, image_rows)
# Draw the face mesh annotations on the image.
image = webcam_img.copy()
img2_new_face = np.zeros_like(image)
if results.multi_face_landmarks:
if True:
for i in range(0, int(len(tmp.TRIANGULATION) / 3)):
triangle_index = [tmp.TRIANGULATION[i * 3],
tmp.TRIANGULATION[i * 3 + 1],
tmp.TRIANGULATION[i * 3 + 2]]
tbas1 = landmark_base_ocv[triangle_index[0]]
tbas2 = landmark_base_ocv[triangle_index[1]]
tbas3 = landmark_base_ocv[triangle_index[2]]
triangle1 = np.array([tbas1, tbas2, tbas3])
rect1 = cv2.boundingRect(triangle1)
(x1, y1, w1, h1) = rect1
cropped_triangle = base_input_image[y1: y1 + h1, x1: x1 + w1]
cropped_tr1_mask = np.zeros((h1, w1), np.uint8)
points = np.array([[tbas1[0] - x1, tbas1[1] - y1],
[tbas2[0] - x1, tbas2[1] - y1],
[tbas3[0] - x1, tbas3[1] - y1]])
cv2.fillConvexPoly(cropped_tr1_mask, points, 255)
ttar1 = landmark_target_ocv[triangle_index[0]]
ttar2 = landmark_target_ocv[triangle_index[1]]
ttar3 = landmark_target_ocv[triangle_index[2]]
triangle2 = np.array([ttar1, ttar2, ttar3])
rect2 = cv2.boundingRect(triangle2)
(x2, y2, w2, h2) = rect2
cropped_tr2_mask = np.zeros((h2, w2), np.uint8)
points2 = np.array([[ttar1[0] - x2, ttar1[1] - y2],
[ttar2[0] - x2, ttar2[1] - y2],
[ttar3[0] - x2, ttar3[1] - y2]])
cv2.fillConvexPoly(cropped_tr2_mask, points2, 255)
# Warp triangles
points = np.float32(points)
points2 = np.float32(points2)
M = cv2.getAffineTransform(points, points2)
warped_triangle = cv2.warpAffine(cropped_triangle, M, (w2, h2))
warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle, mask=cropped_tr2_mask)
# Reconstructing destination face
img2_new_face_rect_area = img2_new_face[y2: y2 + h2, x2: x2 + w2]
img2_new_face_rect_area_gray = cv2.cvtColor(img2_new_face_rect_area, cv2.COLOR_BGR2GRAY)
_, mask_triangles_designed = cv2.threshold(img2_new_face_rect_area_gray, 0, 255,
cv2.THRESH_BINARY_INV)
warped_triangle = cv2.bitwise_and(warped_triangle, warped_triangle,
mask=mask_triangles_designed)
img2_new_face_rect_area = cv2.add(img2_new_face_rect_area, warped_triangle)
img2_new_face[y2: y2 + h2, x2: x2 + w2] = img2_new_face_rect_area
cv2.imshow('mask', img2_new_face)
key = cv2.waitKey(5)
face_mesh.close()
cap.release()
def process_base_face_mesh(face_mesh,
image_file):
base_face_handler = load_base_img(face_mesh, image_file)
base_input_image = base_face_handler["img"].copy()
image_rows, image_cols, _ = base_face_handler["img"].shape
landmark_base_ocv = transform_landmarks_from_tf_to_ocv(base_face_handler["landmarks"], image_cols, image_rows)
return base_face_handler, landmark_base_ocv, base_input_image
if __name__ == "__main__":
main()
triangulation_media_pipe.py
The code is a slightly modified version of this
A possible solution I added was to get rid of the artifacts is to just mask those areas with a blurred version of the same image, but I want the pixel values to be as close as possible to the real image values.