10. Benchmarking Landmark models using data#
written by Tiankang Xie
In the tutorial we will demonstrate how to evaluate pyfeat landmark detection algorithms with evaluation data
import glob
from PIL import Image, ImageDraw
import numpy as np
import matplotlib.pyplot as plt
import scipy.io as sio
import string
from tqdm import tqdm
import pandas as pd
from feat import Detector
import pickle
import os
from feat.data import (
Fex,
ImageDataset,
VideoDataset,
_inverse_face_transform,
_inverse_landmark_transform,
)
from torch.utils.data import DataLoader
We provide the following code for evaluating the normalized mean squared error for landmark detection algorithms. These codes have been slightly modified from: https://github.com/D-X-Y/landmark-detection
def get_kpts(file_path, num_kpts = 68):
"""
Function to read the ground truth landmark labels in 300W
"""
kpts = []
f = open(file_path, 'r')
ln = f.readline()
while not ln.startswith('n_points'):
ln = f.readline()
num_pts = ln.split(':')[1]
num_pts = num_pts.strip('\n').strip(' ')
# checking for the number of keypoints
if float(num_pts) != num_kpts:
print ("encountered file with less than keypoints")
return None
# skipping the line with '{'
ln = f.readline()
ln = f.readline()
while not ln.startswith('}'):
vals = ln.split(' ')[:2]
vals = [v.strip('\n') for v in vals]
vals = [np.float32(v) for v in vals]
kpts.append(vals)
ln = f.readline()
return kpts
def calculate_rmse(result_dir, data_dir):
"""
Function to calculate MSE between predicted and groundtruth land labels
"""
with open(result_dir, 'rb') as fp:
lands, all_img_dir = pickle.load(fp)
paths_errors = []
for i, land_paths in enumerate(all_img_dir):
land = lands[i]
condition = os.path.basename(land_paths).split('_')[0]
if condition == 'indoor':
kpts = get_kpts(data_dir+'01_Indoor/'+os.path.basename(land_paths).replace('png','pts'))
GT_points = np.asarray(kpts)
elif condition == 'outdoor':
kpts = get_kpts(data_dir+'02_Outdoor/'+os.path.basename(land_paths).replace('png','pts'))
GT_points = np.asarray(kpts)
else:
raise ValueError('weird happened')
interocular_distance = np.linalg.norm(GT_points[36,:]-GT_points[45,:], ord=2)
ans_diff = []
for llnd in land[0]:
summ = np.linalg.norm(GT_points - llnd, ord=2, axis=0)
ans_diff.append(summ/(68*interocular_distance)) # normalize with interocular distance
# ans_diff.append(np.sqrt(np.mean(np.square(GT_points - llnd))))
paths_errors.append(np.min(ans_diff)*10)
return np.mean(paths_errors)*100
Provide the path for
data and labels. Which can be found at https://ibug.doc.ic.ac.uk/resources/300-W/
where to save results
data_dir = '/Storage/Data/300W/'
save_result_dir = '/Storage/Projects/pyfeat_testing/Data_Eshin/land_test/'
all_img_dir = glob.glob(data_dir + '01_Indoor/*.png') + glob.glob(data_dir + '02_Outdoor/*.png')
Test of MobileNet#
%%capture
chosen_model = 'mobilenet'
detector = Detector(face_model='retinaface',emotion_model='resmasknet', landmark_model=chosen_model, au_model='xgb', device='cpu')
counter = 0
lands = []
for fp in tqdm(all_img_dir):
data_loader = DataLoader(
ImageDataset(
fp,
output_size=None,
preserve_aspect_ratio=True,
padding=True,
),
num_workers=1,
batch_size=1,
pin_memory=False,
shuffle=False,
)
batch_output = []
for batch_id, batch_data in enumerate(tqdm(data_loader)):
faces = detector.detect_faces(batch_data["Image"])
landmarks = detector.detect_landmarks(batch_data["Image"], detected_faces=faces)
lands.append(landmarks)
# Save Result
with open(save_result_dir+f'{chosen_model}_bench_results.pkl', 'wb') as fp:
pickle.dump((lands, all_img_dir), fp)
mobilenet_normal = calculate_rmse(result_dir=save_result_dir+'mobilenet_bench_results.pkl', data_dir=data_dir)
Normalized mean squared error for the algorithm is
print(mobilenet_normal)
5.769516086484791
Test of MobileFaceNet#
%%capture
chosen_model = 'mobilefacenet'
detector = Detector(face_model='retinaface',emotion_model='resmasknet', landmark_model=chosen_model, au_model='xgb', device='cpu')
counter = 0
lands = []
for fp in tqdm(all_img_dir):
data_loader = DataLoader(
ImageDataset(
fp,
output_size=None,
preserve_aspect_ratio=True,
padding=True,
),
num_workers=1,
batch_size=1,
pin_memory=False,
shuffle=False,
)
batch_output = []
for batch_id, batch_data in enumerate(tqdm(data_loader)):
faces = detector.detect_faces(batch_data["Image"])
landmarks = detector.detect_landmarks(batch_data["Image"], detected_faces=faces)
lands.append(landmarks)
# Save Result
with open(save_result_dir+f'{chosen_model}_bench_results.pkl', 'wb') as fp:
pickle.dump((lands, all_img_dir), fp)
mobilefacenet_normal = calculate_rmse(result_dir=save_result_dir+f'{chosen_model}_bench_results.pkl', data_dir=data_dir)
Normalized mean squared error for the algorithm is
print(mobilefacenet_normal)
4.988652327582802
Test of PFLD#
%%capture
chosen_model = 'pfld'
detector = Detector(face_model='retinaface',emotion_model='resmasknet', landmark_model=chosen_model, au_model='xgb', device='cpu')
counter = 0
lands = []
for fp in tqdm(all_img_dir):
data_loader = DataLoader(
ImageDataset(
fp,
output_size=None,
preserve_aspect_ratio=True,
padding=True,
),
num_workers=1,
batch_size=1,
pin_memory=False,
shuffle=False,
)
batch_output = []
for batch_id, batch_data in enumerate(tqdm(data_loader)):
faces = detector.detect_faces(batch_data["Image"])
landmarks = detector.detect_landmarks(batch_data["Image"], detected_faces=faces)
lands.append(landmarks)
# Save Result
with open(save_result_dir+f'{chosen_model}_bench_results.pkl', 'wb') as fp:
pickle.dump((lands, all_img_dir), fp)
pfld_normal = calculate_rmse(result_dir=save_result_dir+'pfld_bench_results.pkl', data_dir=data_dir)
Normalized mean squared error for the algorithm is
print(pfld_normal)
5.390958738782998