vato007/picar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

So many changes to hand recogniser i don't even know if it still works.

Browse Source 
Was trying to get it to have a probability by using multiple fingers.
Also trying to get it to get the best box from opencv.
This commit is contained in:
Michael Pivato
2019-03-01 15:48:11 +10:30
parent 49d18a021d
commit 983b503463
1 changed files with 140 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
GestureRecognition/SimpleHandRecogniser.py
View File

@@ -1,14 +1,14 @@
from GestureRecognition.handrecogniser import HandRecogniser
import numpy as np import numpy as np
import cv2 import cv2
# import tensorflow as tf
import multiprocessing as mp from GestureRecognition.handrecogniser import HandRecogniser
class SimpleHandRecogniser(HandRecogniser): class SimpleHandRecogniser(HandRecogniser):
def __init__(self, frame): def __init__(self, frame):
self.img = frame self.img = frame
self.graph = None self.graph = None
self.sess = None self.sess = None
self.img_cut = None
def __calc_pos_y(self, x, radius, centre): def __calc_pos_y(self, x, radius, centre):
""" """
@@ -20,17 +20,17 @@ class SimpleHandRecogniser(HandRecogniser):
""" """
Segments the hand from the rest of the image to get a threshold. Segments the hand from the rest of the image to get a threshold.
""" """
self.img_hsv = cv2.GaussianBlur(self.img_hsv,(5,5),0) self.img_cut = cv2.GaussianBlur(self.img_cut, (5, 5), 0)
lower_skin = (0, 0, 153) lower_skin = (0, 0, 153)
upper_skin = (45, 153, 255) upper_skin = (45, 153, 255)
# Only need mask, as we can just use this to do the hand segmentation. # Only need mask, as we can just use this to do the hand segmentation.
self.mask = cv2.inRange(self.img_hsv, lower_skin, upper_skin) self.img_cut = cv2.inRange(self.img_cut, lower_skin, upper_skin)
# Apply another blur to rmeove any small holes/noise # Apply another blur to rmeove any small holes/noise
self.mask = self.__denoise(self.mask) self.img_cut = self.__denoise(self.img_cut)
ret, self.mask = cv2.threshold(self.mask, 50, 255, cv2.THRESH_BINARY) ret, self.img_cut = cv2.threshold(self.img_cut, 50, 255, cv2.THRESH_BINARY)
def __denoise(self, image): def __denoise(self, image):
""" """
@@ -45,29 +45,30 @@ class SimpleHandRecogniser(HandRecogniser):
given threshold pixels, and the radius is by default 55% of the total given threshold pixels, and the radius is by default 55% of the total
size. size.
""" """
k = np.sum(self.mask) / 255 k = np.sum(self.img_cut) / 255
# Taking indices for num of rows. # Taking indices for num of rows.
x_ind = np.arange(0,self.mask.shape[1]) x_ind = np.arange(0, self.img_cut.shape[1])
y_ind = np.arange(0,self.mask.shape[0]) y_ind = np.arange(0, self.img_cut.shape[0])
coords_x = np.zeros((self.mask.shape[0], self.mask.shape[1])) coords_x = np.zeros((self.img_cut.shape[0], self.img_cut.shape[1]))
coords_y = np.zeros((self.mask.shape[0], self.mask.shape[1])) coords_y = np.zeros((self.img_cut.shape[0], self.img_cut.shape[1]))
coords_x[:, :] = x_ind coords_x[:, :] = x_ind
# Even this is extremely quick as it goes through rows in the numpy array, which in python is much faster than columns # Even this is extremely quick as it goes through rows in the numpy array,
# which in python is much faster than columns
for element in y_ind: for element in y_ind:
coords_y[element, :] = element coords_y[element, :] = element
# Now need to get the average x value and y value for centre of gravity # Now need to get the average x value and y value for centre of gravity
centre = (int(np.sum(coords_x[self.mask == 255])/k), int(np.sum(coords_y[self.mask == 255])/k)) centre = (int(np.sum(coords_x[self.img_cut == 255])/k), int(np.sum(coords_y[self.img_cut == 255])/k))
# Calculate radius of circle: # Calculate radius of circle:
# May need to calculate diameter as well. # May need to calculate diameter as well.
# Just take min/max x values and y values # Just take min/max x values and y values
x_min = np.min(coords_x[self.mask == 255]) x_min = np.min(coords_x[self.img_cut == 255])
x_max = np.max(coords_x[self.mask == 255]) x_max = np.max(coords_x[self.img_cut == 255])
y_min = np.min(coords_y[self.mask == 255]) y_min = np.min(coords_y[self.img_cut == 255])
y_max = np.max(coords_y[self.mask == 255]) y_max = np.max(coords_y[self.img_cut == 255])
candidate_pts = [(x_min, y_min), (x_min, y_max), (x_max, y_min), (x_max, y_max)] candidate_pts = [(x_min, y_min), (x_min, y_max), (x_max, y_min), (x_max, y_max)]
radius = 0 radius = 0
@@ -83,12 +84,59 @@ class SimpleHandRecogniser(HandRecogniser):
return radius, centre return radius, centre
def __calc_circles(self, image, radius_percent_range=[0.6, 0.8], step = 0.1):
"""
Calculates the equation of the circle (radius, centre), but with
several radii so that we can get a more accurate estimate of from a given
threshold image, so that the circle is the center of gravity of the
given threshold pixels.
"""
k = np.sum(self.img_cut) / 255
# Taking indices for num of rows.
x_ind = np.arange(0,self.img_cut.shape[1])
y_ind = np.arange(0,self.img_cut.shape[0])
coords_x = np.zeros((self.img_cut.shape[0], self.img_cut.shape[1]))
coords_y = np.zeros((self.img_cut.shape[0], self.img_cut.shape[1]))
coords_x[:,:] = x_ind
# Even this is extremely quick as it goes through rows in the numpy array, which in python is much faster than columns
for element in y_ind:
coords_y[element,:] = element
# Now need to get the average x value and y value for centre of gravity
centre = (int(np.sum(coords_x[self.img_cut == 255])/k), int(np.sum(coords_y[self.img_cut == 255])/k))
# Calculate radius of circle:
# May need to calculate diameter as well.
# Just take min/max x values and y values
x_min = np.min(coords_x[self.img_cut == 255])
x_max = np.max(coords_x[self.img_cut == 255])
y_min = np.min(coords_y[self.img_cut == 255])
y_max = np.max(coords_y[self.img_cut == 255])
candidate_pts = [(x_min, y_min), (x_min, y_max), (x_max, y_min), (x_max, y_max)]
radius = 0
# Check with each point to see which is furthest from the centre.
for pt in candidate_pts:
# Calculate Euclydian Distance
new_distance = ((pt[0] - centre[0])**2 + (pt[1] - centre[1])**2)**(1/2)
if new_distance > radius:
radius = new_distance
radii = []
for i in range(radius_percent_range[0], radius_percent_range[1], step):
radii += int(radius * i)
return radii, centre
def __shift_pixels(self, image, shift_radius): def __shift_pixels(self, image, shift_radius):
image[:, :, 0] = image[:, :, 0] + shift_radius image[:, :, 0] = image[:, :, 0] + shift_radius
image[:, :, 0] = np.where(image[:, :, 0] > 179, image[:, :, 0] - 179, image[:, :, 0]) image[:, :, 0] = np.where(image[:, :, 0] > 179, image[:, :, 0] - 179, image[:, :, 0])
return image return image
def setFrame(self, frame): def set_frame(self, frame):
self.img = frame self.img = frame
# Source: Victor Dibia # Source: Victor Dibia
@@ -156,7 +204,7 @@ class SimpleHandRecogniser(HandRecogniser):
""" """
self.net = cv2.dnn.readNetFromTensorflow(graph_path, names_path) self.net = cv2.dnn.readNetFromTensorflow(graph_path, names_path)
def detect_hand_opencv(self, detection_graph, sess): def detect_hand_opencv(self):
"""Performs hand detection using a CNN from tensorflow using opencv. """Performs hand detection using a CNN from tensorflow using opencv.
detection_graph -- The CNN to use to detect the hand. detection_graph -- The CNN to use to detect the hand.
@@ -165,30 +213,30 @@ class SimpleHandRecogniser(HandRecogniser):
if self.img is None: if self.img is None:
return return
height = self.img.shape[0] rows = self.img.shape[0]
width = self.img.shape[1] cols = self.img.shape[1]
scale = 0.5 self.net.setInput(cv2.dnn.blobFromImage(self.img, size=(300, 300), swapRB=True, crop=False))
cv_out = self.net.forward()
classes = None
net = cv2.dnn.readNetFromTensorflow(detection_graph, sess)
# width is scaled weirdly to ensure we keep tbe same ratio as the original image.
net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False))
netOut = net.forward()
# Format output to look same as tensorflow output.
scores = []
boxes = [] boxes = []
scores = []
for out in netOut: for detection in cv_out[0, 0, :, :]:
for detection in out[0,0]: score = float(detection[2])
scores.append(detection[2]) # TODO: Need to make this the confidence threshold...
boxes.append(detection[3], detection[4], detection[5], detection[6]) if score > 0.6:
# Only doing first class as only trying to find the hand. left = detection[3] * cols
top = detection[4] * rows
right = detection[5] * cols
bottom = detection[6] * rows
boxes.append((left, top, right, bottom))
scores.append(score)
else:
# Scores are in descending order...
break break
return np.array(boxes), np.array(scores)
return boxes, scores
def get_best_hand(self, boxes, scores, conf_thresh, nms_thresh): def get_best_hand(self, boxes, scores, conf_thresh, nms_thresh):
""" """
@@ -250,22 +298,37 @@ class SimpleHandRecogniser(HandRecogniser):
# print("loaded inference graph") # print("loaded inference graph")
# detections, scores = self.detect_hand_tensorflow(self.graph, self.sess) # detections, scores = self.detect_hand_tensorflow(self.graph, self.sess)
print('Loading openCV net')
self.load_cv_net('/Users/piv/Documents/Projects/car/GestureRecognition/frozen_inference_graph.pb',
'/Users/piv/Documents/Projects/car/GestureRecognition/graph.pbtxt')
detections, scores = self.detect_hand_opencv()
# print("Getting best hand") # print("Getting best hand")
# best_hand = self.get_best_hand(detections, scores, 0.7, 0.5) # best_hand = self.get_best_hand(detections, scores, 0.7, 0.5)
# if best_hand is not None: # if best_hand is not None:
# self.img = self.img[best_hand[0] - 30:best_hand[2] + 30, best_hand[1] - 30:best_hand[3] + 30] # self.img = self.img[best_hand[0] - 30:best_hand[2] + 30, best_hand[1] - 30:best_hand[3] + 30]
if len(detections) > 0:
print("Cutting out the hand!")
self.img_cut = self.img[detections[0] - 30:detections[2] + 30, detections[1] - 30:detections[3] + 30]
else:
self.img_cut = self.img
print('Attempting to use pure hand recognition') print('Attempting to use pure hand recognition')
self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV) self.img_cut = cv2.cvtColor(self.img_cut, cv2.COLOR_BGR2HSV)
# Need to shift red pixels so they can be 0-20 rather than 250-~20 # Need to shift red pixels so they can be 0-20 rather than 250-~20
self.img_hsv = self.__shift_pixels(self.img_hsv, 30) self.img_cut = self.__shift_pixels(self.img_cut, 30)
self.img_hsv = self.__denoise(self.img_hsv) self.img_cut = self.__denoise(self.img_cut)
self.__segment_image() self.__segment_image()
print('calculating circle') print('calculating circle')
radius, centre = self.__calc_circle(self.mask) # Could calculate multiple circles to get probability
# for each gesture (i.e. calc num of each gesture recongised and take percentage
# as the probability).
radius, centre = self.__calc_circle(self.img_cut)
print('Got circle') print('Got circle')
# Now go around the circle to calculate num of times going 0->255 or vice-versa. # Now go around the circle to calculate num of times going 0->255 or vice-versa.
@@ -273,7 +336,8 @@ class SimpleHandRecogniser(HandRecogniser):
# Equation of the circle: # Equation of the circle:
# y = sqrt(r2 - (x-c)2) + c # y = sqrt(r2 - (x-c)2) + c
prev_x = centre[0] - radius prev_x = centre[0] - radius
prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre), self.__calc_pos_y(centre[0] - radius, radius, centre)] prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre),
self.__calc_pos_y(centre[0] - radius, radius, centre)]
num_change = 0 num_change = 0
# Make sure x is also within bounds. # Make sure x is also within bounds.
@@ -282,15 +346,8 @@ class SimpleHandRecogniser(HandRecogniser):
x_start = 0 x_start = 0
x_end = centre[0] + radius x_end = centre[0] + radius
if x_end >= self.mask.shape[1]: if x_end >= self.img_cut.shape[1]:
x_end = self.mask.shape[1] - 1 x_end = self.img_cut.shape[1] - 1
# Could batch this function to execute on multiple cores?
# Calc num CPUS.
# num_cores = mp.cpu_count()
# # Calc batch size:
# batch_size = x_end // num_cores
# for b in range(0, num_cores - 1):
# pass
for x in range(x_start, x_end): for x in range(x_start, x_end):
# Need to check circle is inside the bounds. # Need to check circle is inside the bounds.
@@ -300,22 +357,25 @@ class SimpleHandRecogniser(HandRecogniser):
if y[0] < 0: if y[0] < 0:
y[0] = 0 y[0] = 0
if y[0] >= self.mask.shape[0]: if y[0] >= self.img_cut.shape[0]:
y[0] = self.mask.shape[0] - 1 y[0] = self.img_cut.shape[0] - 1
if y[1] < 0: if y[1] < 0:
y[1] = 0 y[1] = 0
if y[1] >= self.mask.shape[0]: if y[1] >= self.img_cut.shape[0]:
y[1] = self.mask.shape[0] - 1 y[1] = self.img_cut.shape[0] - 1
if(self.mask[y[0], x] != self.mask[prev_y[0], prev_x]): if(self.img_cut[y[0], x] != self.img_cut[prev_y[0], prev_x]):
num_change += 1 num_change += 1
if self.mask[y[1], x] != self.mask[prev_y[1], prev_x] and y[0] != y[1]: if self.img_cut[y[1], x] != self.img_cut[prev_y[1], prev_x] and y[0] != y[1]:
num_change += 1 num_change += 1
prev_x = x prev_x = x
prev_y = y prev_y = y
print('Finished calculating, returning') print('Finished calculating, returning')
print(num_change) print(num_change)
return int(num_change / 2 - 1) return int(num_change / 2 - 1), self.img
def get_gesture_multiple_radii(self):
pass
def calc_hand_batch(self, batch): def calc_hand_batch(self, batch):
pass pass