Move get gesture, make circle 0.6
This commit is contained in:
Michael Pivato
@@ -37,7 +37,7 @@ class SimpleHandRecogniser(HandRecogniser):
|
|||||||
"""
|
"""
|
||||||
return cv2.GaussianBlur(image,(5,5),0)
|
return cv2.GaussianBlur(image,(5,5),0)
|
||||||
|
|
||||||
def __calc_circle(self, image, radius_percent = 0.52):
|
def __calc_circle(self, image, radius_percent = 0.6):
|
||||||
"""
|
"""
|
||||||
Calculates the equation of the circle (radius, centre) from a given
|
Calculates the equation of the circle (radius, centre) from a given
|
||||||
threshold image, so that the circle is the center of gravity of the
|
threshold image, so that the circle is the center of gravity of the
|
||||||
@@ -87,77 +87,6 @@ class SimpleHandRecogniser(HandRecogniser):
|
|||||||
image[:,:,0] = np.where(image[:,:,0] > 179, image[:,:,0] - 179, image[:,:,0])
|
image[:,:,0] = np.where(image[:,:,0] > 179, image[:,:,0] - 179, image[:,:,0])
|
||||||
return image
|
return image
|
||||||
|
|
||||||
def get_gesture(self):
|
|
||||||
"""
|
|
||||||
Calculates the actual gesture, returning the number of fingers
|
|
||||||
seen in the image.
|
|
||||||
"""
|
|
||||||
print('Getting Gesture')
|
|
||||||
if self.img is None:
|
|
||||||
print('There is no image')
|
|
||||||
return -1
|
|
||||||
# First cut out the frame using the neural network.
|
|
||||||
self.load_inference_graph()
|
|
||||||
print("loaded inference graph")
|
|
||||||
detections, scores = self.detect_hand_tensorflow(self.graph, self.sess)
|
|
||||||
|
|
||||||
print('Attempting to use pure hand recognition')
|
|
||||||
self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV)
|
|
||||||
|
|
||||||
# Need to shift red pixels so they can be 0-20 rather than 250-~20
|
|
||||||
self.img_hsv = self.__shift_pixels(self.img_hsv, 30)
|
|
||||||
|
|
||||||
self.img_hsv = self.__denoise(self.img_hsv)
|
|
||||||
self.__segment_image()
|
|
||||||
|
|
||||||
print('calculating circle')
|
|
||||||
radius, centre = self.__calc_circle(self.mask)
|
|
||||||
print('Got circle')
|
|
||||||
|
|
||||||
# Now go around the circle to calculate num of times going 0->255 or vice-versa.
|
|
||||||
# First just do it the naive way with loops.
|
|
||||||
# Equation of the circle:
|
|
||||||
# y = sqrt(r2 - (x-c)2) + c
|
|
||||||
prev_x = centre[0] - radius
|
|
||||||
prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre), self.__calc_pos_y(centre[0] - radius, radius, centre)]
|
|
||||||
num_change = 0
|
|
||||||
|
|
||||||
# Make sure x is also within bounds.
|
|
||||||
x_start = centre[0] - radius + 1
|
|
||||||
if x_start < 0:
|
|
||||||
x_start = 0
|
|
||||||
|
|
||||||
x_end = centre[0] + radius
|
|
||||||
if x_end >= self.mask.shape[1]:
|
|
||||||
x_end = self.mask.shape[1] - 1
|
|
||||||
print(x_start)
|
|
||||||
print(x_end)
|
|
||||||
print(self.mask.shape)
|
|
||||||
for x in range(x_start, x_end):
|
|
||||||
# Need to check circle is inside the bounds.
|
|
||||||
ypos = self.__calc_pos_y(x, radius, centre)
|
|
||||||
# y above centre (ypos) and y below radius)
|
|
||||||
y = [ypos, centre[1] - (ypos-centre[1])]
|
|
||||||
|
|
||||||
if y[0] < 0:
|
|
||||||
y[0] = 0
|
|
||||||
if y[0] >= self.mask.shape[0]:
|
|
||||||
y[0] = self.mask.shape[0] - 1
|
|
||||||
if y[1] < 0:
|
|
||||||
y[1] = 0
|
|
||||||
if y[1] >= self.mask.shape[0]:
|
|
||||||
y[1] = self.mask.shape[0] - 1
|
|
||||||
if(self.mask[y[0], x] != self.mask[prev_y[0], prev_x]):
|
|
||||||
num_change += 1
|
|
||||||
if self.mask[y[1], x] != self.mask[prev_y[1], prev_x] and y[0] != y[1]:
|
|
||||||
num_change += 1
|
|
||||||
prev_x = x
|
|
||||||
prev_y = y
|
|
||||||
|
|
||||||
print('Finished calculating, returning')
|
|
||||||
|
|
||||||
return num_change / 2 - 1
|
|
||||||
|
|
||||||
def setFrame(self, frame):
|
def setFrame(self, frame):
|
||||||
self.img = frame
|
self.img = frame
|
||||||
|
|
||||||
@@ -170,7 +99,7 @@ class SimpleHandRecogniser(HandRecogniser):
|
|||||||
"""Loads a tensorflow model checkpoint into memory"""
|
"""Loads a tensorflow model checkpoint into memory"""
|
||||||
|
|
||||||
if self.graph != None and self.sess != None:
|
if self.graph != None and self.sess != None:
|
||||||
# Don't load more than once.
|
# Don't load more than once, to save time...
|
||||||
return
|
return
|
||||||
|
|
||||||
PATH_TO_CKPT = '/Users/piv/Documents/Projects/car/GestureRecognition/frozen_inference_graph.pb'
|
PATH_TO_CKPT = '/Users/piv/Documents/Projects/car/GestureRecognition/frozen_inference_graph.pb'
|
||||||
@@ -217,39 +146,39 @@ class SimpleHandRecogniser(HandRecogniser):
|
|||||||
print('finished detection')
|
print('finished detection')
|
||||||
return np.squeeze(boxes), np.squeeze(scores)
|
return np.squeeze(boxes), np.squeeze(scores)
|
||||||
|
|
||||||
def detect_hand_opencv(self, detection_graph, sess):
|
# def detect_hand_opencv(self, detection_graph, sess):
|
||||||
"""Performs hand detection using a CNN from tensorflow using opencv.
|
# """Performs hand detection using a CNN from tensorflow using opencv.
|
||||||
|
|
||||||
detection_graph -- The CNN to use to detect the hand.
|
# detection_graph -- The CNN to use to detect the hand.
|
||||||
sess -- THe tensorflow session for the given graph
|
# sess -- THe tensorflow session for the given graph
|
||||||
"""
|
# """
|
||||||
if self.img is None:
|
# if self.img is None:
|
||||||
return
|
# return
|
||||||
|
|
||||||
height = self.img.shape[0]
|
# height = self.img.shape[0]
|
||||||
width = self.img.shape[1]
|
# width = self.img.shape[1]
|
||||||
|
|
||||||
scale = 0.5
|
# scale = 0.5
|
||||||
|
|
||||||
classes = None
|
# classes = None
|
||||||
|
|
||||||
net = cv2.dnn.readNetFromTensorflow(detection_graph, sess)
|
# net = cv2.dnn.readNetFromTensorflow(detection_graph, sess)
|
||||||
|
|
||||||
# width is scaled weirdly to ensure we keep tbe same ratio as the original image.
|
# # width is scaled weirdly to ensure we keep tbe same ratio as the original image.
|
||||||
net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False))
|
# net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False))
|
||||||
netOut = net.forward()
|
# netOut = net.forward()
|
||||||
|
|
||||||
# Format output to look same as tensorflow output.
|
# # Format output to look same as tensorflow output.
|
||||||
scores = []
|
# scores = []
|
||||||
boxes = []
|
# boxes = []
|
||||||
|
|
||||||
for out in netOut:
|
# for out in netOut:
|
||||||
for detection in out[0,0]:
|
# for detection in out[0,0]:
|
||||||
scores.append(detection[2])
|
# scores.append(detection[2])
|
||||||
boxes.append(detection[3], detection[4], detection[5], detection[6])
|
# boxes.append(detection[3], detection[4], detection[5], detection[6])
|
||||||
# Only doing first class as only trying to find the hand.
|
# # Only doing first class as only trying to find the hand.
|
||||||
break
|
# break
|
||||||
return np.array(boxes), np.array(scores)
|
# return np.array(boxes), np.array(scores)
|
||||||
|
|
||||||
def get_best_hand(self, boxes, scores, conf_thresh, nms_thresh):
|
def get_best_hand(self, boxes, scores, conf_thresh, nms_thresh):
|
||||||
"""
|
"""
|
||||||
@@ -257,22 +186,119 @@ class SimpleHandRecogniser(HandRecogniser):
|
|||||||
boxes, as well as the overall size of each box to determine which hand (if multiple present)
|
boxes, as well as the overall size of each box to determine which hand (if multiple present)
|
||||||
should be tested to recognise.
|
should be tested to recognise.
|
||||||
"""
|
"""
|
||||||
# First remove any boxes below confidence threshold
|
print(scores)
|
||||||
confident_bs = boxes[scores > conf_thresh]
|
boxes = boxes[scores > conf_thresh]
|
||||||
|
scores = scores[scores > conf_thresh]
|
||||||
# Then use NMS to get rid of heavily overlapping boxes.
|
# Use NMS to get rid of heavily overlapping boxes.
|
||||||
# This wasn't used in the tensorflow example that was found, however probably a
|
# This wasn't used in the tensorflow example that was found, however probably a
|
||||||
# good idea to use it just in case.
|
# good idea to use it just in case.
|
||||||
indices = cv2.dnn.NMSBoxes(boxes, scores, conf_thresh, nms_thresh)
|
print(boxes.shape)
|
||||||
|
if boxes.shape[0] == 0:
|
||||||
|
print("No good boxes found")
|
||||||
|
return None
|
||||||
|
elif boxes.shape[0] == 1:
|
||||||
|
print("Only one good box!")
|
||||||
|
box = boxes[0]
|
||||||
|
box[0] = box[0] * self.img.shape[0]
|
||||||
|
box[1] = box[1] * self.img.shape[1]
|
||||||
|
box[2] = box[2] * self.img.shape[0]
|
||||||
|
box[3] = box[3] * self.img.shape[1]
|
||||||
|
return box.astype(int)
|
||||||
|
else:
|
||||||
|
boxes[:][2] = ((boxes[:][2] - boxes[:][0]) * self.img.shape[0]).astype(int)
|
||||||
|
boxes[:][3] = ((boxes[:][3] - boxes[:][1]) * self.img.shape[1]).astype(int)
|
||||||
|
boxes[:][0] = (boxes[:][0] * self.img.shape[0]).astype(int)
|
||||||
|
boxes[:][1] = (boxes[:][1] * self.img.shape[1]).astype(int)
|
||||||
|
|
||||||
|
# Can't seem to get this to work...
|
||||||
|
# indices = cv2.dnn.NMSBoxes(boxes, scores, conf_thresh, nms_thresh)
|
||||||
|
|
||||||
|
print("Num boxes: %s" % boxes.shape[0])
|
||||||
# Finally calculate area of each box to determine which hand is clearest (biggest in image)
|
# Finally calculate area of each box to determine which hand is clearest (biggest in image)
|
||||||
# Just does the most confident for now.
|
# Just does the most confident for now.
|
||||||
max_conf = 0
|
best_box = boxes[0]
|
||||||
max_index = 0
|
best_index = None
|
||||||
for conf in scores:
|
i = 0
|
||||||
if conf > max_conf:
|
for box in boxes:
|
||||||
max_conf = conf
|
if box[2] * box[3] > best_box[2] * best_box[3]:
|
||||||
max_index = conf
|
best_box = box
|
||||||
return boxes[max_index]
|
best_index = i
|
||||||
|
i += 1
|
||||||
|
return boxes[i - 1]
|
||||||
|
|
||||||
|
def get_gesture(self):
|
||||||
|
"""
|
||||||
|
Calculates the actual gesture, returning the number of fingers
|
||||||
|
seen in the image.
|
||||||
|
"""
|
||||||
|
print('Getting Gesture')
|
||||||
|
if self.img is None:
|
||||||
|
print('There is no image')
|
||||||
|
return -1
|
||||||
|
# First cut out the frame using the neural network.
|
||||||
|
self.load_inference_graph()
|
||||||
|
print("loaded inference graph")
|
||||||
|
detections, scores = self.detect_hand_tensorflow(self.graph, self.sess)
|
||||||
|
|
||||||
|
print("Getting best hand")
|
||||||
|
best_hand = self.get_best_hand(detections, scores, 0.7, 0.5)
|
||||||
|
if best_hand is not None:
|
||||||
|
self.img = self.img[best_hand[0] - 30:best_hand[2] + 30, best_hand[1] - 30:best_hand[3] + 30]
|
||||||
|
|
||||||
|
print('Attempting to use pure hand recognition')
|
||||||
|
self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV)
|
||||||
|
|
||||||
|
# Need to shift red pixels so they can be 0-20 rather than 250-~20
|
||||||
|
self.img_hsv = self.__shift_pixels(self.img_hsv, 30)
|
||||||
|
|
||||||
|
self.img_hsv = self.__denoise(self.img_hsv)
|
||||||
|
self.__segment_image()
|
||||||
|
|
||||||
|
print('calculating circle')
|
||||||
|
radius, centre = self.__calc_circle(self.mask)
|
||||||
|
print('Got circle')
|
||||||
|
|
||||||
|
# Now go around the circle to calculate num of times going 0->255 or vice-versa.
|
||||||
|
# First just do it the naive way with loops.
|
||||||
|
# Equation of the circle:
|
||||||
|
# y = sqrt(r2 - (x-c)2) + c
|
||||||
|
prev_x = centre[0] - radius
|
||||||
|
prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre), self.__calc_pos_y(centre[0] - radius, radius, centre)]
|
||||||
|
num_change = 0
|
||||||
|
|
||||||
|
# Make sure x is also within bounds.
|
||||||
|
x_start = centre[0] - radius + 1
|
||||||
|
if x_start < 0:
|
||||||
|
x_start = 0
|
||||||
|
|
||||||
|
x_end = centre[0] + radius
|
||||||
|
if x_end >= self.mask.shape[1]:
|
||||||
|
x_end = self.mask.shape[1] - 1
|
||||||
|
# Could batch this function to execute on multiple cores?
|
||||||
|
for x in range(x_start, x_end):
|
||||||
|
# Need to check circle is inside the bounds.
|
||||||
|
ypos = self.__calc_pos_y(x, radius, centre)
|
||||||
|
# y above centre (ypos) and y below radius)
|
||||||
|
y = [ypos, centre[1] - (ypos-centre[1])]
|
||||||
|
|
||||||
|
if y[0] < 0:
|
||||||
|
y[0] = 0
|
||||||
|
if y[0] >= self.mask.shape[0]:
|
||||||
|
y[0] = self.mask.shape[0] - 1
|
||||||
|
if y[1] < 0:
|
||||||
|
y[1] = 0
|
||||||
|
if y[1] >= self.mask.shape[0]:
|
||||||
|
y[1] = self.mask.shape[0] - 1
|
||||||
|
if(self.mask[y[0], x] != self.mask[prev_y[0], prev_x]):
|
||||||
|
num_change += 1
|
||||||
|
if self.mask[y[1], x] != self.mask[prev_y[1], prev_x] and y[0] != y[1]:
|
||||||
|
num_change += 1
|
||||||
|
prev_x = x
|
||||||
|
prev_y = y
|
||||||
|
|
||||||
|
print('Finished calculating, returning')
|
||||||
|
print(num_change)
|
||||||
|
return int(num_change / 2 - 1)
|
||||||
|
|
||||||
|
def calc_hand_batch(self, batch):
|
||||||
|
pass
|
||||||
Reference in New Issue
Block a user