vato007/picar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move get gesture, make circle 0.6

Browse Source
This commit is contained in:
Michael Pivato
2019-02-05 11:54:17 +10:30
parent 6289451f26
commit 4c25649c26
1 changed files with 137 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

248
GestureRecognition/SimpleHandRecogniser.py
View File

@@ -37,7 +37,7 @@ class SimpleHandRecogniser(HandRecogniser):
"""
return cv2.GaussianBlur(image,(5,5),0)
def __calc_circle(self, image, radius_percent = 0.52):
def __calc_circle(self, image, radius_percent = 0.6):
"""
Calculates the equation of the circle (radius, centre) from a given
threshold image, so that the circle is the center of gravity of the
@@ -87,77 +87,6 @@ class SimpleHandRecogniser(HandRecogniser):
image[:,:,0] = np.where(image[:,:,0] > 179, image[:,:,0] - 179, image[:,:,0])
return image
def get_gesture(self):
"""
Calculates the actual gesture, returning the number of fingers
seen in the image.
"""
print('Getting Gesture')
if self.img is None:
print('There is no image')
return -1
# First cut out the frame using the neural network.
self.load_inference_graph()
print("loaded inference graph")
detections, scores = self.detect_hand_tensorflow(self.graph, self.sess)
print('Attempting to use pure hand recognition')
self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV)
# Need to shift red pixels so they can be 0-20 rather than 250-~20
self.img_hsv = self.__shift_pixels(self.img_hsv, 30)
self.img_hsv = self.__denoise(self.img_hsv)
self.__segment_image()
print('calculating circle')
radius, centre = self.__calc_circle(self.mask)
print('Got circle')
# Now go around the circle to calculate num of times going 0->255 or vice-versa.
# First just do it the naive way with loops.
# Equation of the circle:
# y = sqrt(r2 - (x-c)2) + c
prev_x = centre[0] - radius
prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre), self.__calc_pos_y(centre[0] - radius, radius, centre)]
num_change = 0
# Make sure x is also within bounds.
x_start = centre[0] - radius + 1
if x_start < 0:
x_start = 0
x_end = centre[0] + radius
if x_end >= self.mask.shape[1]:
x_end = self.mask.shape[1] - 1
print(x_start)
print(x_end)
print(self.mask.shape)
for x in range(x_start, x_end):
# Need to check circle is inside the bounds.
ypos = self.__calc_pos_y(x, radius, centre)
# y above centre (ypos) and y below radius)
y = [ypos, centre[1] - (ypos-centre[1])]
if y[0] < 0:
y[0] = 0
if y[0] >= self.mask.shape[0]:
y[0] = self.mask.shape[0] - 1
if y[1] < 0:
y[1] = 0
if y[1] >= self.mask.shape[0]:
y[1] = self.mask.shape[0] - 1
if(self.mask[y[0], x] != self.mask[prev_y[0], prev_x]):
num_change += 1
if self.mask[y[1], x] != self.mask[prev_y[1], prev_x] and y[0] != y[1]:
num_change += 1
prev_x = x
prev_y = y
print('Finished calculating, returning')
return num_change / 2 - 1
def setFrame(self, frame):
self.img = frame
@@ -170,7 +99,7 @@ class SimpleHandRecogniser(HandRecogniser):
"""Loads a tensorflow model checkpoint into memory"""
if self.graph != None and self.sess != None:
# Don't load more than once.
# Don't load more than once, to save time...
return
PATH_TO_CKPT = '/Users/piv/Documents/Projects/car/GestureRecognition/frozen_inference_graph.pb'
@@ -217,39 +146,39 @@ class SimpleHandRecogniser(HandRecogniser):
print('finished detection')
return np.squeeze(boxes), np.squeeze(scores)
def detect_hand_opencv(self, detection_graph, sess):
"""Performs hand detection using a CNN from tensorflow using opencv.
# def detect_hand_opencv(self, detection_graph, sess):
# """Performs hand detection using a CNN from tensorflow using opencv.
detection_graph -- The CNN to use to detect the hand.
sess -- THe tensorflow session for the given graph
"""
if self.img is None:
return
# detection_graph -- The CNN to use to detect the hand.
# sess -- THe tensorflow session for the given graph
# """
# if self.img is None:
# return
height = self.img.shape[0]
width = self.img.shape[1]
# height = self.img.shape[0]
# width = self.img.shape[1]
scale = 0.5
# scale = 0.5
classes = None
# classes = None
net = cv2.dnn.readNetFromTensorflow(detection_graph, sess)
# net = cv2.dnn.readNetFromTensorflow(detection_graph, sess)
# width is scaled weirdly to ensure we keep tbe same ratio as the original image.
net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False))
netOut = net.forward()
# # width is scaled weirdly to ensure we keep tbe same ratio as the original image.
# net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False))
# netOut = net.forward()
# Format output to look same as tensorflow output.
scores = []
boxes = []
# # Format output to look same as tensorflow output.
# scores = []
# boxes = []
for out in netOut:
for detection in out[0,0]:
scores.append(detection[2])
boxes.append(detection[3], detection[4], detection[5], detection[6])
# Only doing first class as only trying to find the hand.
break
return np.array(boxes), np.array(scores)
# for out in netOut:
# for detection in out[0,0]:
# scores.append(detection[2])
# boxes.append(detection[3], detection[4], detection[5], detection[6])
# # Only doing first class as only trying to find the hand.
# break
# return np.array(boxes), np.array(scores)
def get_best_hand(self, boxes, scores, conf_thresh, nms_thresh):
"""
@@ -257,22 +186,119 @@ class SimpleHandRecogniser(HandRecogniser):
boxes, as well as the overall size of each box to determine which hand (if multiple present)
should be tested to recognise.
"""
# First remove any boxes below confidence threshold
confident_bs = boxes[scores > conf_thresh]
# Then use NMS to get rid of heavily overlapping boxes.
print(scores)
boxes = boxes[scores > conf_thresh]
scores = scores[scores > conf_thresh]
# Use NMS to get rid of heavily overlapping boxes.
# This wasn't used in the tensorflow example that was found, however probably a
# good idea to use it just in case.
indices = cv2.dnn.NMSBoxes(boxes, scores, conf_thresh, nms_thresh)
print(boxes.shape)
if boxes.shape[0] == 0:
print("No good boxes found")
return None
elif boxes.shape[0] == 1:
print("Only one good box!")
box = boxes[0]
box[0] = box[0] * self.img.shape[0]
box[1] = box[1] * self.img.shape[1]
box[2] = box[2] * self.img.shape[0]
box[3] = box[3] * self.img.shape[1]
return box.astype(int)
else:
boxes[:][2] = ((boxes[:][2] - boxes[:][0]) * self.img.shape[0]).astype(int)
boxes[:][3] = ((boxes[:][3] - boxes[:][1]) * self.img.shape[1]).astype(int)
boxes[:][0] = (boxes[:][0] * self.img.shape[0]).astype(int)
boxes[:][1] = (boxes[:][1] * self.img.shape[1]).astype(int)
# Can't seem to get this to work...
# indices = cv2.dnn.NMSBoxes(boxes, scores, conf_thresh, nms_thresh)
print("Num boxes: %s" % boxes.shape[0])
# Finally calculate area of each box to determine which hand is clearest (biggest in image)
# Just does the most confident for now.
max_conf = 0
max_index = 0
for conf in scores:
if conf > max_conf:
max_conf = conf
max_index = conf
return boxes[max_index]
best_box = boxes[0]
best_index = None
i = 0
for box in boxes:
if box[2] * box[3] > best_box[2] * best_box[3]:
best_box = box
best_index = i
i += 1
return boxes[i - 1]
def get_gesture(self):
"""
Calculates the actual gesture, returning the number of fingers
seen in the image.
"""
print('Getting Gesture')
if self.img is None:
print('There is no image')
return -1
# First cut out the frame using the neural network.
self.load_inference_graph()
print("loaded inference graph")
detections, scores = self.detect_hand_tensorflow(self.graph, self.sess)
print("Getting best hand")
best_hand = self.get_best_hand(detections, scores, 0.7, 0.5)
if best_hand is not None:
self.img = self.img[best_hand[0] - 30:best_hand[2] + 30, best_hand[1] - 30:best_hand[3] + 30]
print('Attempting to use pure hand recognition')
self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV)
# Need to shift red pixels so they can be 0-20 rather than 250-~20
self.img_hsv = self.__shift_pixels(self.img_hsv, 30)
self.img_hsv = self.__denoise(self.img_hsv)
self.__segment_image()
print('calculating circle')
radius, centre = self.__calc_circle(self.mask)
print('Got circle')
# Now go around the circle to calculate num of times going 0->255 or vice-versa.
# First just do it the naive way with loops.
# Equation of the circle:
# y = sqrt(r2 - (x-c)2) + c
prev_x = centre[0] - radius
prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre), self.__calc_pos_y(centre[0] - radius, radius, centre)]
num_change = 0
# Make sure x is also within bounds.
x_start = centre[0] - radius + 1
if x_start < 0:
x_start = 0
x_end = centre[0] + radius
if x_end >= self.mask.shape[1]:
x_end = self.mask.shape[1] - 1
# Could batch this function to execute on multiple cores?
for x in range(x_start, x_end):
# Need to check circle is inside the bounds.
ypos = self.__calc_pos_y(x, radius, centre)
# y above centre (ypos) and y below radius)
y = [ypos, centre[1] - (ypos-centre[1])]
if y[0] < 0:
y[0] = 0
if y[0] >= self.mask.shape[0]:
y[0] = self.mask.shape[0] - 1
if y[1] < 0:
y[1] = 0
if y[1] >= self.mask.shape[0]:
y[1] = self.mask.shape[0] - 1
if(self.mask[y[0], x] != self.mask[prev_y[0], prev_x]):
num_change += 1
if self.mask[y[1], x] != self.mask[prev_y[1], prev_x] and y[0] != y[1]:
num_change += 1
prev_x = x
prev_y = y
print('Finished calculating, returning')
print(num_change)
return int(num_change / 2 - 1)
def calc_hand_batch(self, batch):
pass