From 4c25649c26e8bb6baabd705ca04b526ee0825069 Mon Sep 17 00:00:00 2001 From: Michael Pivato Date: Tue, 5 Feb 2019 11:54:17 +1030 Subject: [PATCH] Move get gesture, make circle 0.6 --- GestureRecognition/SimpleHandRecogniser.py | 248 ++++++++++++--------- 1 file changed, 137 insertions(+), 111 deletions(-) diff --git a/GestureRecognition/SimpleHandRecogniser.py b/GestureRecognition/SimpleHandRecogniser.py index e3f1810..d8fd5ad 100644 --- a/GestureRecognition/SimpleHandRecogniser.py +++ b/GestureRecognition/SimpleHandRecogniser.py @@ -37,7 +37,7 @@ class SimpleHandRecogniser(HandRecogniser): """ return cv2.GaussianBlur(image,(5,5),0) - def __calc_circle(self, image, radius_percent = 0.52): + def __calc_circle(self, image, radius_percent = 0.6): """ Calculates the equation of the circle (radius, centre) from a given threshold image, so that the circle is the center of gravity of the @@ -87,77 +87,6 @@ class SimpleHandRecogniser(HandRecogniser): image[:,:,0] = np.where(image[:,:,0] > 179, image[:,:,0] - 179, image[:,:,0]) return image - def get_gesture(self): - """ - Calculates the actual gesture, returning the number of fingers - seen in the image. - """ - print('Getting Gesture') - if self.img is None: - print('There is no image') - return -1 - # First cut out the frame using the neural network. - self.load_inference_graph() - print("loaded inference graph") - detections, scores = self.detect_hand_tensorflow(self.graph, self.sess) - - print('Attempting to use pure hand recognition') - self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV) - - # Need to shift red pixels so they can be 0-20 rather than 250-~20 - self.img_hsv = self.__shift_pixels(self.img_hsv, 30) - - self.img_hsv = self.__denoise(self.img_hsv) - self.__segment_image() - - print('calculating circle') - radius, centre = self.__calc_circle(self.mask) - print('Got circle') - - # Now go around the circle to calculate num of times going 0->255 or vice-versa. - # First just do it the naive way with loops. - # Equation of the circle: - # y = sqrt(r2 - (x-c)2) + c - prev_x = centre[0] - radius - prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre), self.__calc_pos_y(centre[0] - radius, radius, centre)] - num_change = 0 - - # Make sure x is also within bounds. - x_start = centre[0] - radius + 1 - if x_start < 0: - x_start = 0 - - x_end = centre[0] + radius - if x_end >= self.mask.shape[1]: - x_end = self.mask.shape[1] - 1 - print(x_start) - print(x_end) - print(self.mask.shape) - for x in range(x_start, x_end): - # Need to check circle is inside the bounds. - ypos = self.__calc_pos_y(x, radius, centre) - # y above centre (ypos) and y below radius) - y = [ypos, centre[1] - (ypos-centre[1])] - - if y[0] < 0: - y[0] = 0 - if y[0] >= self.mask.shape[0]: - y[0] = self.mask.shape[0] - 1 - if y[1] < 0: - y[1] = 0 - if y[1] >= self.mask.shape[0]: - y[1] = self.mask.shape[0] - 1 - if(self.mask[y[0], x] != self.mask[prev_y[0], prev_x]): - num_change += 1 - if self.mask[y[1], x] != self.mask[prev_y[1], prev_x] and y[0] != y[1]: - num_change += 1 - prev_x = x - prev_y = y - - print('Finished calculating, returning') - - return num_change / 2 - 1 - def setFrame(self, frame): self.img = frame @@ -170,7 +99,7 @@ class SimpleHandRecogniser(HandRecogniser): """Loads a tensorflow model checkpoint into memory""" if self.graph != None and self.sess != None: - # Don't load more than once. + # Don't load more than once, to save time... return PATH_TO_CKPT = '/Users/piv/Documents/Projects/car/GestureRecognition/frozen_inference_graph.pb' @@ -217,39 +146,39 @@ class SimpleHandRecogniser(HandRecogniser): print('finished detection') return np.squeeze(boxes), np.squeeze(scores) - def detect_hand_opencv(self, detection_graph, sess): - """Performs hand detection using a CNN from tensorflow using opencv. + # def detect_hand_opencv(self, detection_graph, sess): + # """Performs hand detection using a CNN from tensorflow using opencv. - detection_graph -- The CNN to use to detect the hand. - sess -- THe tensorflow session for the given graph - """ - if self.img is None: - return + # detection_graph -- The CNN to use to detect the hand. + # sess -- THe tensorflow session for the given graph + # """ + # if self.img is None: + # return - height = self.img.shape[0] - width = self.img.shape[1] + # height = self.img.shape[0] + # width = self.img.shape[1] - scale = 0.5 + # scale = 0.5 - classes = None + # classes = None - net = cv2.dnn.readNetFromTensorflow(detection_graph, sess) + # net = cv2.dnn.readNetFromTensorflow(detection_graph, sess) - # width is scaled weirdly to ensure we keep tbe same ratio as the original image. - net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False)) - netOut = net.forward() + # # width is scaled weirdly to ensure we keep tbe same ratio as the original image. + # net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False)) + # netOut = net.forward() - # Format output to look same as tensorflow output. - scores = [] - boxes = [] + # # Format output to look same as tensorflow output. + # scores = [] + # boxes = [] - for out in netOut: - for detection in out[0,0]: - scores.append(detection[2]) - boxes.append(detection[3], detection[4], detection[5], detection[6]) - # Only doing first class as only trying to find the hand. - break - return np.array(boxes), np.array(scores) + # for out in netOut: + # for detection in out[0,0]: + # scores.append(detection[2]) + # boxes.append(detection[3], detection[4], detection[5], detection[6]) + # # Only doing first class as only trying to find the hand. + # break + # return np.array(boxes), np.array(scores) def get_best_hand(self, boxes, scores, conf_thresh, nms_thresh): """ @@ -257,22 +186,119 @@ class SimpleHandRecogniser(HandRecogniser): boxes, as well as the overall size of each box to determine which hand (if multiple present) should be tested to recognise. """ - # First remove any boxes below confidence threshold - confident_bs = boxes[scores > conf_thresh] - - # Then use NMS to get rid of heavily overlapping boxes. + print(scores) + boxes = boxes[scores > conf_thresh] + scores = scores[scores > conf_thresh] + # Use NMS to get rid of heavily overlapping boxes. # This wasn't used in the tensorflow example that was found, however probably a # good idea to use it just in case. - indices = cv2.dnn.NMSBoxes(boxes, scores, conf_thresh, nms_thresh) + print(boxes.shape) + if boxes.shape[0] == 0: + print("No good boxes found") + return None + elif boxes.shape[0] == 1: + print("Only one good box!") + box = boxes[0] + box[0] = box[0] * self.img.shape[0] + box[1] = box[1] * self.img.shape[1] + box[2] = box[2] * self.img.shape[0] + box[3] = box[3] * self.img.shape[1] + return box.astype(int) + else: + boxes[:][2] = ((boxes[:][2] - boxes[:][0]) * self.img.shape[0]).astype(int) + boxes[:][3] = ((boxes[:][3] - boxes[:][1]) * self.img.shape[1]).astype(int) + boxes[:][0] = (boxes[:][0] * self.img.shape[0]).astype(int) + boxes[:][1] = (boxes[:][1] * self.img.shape[1]).astype(int) + # Can't seem to get this to work... + # indices = cv2.dnn.NMSBoxes(boxes, scores, conf_thresh, nms_thresh) + + print("Num boxes: %s" % boxes.shape[0]) # Finally calculate area of each box to determine which hand is clearest (biggest in image) # Just does the most confident for now. - max_conf = 0 - max_index = 0 - for conf in scores: - if conf > max_conf: - max_conf = conf - max_index = conf - return boxes[max_index] + best_box = boxes[0] + best_index = None + i = 0 + for box in boxes: + if box[2] * box[3] > best_box[2] * best_box[3]: + best_box = box + best_index = i + i += 1 + return boxes[i - 1] - + def get_gesture(self): + """ + Calculates the actual gesture, returning the number of fingers + seen in the image. + """ + print('Getting Gesture') + if self.img is None: + print('There is no image') + return -1 + # First cut out the frame using the neural network. + self.load_inference_graph() + print("loaded inference graph") + detections, scores = self.detect_hand_tensorflow(self.graph, self.sess) + + print("Getting best hand") + best_hand = self.get_best_hand(detections, scores, 0.7, 0.5) + if best_hand is not None: + self.img = self.img[best_hand[0] - 30:best_hand[2] + 30, best_hand[1] - 30:best_hand[3] + 30] + + print('Attempting to use pure hand recognition') + self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV) + + # Need to shift red pixels so they can be 0-20 rather than 250-~20 + self.img_hsv = self.__shift_pixels(self.img_hsv, 30) + + self.img_hsv = self.__denoise(self.img_hsv) + self.__segment_image() + + print('calculating circle') + radius, centre = self.__calc_circle(self.mask) + print('Got circle') + + # Now go around the circle to calculate num of times going 0->255 or vice-versa. + # First just do it the naive way with loops. + # Equation of the circle: + # y = sqrt(r2 - (x-c)2) + c + prev_x = centre[0] - radius + prev_y = [self.__calc_pos_y(centre[0] - radius, radius, centre), self.__calc_pos_y(centre[0] - radius, radius, centre)] + num_change = 0 + + # Make sure x is also within bounds. + x_start = centre[0] - radius + 1 + if x_start < 0: + x_start = 0 + + x_end = centre[0] + radius + if x_end >= self.mask.shape[1]: + x_end = self.mask.shape[1] - 1 + # Could batch this function to execute on multiple cores? + for x in range(x_start, x_end): + # Need to check circle is inside the bounds. + ypos = self.__calc_pos_y(x, radius, centre) + # y above centre (ypos) and y below radius) + y = [ypos, centre[1] - (ypos-centre[1])] + + if y[0] < 0: + y[0] = 0 + if y[0] >= self.mask.shape[0]: + y[0] = self.mask.shape[0] - 1 + if y[1] < 0: + y[1] = 0 + if y[1] >= self.mask.shape[0]: + y[1] = self.mask.shape[0] - 1 + if(self.mask[y[0], x] != self.mask[prev_y[0], prev_x]): + num_change += 1 + if self.mask[y[1], x] != self.mask[prev_y[1], prev_x] and y[0] != y[1]: + num_change += 1 + prev_x = x + prev_y = y + + print('Finished calculating, returning') + print(num_change) + return int(num_change / 2 - 1) + + def calc_hand_batch(self, batch): + pass \ No newline at end of file