diff --git a/GestureRecognition/SimpleHandRecogniser.py b/GestureRecognition/SimpleHandRecogniser.py index d8fd5ad..35039f4 100644 --- a/GestureRecognition/SimpleHandRecogniser.py +++ b/GestureRecognition/SimpleHandRecogniser.py @@ -1,7 +1,8 @@ from GestureRecognition.handrecogniser import HandRecogniser import numpy as np import cv2 -import tensorflow as tf +# import tensorflow as tf +import multiprocessing as mp class SimpleHandRecogniser(HandRecogniser): def __init__(self, frame): @@ -95,25 +96,25 @@ class SimpleHandRecogniser(HandRecogniser): # Taken the code straight from his example, as it works perfectly. This is specifically # from the load_inference_graph method that he wrote, and will load the graph into # memory if one has not already been loaded for this object. - def load_inference_graph(self): - """Loads a tensorflow model checkpoint into memory""" + # def load_inference_graph(self): + # """Loads a tensorflow model checkpoint into memory""" - if self.graph != None and self.sess != None: - # Don't load more than once, to save time... - return + # if self.graph != None and self.sess != None: + # # Don't load more than once, to save time... + # return - PATH_TO_CKPT = '/Users/piv/Documents/Projects/car/GestureRecognition/frozen_inference_graph.pb' - # load frozen tensorflow model into memory - detection_graph = tf.Graph() - with detection_graph.as_default(): - od_graph_def = tf.GraphDef() - with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: - serialized_graph = fid.read() - od_graph_def.ParseFromString(serialized_graph) - tf.import_graph_def(od_graph_def, name='') - sess = tf.Session(graph=detection_graph) - self.graph = detection_graph - self.sess = sess + # PATH_TO_CKPT = '/Users/piv/Documents/Projects/car/GestureRecognition/frozen_inference_graph.pb' + # # load frozen tensorflow model into memory + # detection_graph = tf.Graph() + # with detection_graph.as_default(): + # od_graph_def = tf.GraphDef() + # with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid: + # serialized_graph = fid.read() + # od_graph_def.ParseFromString(serialized_graph) + # tf.import_graph_def(od_graph_def, name='') + # sess = tf.Session(graph=detection_graph) + # self.graph = detection_graph + # self.sess = sess # Source: Victor Dibia @@ -121,64 +122,73 @@ class SimpleHandRecogniser(HandRecogniser): # Taken the code straight from his example, as it works perfectly. This is specifically # from the detect_hand method that he wrote, as other processing is required for the # hand recognition to work correctly. - def detect_hand_tensorflow(self, detection_graph, sess): - """ Detects hands in a frame using a CNN + # def detect_hand_tensorflow(self, detection_graph, sess): + # """ Detects hands in a frame using a CNN - detection_graph -- The CNN to use to detect the hand. - sess -- THe tensorflow session for the given graph - """ - - image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') - - detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0') - - detection_scores = detection_graph.get_tensor_by_name('detection_scores:0') - - detection_classes = detection_graph.get_tensor_by_name('detection_classes:0') - - num_detections = detection_graph.get_tensor_by_name('num_detections:0') - - img_expanded = np.expand_dims(self.img, axis=0) - - (boxes, scores, classes, num) = sess.run( - [detection_boxes, detection_scores, detection_classes, num_detections], - feed_dict={image_tensor: img_expanded}) - print('finished detection') - return np.squeeze(boxes), np.squeeze(scores) - - # def detect_hand_opencv(self, detection_graph, sess): - # """Performs hand detection using a CNN from tensorflow using opencv. - # detection_graph -- The CNN to use to detect the hand. # sess -- THe tensorflow session for the given graph # """ - # if self.img is None: - # return - # height = self.img.shape[0] - # width = self.img.shape[1] + # image_tensor = detection_graph.get_tensor_by_name('image_tensor:0') - # scale = 0.5 + # detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0') - # classes = None + # detection_scores = detection_graph.get_tensor_by_name('detection_scores:0') - # net = cv2.dnn.readNetFromTensorflow(detection_graph, sess) + # detection_classes = detection_graph.get_tensor_by_name('detection_classes:0') - # # width is scaled weirdly to ensure we keep tbe same ratio as the original image. - # net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False)) - # netOut = net.forward() + # num_detections = detection_graph.get_tensor_by_name('num_detections:0') - # # Format output to look same as tensorflow output. - # scores = [] - # boxes = [] + # img_expanded = np.expand_dims(self.img, axis=0) - # for out in netOut: - # for detection in out[0,0]: - # scores.append(detection[2]) - # boxes.append(detection[3], detection[4], detection[5], detection[6]) - # # Only doing first class as only trying to find the hand. - # break - # return np.array(boxes), np.array(scores) + # (boxes, scores, classes, num) = sess.run( + # [detection_boxes, detection_scores, detection_classes, num_detections], + # feed_dict={image_tensor: img_expanded}) + # print('finished detection') + # return np.squeeze(boxes), np.squeeze(scores) + + def load_cv_net(self, graph_path, names_path): + """Loads a tensorflow neural object detection network using openCV + + Arguments + graph_path: Path to the tensorflow frozen inference graph (something.pb) + names_path: Path to the tensorflow (something.pbtext) file. + """ + self.net = cv2.dnn.readNetFromTensorflow(graph_path, names_path) + + def detect_hand_opencv(self, detection_graph, sess): + """Performs hand detection using a CNN from tensorflow using opencv. + + detection_graph -- The CNN to use to detect the hand. + sess -- THe tensorflow session for the given graph + """ + if self.img is None: + return + + height = self.img.shape[0] + width = self.img.shape[1] + + scale = 0.5 + + classes = None + + net = cv2.dnn.readNetFromTensorflow(detection_graph, sess) + + # width is scaled weirdly to ensure we keep tbe same ratio as the original image. + net.setInput(cv2.dnn.blobFromImage(self.img, scale, size=(300, 300 * (width/height)), swapRB=True, crop=False)) + netOut = net.forward() + + # Format output to look same as tensorflow output. + scores = [] + boxes = [] + + for out in netOut: + for detection in out[0,0]: + scores.append(detection[2]) + boxes.append(detection[3], detection[4], detection[5], detection[6]) + # Only doing first class as only trying to find the hand. + break + return np.array(boxes), np.array(scores) def get_best_hand(self, boxes, scores, conf_thresh, nms_thresh): """ @@ -236,14 +246,14 @@ class SimpleHandRecogniser(HandRecogniser): print('There is no image') return -1 # First cut out the frame using the neural network. - self.load_inference_graph() - print("loaded inference graph") - detections, scores = self.detect_hand_tensorflow(self.graph, self.sess) + # self.load_inference_graph() + # print("loaded inference graph") + # detections, scores = self.detect_hand_tensorflow(self.graph, self.sess) - print("Getting best hand") - best_hand = self.get_best_hand(detections, scores, 0.7, 0.5) - if best_hand is not None: - self.img = self.img[best_hand[0] - 30:best_hand[2] + 30, best_hand[1] - 30:best_hand[3] + 30] + # print("Getting best hand") + # best_hand = self.get_best_hand(detections, scores, 0.7, 0.5) + # if best_hand is not None: + # self.img = self.img[best_hand[0] - 30:best_hand[2] + 30, best_hand[1] - 30:best_hand[3] + 30] print('Attempting to use pure hand recognition') self.img_hsv = cv2.cvtColor(self.img, cv2.COLOR_BGR2HSV) @@ -275,6 +285,13 @@ class SimpleHandRecogniser(HandRecogniser): if x_end >= self.mask.shape[1]: x_end = self.mask.shape[1] - 1 # Could batch this function to execute on multiple cores? + # Calc num CPUS. + # num_cores = mp.cpu_count() + # # Calc batch size: + # batch_size = x_end // num_cores + # for b in range(0, num_cores - 1): + # pass + for x in range(x_start, x_end): # Need to check circle is inside the bounds. ypos = self.__calc_pos_y(x, radius, centre)