update to accept 48x48 eyes images

Author: GreenWizard2015

Core/CDataSampler_utils.py

@@ -74,6 +74,8 @@ def toTensor(data, params, userId, placeId, screenId):
   N = tf.shape(points)[0]
   imgA = tf.cast(imgA, tf.float32) / 255.
   imgB = tf.cast(imgB, tf.float32) / 255.
+  tf.assert_equal(tf.shape(imgA), (N, 48, 48))
+  tf.assert_equal(tf.shape(imgA), tf.shape(imgB))
   userId = tf.fill((N, 1), userId)
   placeId = tf.fill((N, 1), placeId)
   screenId = tf.fill((N, 1), screenId)
@@ -82,16 +84,45 @@ def toTensor(data, params, userId, placeId, screenId):
     x,
     tf.concat([(N // timesteps, timesteps), tf.shape(x)[1:]], axis=-1)
   )
+  # apply center crop
+  fraction = 32.0 / 48.0
+  pos = tf.constant(
+    [[0.5 - fraction / 2, 0.5 - fraction / 2, 0.5 + fraction / 2, 0.5 + fraction / 2]],
+    dtype=tf.float32
+  )
+  pos = tf.tile(pos, [N, 1])
+  withCrop = lambda x: tf.image.crop_and_resize(
+    x[..., None],
+    boxes=pos,
+    box_indices=tf.range(N), crop_size=(32, 32),
+  )[..., 0]
+
   clean = {
     'time': reshape(T),
     'points': reshape(points),
-    'left eye': reshape(imgA),
-    'right eye': reshape(imgB),
+    'left eye': reshape(withCrop(imgA)),
+    'right eye': reshape(withCrop(imgB)),
     'userId': reshape(userId),
     'placeId': reshape(placeId),
     'screenId': reshape(screenId),
   }
   ##########################
+  # random crop 32x32 eyes
+  fraction = 32.0 / 48.0
+  pos = tf.random.uniform((N, 2), minval=0.0, maxval=1.0 - fraction)
+  boxes = tf.concat([pos, pos + fraction], axis=-1)
+  tf.assert_equal(tf.shape(boxes), (N, 4))
+  imgA = tf.image.crop_and_resize(
+    imgA[..., None],
+    boxes=boxes,
+    box_indices=tf.range(N), crop_size=(32, 32),
+  )[..., 0]
+  imgB = tf.image.crop_and_resize(
+    imgB[..., None],
+    boxes=boxes,
+    box_indices=tf.range(N), crop_size=(32, 32),
+  )[..., 0]
+  ##########################
   def clip(x): return tf.clip_by_value(x, 0., 1.)
 
   def sampleBrightness(a, b, mid=1.0):

Core/CEyeTracker.py

@@ -92,7 +92,7 @@ def _extract(self, image, pts, isBGR):
     if len(pts) < 1: return EMPTY
 
     HW = np.array(image.shape[:2][::-1])
-    roi = self._circleROI(pts, padding=1.25)
+    roi = self._circleROI(pts, padding=1.5)
     if roi is None: return EMPTY
     A, B = roi
     A = A.clip(min=0, max=HW)
@@ -103,8 +103,11 @@ def _extract(self, image, pts, isBGR):
     if np.min(crop.shape[:2]) < 8:
       return np.zeros(sz, np.uint8), rect
 
-    crop = cv2.resize(crop, sz)
+    crop = cv2.resize(crop, (48, 48)) # 48x48, not 32x32
     crop = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY if isBGR else cv2.COLOR_RGB2GRAY)
+    # center crop 32x32
+    d = (48 - 32) // 2
+    crop = crop[d:d+32, d:d+32]
     return crop.astype(np.uint8), rect
 
   def _processFace(self, pose, image):

scripts/download-remote.py

@@ -27,8 +27,8 @@ def deserialize(buffer):
   offset = 0
   samples = []
   # read header (uint8)
-  version = np.frombuffer(buffer, dtype=np.uint8, count=1, offset=offset)
-  if version[0] != 1:
+  version = np.frombuffer(buffer, dtype=np.uint8, count=1, offset=offset)[0]
+  if not (version in [2]): # only version 2 is supported
     raise ValueError('Invalid version %d' % version[0])
   offset += 1
 
@@ -38,6 +38,8 @@ def deserialize(buffer):
   offset += 36
   screenId = buffer[offset:offset+36].decode('utf-8')
   offset += 36
+
+  EYE_SIZE = 32 if 1 == version else 48
   # read samples
   while offset < len(buffer):
     sample = {
@@ -51,15 +53,16 @@ def deserialize(buffer):
     sample['time'] = time_data[0]
     offset += 4
 
-    # Read leftEye (1024 uint8)
-    sample['leftEye'] = np.frombuffer(buffer, dtype=np.uint8, count=32*32, offset=offset) \
-      .reshape(32, 32)
-    offset += 32 * 32
+    # Read leftEye (uint8)
+    EYE_COUNT = EYE_SIZE * EYE_SIZE
+    sample['leftEye'] = np.frombuffer(buffer, dtype=np.uint8, count=EYE_COUNT, offset=offset) \
+      .reshape(EYE_SIZE, EYE_SIZE)
+    offset += EYE_COUNT
 
-    # Read rightEye (1024 uint8)
-    sample['rightEye'] = np.frombuffer(buffer, dtype=np.uint8, count=32*32, offset=offset) \
-      .reshape(32, 32)
-    offset += 32 * 32
+    # Read rightEye (uint8)
+    sample['rightEye'] = np.frombuffer(buffer, dtype=np.uint8, count=EYE_COUNT, offset=offset) \
+      .reshape(EYE_SIZE, EYE_SIZE)
+    offset += EYE_COUNT
 
     # Read points (float32)
     sample['points'] = np.frombuffer(buffer, dtype='>f4', count=2*478, offset=offset) \
@@ -88,6 +91,18 @@ def deserialize(buffer):
   # rename "leftEye" and "rightEye" to "left eye" and "right eye"
   res['left eye'] = res.pop('leftEye')
   res['right eye'] = res.pop('rightEye')
+  if 1 == version: # upscale to 48x48
+    import cv2
+    res['left eye'] = np.stack(
+      [cv2.resize(img[..., None], (48, 48)) for img in res['left eye']]
+    )
+    res['right eye'] = np.stack([
+      cv2.resize(img[..., None], (48, 48)) for img in res['right eye']
+    ])
+    pass
+  
+  assert res['left eye'].shape[1:] == (48, 48), 'Invalid shape for left eye. Got %s' % str(res['left eye'].shape)
+  assert res['right eye'].shape[1:] == (48, 48), 'Invalid shape for right eye. Got %s' % str(res['right eye'].shape)
   return res
 
 def find_free_name(folder, base_name, extension=".npz"):
@@ -158,15 +173,17 @@ def main(args):
   shutil.rmtree(os.path.join(folder, 'remote'), ignore_errors=True)
   # get the list of files from the remote server
   urls = requests.get(args.url).json()
-  print('Found %d files on the remote server' % len(urls))
-  for file in urls:
+  N = len(urls)
+  L = len(str(N))
+  print('Found %d files on the remote server' % N)
+  for i, file in enumerate(urls):
     response = requests.get(file)
     content = IO.BytesIO(response.content)
     # read first file in the gz archive
     with gzip.open(content, 'rb') as f:
       first_file = f.read()
       samples = deserialize(first_file)
-      print('Read %d samples from %s' % (len(samples['time']), file))
+      print(f'[{i:0{L}d}/{N:0{L}d}] Read {len(samples["time"])} samples from {file}')
 
       # don't want to messing up with such cases
       userId = np.unique(samples['userId'])
@@ -182,7 +199,6 @@ def main(args):
         saveChunk(chunk, os.path.join(folder, 'remote'))
         continue
       pass
-
   return
 
 if __name__ == '__main__':