Opencv实现眼睛控制鼠标的实践

# For monitoring WEB camera and perfORMing image minipulations
import cv2
# For performing array operations
import numpy as np
# For creating and removing directories
import os
import shutil
# For recognizing and performing actions on mouse presses
from pynput.mouse import Listener

from pynput.mouse import Listener
def on_click(x, y, button, pressed):
"""
  Args:
    x: the x-coordinate of the mouse
    y: the y-coordinate of the mouse
    button: 1 or 0, depending on right-click or left-click
    pressed: 1 or 0, whether the mouse was pressed or released
  """
if pressed:
print (x, y)
with Listener(on_click = on_click) as listener:
  listener.join()

import cv2
# Load the cascade classifier detection object
cascade = cv2.CascadeClassifier("haarcascade_eye.xml")
# Turn on the web camera
video_capture = cv2.VideoCapture(0)
# Read data from the web camera (get the frame)
_, frame = video_capture.read()
# Convert the image to grayscale
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Predict the bounding box of the eyes
boxes = cascade.detectMultiScale(gray, 1.3, 10)
# Filter out images taken from a bad angle with errors
# We want to make sure both eyes were detected, and nothing else
if len(boxes) == 2:
eyes = []
for box in boxes:
    # Get the rectangle parameters for the detected eye
x, y, w, h = box
    # Crop the bounding box from the frame
eye = frame[y:y + h, x:x + w]
    # Resize the crop to 32x32
eye = cv2.resize(eye, (32, 32))
    # Normalize
eye = (eye - eye.min()) / (eye.max() - eye.min())
    # Further crop to just around the eyeball
eye = eye[10:-10, 5:-5]
    # Scale between [0, 255] and convert to int datatype
eye = (eye * 255).astype(np.uint8)
    # Add the current eye to the list of 2 eyes
eyes.append(eye)
  # Concatenate the two eye images into one
eyes = np.hstack(eyes)

def normalize(x):
  minn, maxx = x.min(), x.max()
return (x - minn) / (maxx - minn)

def scan(image_size=(32, 32)):
_, frame = video_capture.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
boxes = cascade.detectMultiScale(gray, 1.3, 10)
if len(boxes) == 2:
eyes = []
for box in boxes:
x, y, w, h = box
eye = frame[y:y + h, x:x + w]
eye = cv2.resize(eye, image_size)
eye = normalize(eye)
eye = eye[10:-10, 5:-5]
eyes.append(eye)
return (np.hstack(eyes) * 255).astype(np.uint8)
else:
return None

def on_click(x, y, button, pressed):
# If the action was a mouse PRESS (not a RELEASE)
if pressed:
# Crop the eyes
    eyes = scan()
# If the function returned None, something went wrong
if not eyes is None:
# Save the image
      filename = root + "{} {} {}.jpeg".format(x, y, button)
      cv2.imwrite(filename, eyes)

import cv2
import numpy as np
import os
import shutil
from pynput.mouse import Listener
 
 
root = input("Enter the directory to store the images: ")
if os.path.isdir(root):
  resp = ""
while not resp in ["Y", "N"]:
    resp = input("This directory already exists. If you continue, the contents of the existing directory will be deleted. If you would still like to proceed, enter [Y]. Otherwise, enter [N]: ")
if resp == "Y": 
    shutil.rmtree(root)
else:
    exit()
os.mkdir(root)
 
 
# Normalization helper function
def normalize(x):
  minn, maxx = x.min(), x.max()
return (x - minn) / (maxx - minn)
 
 
# Eye cropping function
def scan(image_size=(32, 32)):
  _, frame = video_capture.read()
  gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
  boxes = cascade.detectMultiScale(gray, 1.3, 10)
if len(boxes) == 2:
    eyes = []
for box in boxes:
      x, y, w, h = box
      eye = frame[y:y + h, x:x + w]
      eye = cv2.resize(eye, image_size)
      eye = normalize(eye)
      eye = eye[10:-10, 5:-5]
      eyes.append(eye)
return (np.hstack(eyes) * 255).astype(np.uint8)
else:
return None
 
 
def on_click(x, y, button, pressed):
# If the action was a mouse PRESS (not a RELEASE)
if pressed:
# Crop the eyes
    eyes = scan()
# If the function returned None, something went wrong
if not eyes is None:
# Save the image
      filename = root + "{} {} {}.jpeg".format(x, y, button)
      cv2.imwrite(filename, eyes)
 
 
cascade = cv2.CascadeClassifier("haarcascade_eye.xml")
video_capture = cv2.VideoCapture(0)
 
 
with Listener(on_click = on_click) as listener:
  listener.join()

import numpy as np
import os
import cv2
import pyautogui
from Tensorflow.keras.models import *
from tensorflow.keras.layers import *
from tensorflow.keras.optimizers import *

cascade = cv2.CascadeClassifier("haarcascade_eye.xml")
video_capture = cv2.VideoCapture(0)

def normalize(x):
  minn, maxx = x.min(), x.max()
return (x - minn) / (maxx - minn)

def scan(image_size=(32, 32)):
_, frame = video_capture.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
boxes = cascade.detectMultiScale(gray, 1.3, 10)
if len(boxes) == 2:
eyes = []
for box in boxes:
x, y, w, h = box
eye = frame[y:y + h, x:x + w]
eye = cv2.resize(eye, image_size)
eye = normalize(eye)
eye = eye[10:-10, 5:-5]
eyes.append(eye)
return (np.hstack(eyes) * 255).astype(np.uint8)
else:
return None

# Note that there are actually 2560x1440 pixels on my screen
# I am simply recording one less, so that when we divide by these
# numbers, we will normalize between 0 and 1. Note that mouse
# coordinates are reported starting at (0, 0), not (1, 1)
width, height = 2559, 1439

filepaths = os.listdir(root)
X, Y = [], []
for filepath in filepaths:
x, y, _ = filepath.split(' ')
x = float(x) / width
y = float(y) / height
X.append(cv2.imread(root + filepath))
Y.append([x, y])
X = np.array(X) / 255.0
Y = np.array(Y)
print (X.shape, Y.shape)

model = Sequential()
model.add(Conv2D(32, 3, 2, activation = 'relu', input_shape = (12, 44, 3)))
model.add(Conv2D(64, 2, 2, activation = 'relu'))
model.add(Flatten())
model.add(Dense(32, activation = 'relu'))
model.add(Dense(2, activation = 'sigmoid'))
model.compile(optimizer = "adam", loss = "mean_squared_error")
model.summary()

epochs = 200
for epoch in range(epochs):
  model.fit(X, Y, batch_size = 32)

while True:
  eyes = scan()
if not eyes is None:
      eyes = np.expand_dims(eyes / 255.0, axis = 0)
      x, y = model.predict(eyes)[0]
      pyautogui.moveTo(x * width, y * height)