275 lines
9.6 KiB
Python
275 lines
9.6 KiB
Python
import cv2
|
|
import glob
|
|
import os
|
|
from datetime import datetime
|
|
from typing import Tuple
|
|
|
|
import numpy as np
|
|
from sklearn import cluster, metrics, svm
|
|
from sklearn.externals import joblib
|
|
from sklearn.pipeline import make_pipeline
|
|
from sklearn.preprocessing import StandardScaler
|
|
from sklearn import neural_network
|
|
|
|
from util import RANK, POSITION, imwrite, PIECE, COLOR, Squares, OUR_PIECES
|
|
|
|
|
|
def generate_centers(number_of_clusters, sift: cv2.xfeatures2d_SIFT):
|
|
features = None
|
|
for piece in OUR_PIECES:
|
|
for color in COLOR:
|
|
for filename in glob.glob(f"training_images/{piece}/{color}_square/*.png"):
|
|
image = cv2.imread(filename)
|
|
#image = selective_search(image, use_fast=True)
|
|
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
|
kp, desc = sift.detectAndCompute(gray, None)
|
|
print(f"{piece}, {color}, {filename}")
|
|
|
|
if features is None:
|
|
features = np.array(desc)
|
|
else:
|
|
print(f"{piece}, {color}, {filename}")
|
|
features = np.vstack((features, desc))
|
|
|
|
features = np.array(features)
|
|
k_means = cluster.KMeans(number_of_clusters)
|
|
k_means.fit(features)
|
|
return k_means.cluster_centers_
|
|
|
|
|
|
def generate_bag_of_words(image, centers, sift: cv2.xfeatures2d_SIFT):
|
|
num_centers = centers.shape[0]
|
|
histogram = np.zeros((1, num_centers))
|
|
|
|
gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
|
|
kp, desc = sift.detectAndCompute(gray_image, None)
|
|
|
|
if not kp:
|
|
return histogram
|
|
|
|
distances = metrics.pairwise.pairwise_distances(desc, centers)
|
|
best_centers = np.argmin(distances, axis=1)
|
|
|
|
for i in best_centers: # TODO: Could do this way faster in one line with numpy somehow
|
|
histogram[0, i] += + 1
|
|
|
|
return histogram / np.sum(histogram)
|
|
|
|
|
|
def do_pre_processing() -> None:
|
|
sift = cv2.xfeatures2d.SIFT_create()
|
|
centers = generate_centers(8, sift)
|
|
|
|
np.save("training_data/centers", centers)
|
|
|
|
for piece in OUR_PIECES:
|
|
for color in COLOR:
|
|
for filename in glob.glob(f"training_images/{piece}/{color}_square/*.png"):
|
|
image = cv2.imread(filename)
|
|
#image = selective_search(image, image_name=filename, use_fast=True)
|
|
bow_features = generate_bag_of_words(image, centers, sift)
|
|
np.save(f"training_data/{piece}/{color}_square/{os.path.basename(filename)}", bow_features)
|
|
|
|
|
|
def load_training_data(piece: PIECE, color: COLOR) -> Tuple[np.array, np.array]:
|
|
X = []
|
|
Y = []
|
|
for p in OUR_PIECES:
|
|
for filename in glob.glob(f"training_data/{piece}/{color}_square/*.npy"):
|
|
data = np.load(filename)
|
|
X.append(data[0])
|
|
Y.append(p == piece)
|
|
return np.array(X), np.array(Y)
|
|
|
|
|
|
def train_empty_or_piece_hist() -> None:
|
|
for square_color in COLOR:
|
|
X = []
|
|
Y = []
|
|
for piece in OUR_PIECES + (PIECE.EMPTY,):
|
|
for filename in glob.glob(f"training_images/{piece}/{square_color}_square/*.png"):
|
|
img = cv2.imread(filename)
|
|
y, x = np.histogram(img.ravel(), bins=32, range=[0, 256])
|
|
X.append(y)
|
|
Y.append(piece == PIECE.EMPTY)
|
|
|
|
classifier = make_pipeline(StandardScaler(),
|
|
svm.SVC(C=10.0, gamma=0.01, probability=True))
|
|
classifier.fit(np.array(X), np.array(Y))
|
|
joblib.dump(classifier, f"classifiers/classifier_empty/white_piece_on_{square_color}_square.pkl")
|
|
|
|
|
|
def train_pieces_svm() -> None:
|
|
for piece in OUR_PIECES:
|
|
for color in COLOR:
|
|
total_weights = len(glob.glob(f"training_images/{piece}/{color}_square/*.png"))
|
|
|
|
for piece in OUR_PIECES:
|
|
for color in COLOR:
|
|
current_weight = len(glob.glob(f"training_images/{piece}/{color}_square/*.png"))
|
|
print(f"Training for piece: {piece}")
|
|
X, Y = load_training_data(piece, color)
|
|
classifier = svm.SVC(C=10, gamma=0.01, class_weight={0: 15, 1: 0.8}, probability=True)
|
|
classifier.fit(X, Y)
|
|
joblib.dump(classifier, f"classifiers/classifier_{piece}/{color}.pkl")
|
|
|
|
|
|
def train_pieces_svm_canny() -> None:
|
|
for square_color in COLOR:
|
|
X = []
|
|
Y = []
|
|
for piece in OUR_PIECES + (PIECE.EMPTY,):
|
|
for filename in glob.glob(f"training_images/{piece}/{square_color}_square/*.png"):
|
|
img = cv2.imread(filename)
|
|
y, x = np.histogram(img.ravel(), bins=32, range=[0, 256])
|
|
X.append(y)
|
|
Y.append(piece == PIECE.EMPTY)
|
|
|
|
classifier = make_pipeline(StandardScaler(),
|
|
svm.SVC(C=10.0, gamma=0.01, probability=True))
|
|
classifier.fit(np.array(X), np.array(Y))
|
|
joblib.dump(classifier, f"classifiers/classifier_empty/white_piece_on_{square_color}_square.pkl")
|
|
|
|
|
|
def warp_board(camera_image, debug_image=None, dst_points: list = None) -> np.ndarray:
|
|
baseline = cv2.imread("new_baseline_board.png")
|
|
|
|
camera_image_gray = cv2.cvtColor(camera_image, cv2.COLOR_BGR2GRAY)
|
|
baseline_gray = cv2.cvtColor(baseline, cv2.COLOR_BGR2GRAY)
|
|
|
|
sift = cv2.xfeatures2d.SIFT_create()
|
|
camera_image_keypoints = sift.detect(camera_image_gray, None)
|
|
baseline_keypoints = sift.detect(baseline_gray, None)
|
|
|
|
camera_image_keypoints, des = sift.compute(camera_image_gray, camera_image_keypoints)
|
|
baseline_keypoints, des2 = sift.compute(baseline_gray, baseline_keypoints)
|
|
|
|
if debug_image is not None:
|
|
cv2.drawKeypoints(camera_image, keypoints=camera_image_keypoints, outImage=debug_image)
|
|
cv2.imwrite("keypoints_img.jpg", camera_image)
|
|
|
|
# FLANN parameters
|
|
FLANN_INDEX_KDTREE = 0
|
|
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=8)
|
|
search_params = dict(checks=100) # or pass empty dictionary
|
|
|
|
flann = cv2.FlannBasedMatcher(index_params,search_params)
|
|
matches = flann.knnMatch(des, des2, k=2)
|
|
|
|
# Need to draw only good matches, so create a mask
|
|
matchesMask = [[0, 0] for _ in range(len(matches))]
|
|
|
|
# Ratio test as per Lowe's paper
|
|
good_matches = []
|
|
for i, (m, n) in enumerate(matches):
|
|
if m.distance < 0.55*n.distance:
|
|
matchesMask[i] = [1, 0]
|
|
good_matches.append([m, n])
|
|
|
|
img3 = cv2.drawMatchesKnn(
|
|
camera_image,
|
|
camera_image_keypoints,
|
|
baseline,
|
|
baseline_keypoints,
|
|
matches,
|
|
None,
|
|
matchColor=(0, 255, 0),
|
|
singlePointColor=(255, 0, 0),
|
|
matchesMask=matchesMask,
|
|
flags=0
|
|
)
|
|
cv2.imwrite("matches.jpg", img3)
|
|
|
|
# Extract location of good matches
|
|
points1 = np.zeros((len(good_matches), 2), dtype=np.float32)
|
|
points2 = np.zeros((len(good_matches), 2), dtype=np.float32)
|
|
|
|
for i, (m, n) in enumerate(good_matches):
|
|
points1[i, :] = camera_image_keypoints[m.queryIdx].pt
|
|
points2[i, :] = baseline_keypoints[m.trainIdx].pt
|
|
|
|
if dst_points is not None:
|
|
dst_points.extend(zip(points1, points2))
|
|
|
|
h, mask = cv2.findHomography(points1, points2, cv2.RANSAC)
|
|
|
|
height, width, channels = baseline.shape
|
|
return cv2.warpPerspective(camera_image, h, (width, height))
|
|
|
|
|
|
def get_square(warped_board: np.ndarray, position: POSITION) -> np.ndarray:
|
|
width, _, _ = warped_board.shape # board is square anyway
|
|
side = int(width * 0.045)
|
|
size = width - 2 * side
|
|
square_size = size // 8
|
|
padding = 2
|
|
|
|
x1 = side + (square_size * (position.file - 1))
|
|
x2 = x1 + square_size + padding
|
|
y1 = max(0, side + (square_size * (8 - position.rank)) - padding) # 8 - rank because chessboard is from 8 to 1
|
|
|
|
y2 = min(width, y1 + square_size + padding)
|
|
|
|
square = warped_board[y1:y2, x1:x2]
|
|
return square
|
|
|
|
|
|
def get_squares(warped_board: np.ndarray) -> Squares:
|
|
# cv2.imwrite(f"warped_square_{square}.png", square)
|
|
return {position: get_square(warped_board, position)
|
|
for position in POSITION}
|
|
|
|
|
|
def save_empty_fields(warped_board: np.ndarray, skip_rank: RANK = None) -> None:
|
|
for position in POSITION:
|
|
if position.rank == skip_rank:
|
|
continue
|
|
square = get_square(warped_board, position)
|
|
imwrite(f"training_images/empty/{position.color}_square/training_{position}_{datetime.utcnow().timestamp()}.png", square)
|
|
|
|
|
|
def load_data_nn(spec_piece, color):
|
|
X = None
|
|
Y = None
|
|
for piece in OUR_PIECES:
|
|
piece_class = int(spec_piece == piece)
|
|
|
|
for filename in glob.glob(f"training_images/{piece}/{color}_square/*.png"):
|
|
image = cv2.imread(filename)
|
|
data = np.reshape(image, (1, np.product(image.shape)))
|
|
if X is None:
|
|
if piece_class == 1:
|
|
for _ in range(10):
|
|
X = np.array(data)
|
|
Y = np.array([piece_class])
|
|
else:
|
|
for _ in range(5):
|
|
X = np.array(data)
|
|
Y = np.array([piece_class])
|
|
else:
|
|
if piece_class == 1:
|
|
for _ in range(10):
|
|
X = np.vstack((X, data))
|
|
Y = np.vstack((Y, [piece_class]))
|
|
else:
|
|
for _ in range(5):
|
|
X = np.vstack((X, data))
|
|
Y = np.vstack((Y, [piece_class]))
|
|
return X, Y
|
|
|
|
|
|
def train_nn():
|
|
for piece in OUR_PIECES:
|
|
for color in COLOR:
|
|
X, Y = load_data_nn(piece, color)
|
|
|
|
classifier = neural_network.MLPClassifier(hidden_layer_sizes=256)
|
|
|
|
classifier.fit(X, Y)
|
|
|
|
joblib.dump(classifier, f"classifiers/neural_net_{piece}/{color}.pkl")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
train_nn()
|