From 99783ee4f823e37011e87e9cdda6ff7f15b5a26f Mon Sep 17 00:00:00 2001 From: Alexander Munch-Hansen Date: Tue, 20 Mar 2018 13:03:21 +0100 Subject: [PATCH] clean up and move things to network.py --- bot.py | 14 ++----- game.py | 55 ------------------------- main.py | 10 ++--- network.py | 115 ++++++++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 113 insertions(+), 81 deletions(-) diff --git a/bot.py b/bot.py index efcd101..297f203 100644 --- a/bot.py +++ b/bot.py @@ -12,16 +12,9 @@ class Bot: self.cup = Cup() self.sym = sym self.graph = tf.Graph() - with self.graph.as_default(): - self.session = tf.Session() - self.network = Network(self.session, config, name) - self.network.restore_model() - variables_names = [v.name for v in tf.trainable_variables()] - values = self.session.run(variables_names) - for k, v in zip(variables_names, values): - print("Variable: ", k) - print("Shape: ", v.shape) - print(v) + + self.network = Network(config, name) + self.network.restore_model() def restore_model(self): with self.graph.as_default(): @@ -36,6 +29,7 @@ class Bot: def get_network(self): return self.network + # TODO: DEPRECATE def make_move(self, board, sym, roll): # print(Board.pretty(board)) legal_moves = Board.calculate_legal_states(board, sym, roll) diff --git a/game.py b/game.py index 086caf9..b21d047 100644 --- a/game.py +++ b/game.py @@ -83,61 +83,6 @@ class Game: print(Board.outcome(self.board)) - def train_model(self, episodes=1000, save_step_size = 100, trained_eps = 0): - start_time = time.time() - - def print_time_estimate(eps_completed): - cur_time = time.time() - time_diff = cur_time - start_time - eps_per_sec = eps_completed / time_diff - secs_per_ep = time_diff / eps_completed - eps_remaining = (episodes - eps_completed) - sys.stderr.write("[TRAIN] Averaging {per_sec} episodes per second\n".format(per_sec = round(eps_per_sec, 2))) - sys.stderr.write("[TRAIN] {eps_remaining} episodes remaining; approx. {time_remaining} seconds remaining\n".format(eps_remaining = eps_remaining, time_remaining = int(eps_remaining * secs_per_ep))) - - - sys.stderr.write("[TRAIN] Training {} episodes and save_step_size {}\n".format(episodes, save_step_size)) - outcomes = [] - for episode in range(1, episodes + 1): - sys.stderr.write("[TRAIN] Episode {}".format(episode + trained_eps)) - self.board = Board.initial_state - - prev_board, prev_board_value = self.best_move_and_score() - # find the best move here, make this move, then change turn as the - # first thing inside of the while loop and then call - # best_move_and_score to get V_t+1 - - while Board.outcome(self.board) is None: - self.next_round() - cur_board, cur_board_value = self.best_move_and_score() - self.p1.get_network().train(prev_board, cur_board_value) - prev_board = cur_board - - # print("-"*30) - # print(Board.pretty(self.board)) - # print("/"*30) - sys.stderr.write("\t outcome {}".format(Board.outcome(self.board)[1])) - outcomes.append(Board.outcome(self.board)[1]) - final_score = np.array([ Board.outcome(self.board)[1] ]).reshape((1, 1)) - self.p1.get_network().train(prev_board, final_score) - - sys.stderr.write("\n") - - if episode % min(save_step_size, episodes) == 0: - sys.stderr.write("[TRAIN] Saving model...\n") - self.p1.get_network().save_model(episode+trained_eps) - sys.stderr.write("[TRAIN] Loading model for training opponent...\n") - self.p2.restore_model() - - if episode % 50 == 0: - print_time_estimate(episode) - - sys.stderr.write("[TRAIN] Saving model for final episode...\n") - self.p1.get_network().save_model(episode+trained_eps) - self.p2.restore_model() - - return outcomes - def next_round_test(self): print(self.board) print() diff --git a/main.py b/main.py index cdf2feb..5213958 100644 --- a/main.py +++ b/main.py @@ -65,6 +65,7 @@ parser.add_argument('--list-models', action='store_true', args = parser.parse_args() config = { + 'model': args.model, 'model_path': os.path.join(model_storage_path, args.model), 'episode_count': args.episode_count, 'eval_methods': args.eval_methods, @@ -86,10 +87,8 @@ if not os.path.isdir(log_path): os.mkdir(log_path) -# Set up game -import game -g = game.Game(config = config) -g.set_up_bots() +# Set up network +from network import Network # Set up variables @@ -111,9 +110,10 @@ if args.list_models: sys.stderr.write(" {name}: {eps_trained}\n".format(name = model[0], eps_trained = model[1])) elif args.train: + network = Network(config, config['model']) eps = config['start_episode'] while True: - train_outcome = g.train_model(episodes = episode_count, trained_eps = eps) + train_outcome = network.train_model(episodes = episode_count, trained_eps = eps) eps += episode_count log_train_outcome(train_outcome, trained_eps = eps) if config['eval_after_train']: diff --git a/network.py b/network.py index bb4c22a..30b54b8 100644 --- a/network.py +++ b/network.py @@ -3,7 +3,10 @@ from cup import Cup import numpy as np from board import Board import os - +import time +import sys +import random + class Network: hidden_size = 40 input_size = 26 @@ -17,17 +20,11 @@ class Network: def custom_tanh(self, x, name=None): return tf.scalar_mul(tf.constant(2.00), tf.tanh(x, name)) - def __init__(self, session, config, name): + def __init__(self, config, name): self.config = config - self.session = session + self.session = tf.Session() self.checkpoint_path = config['model_path'] self.name = name - - # Restore trained episode count for model - episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained") - if os.path.isfile(episode_count_path): - with open(episode_count_path, 'r') as f: - self.config['start_episode'] = int(f.read()) # input = x self.x = tf.placeholder('float', [1, Network.input_size], name='x') @@ -52,6 +49,8 @@ class Network: # tf.reduce_sum basically finds the sum of its input, so this gives the # difference between the two values, in case they should be lists, which # they might be if our input changes + + # TODO: Alexander thinks that self.value will be computed twice (instead of once) difference_in_values = tf.reduce_sum(self.value_next - self.value, name='difference') trainable_vars = tf.trainable_variables() @@ -71,6 +70,8 @@ class Network: self.saver = tf.train.Saver(max_to_keep=1) self.session.run(tf.global_variables_initializer()) + self.restore_model() + def eval_state(self, state): # Run state through a network @@ -122,12 +123,25 @@ class Network: print("[NETWK] ({name}) Restoring model from:".format(name = self.name), str(latest_checkpoint)) self.saver.restore(self.session, latest_checkpoint) + variables_names = [v.name for v in tf.trainable_variables()] + values = self.session.run(variables_names) + for k, v in zip(variables_names, values): + print("Variable: ", k) + print("Shape: ", v.shape) + print(v) + + # Restore trained episode count for model + episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained") + if os.path.isfile(episode_count_path): + with open(episode_count_path, 'r') as f: + self.config['start_episode'] = int(f.read()) # Have a circular dependency, #fuck, need to rewrite something - def train(self, board, v_next): + def adjust_weights(self, board, v_next): # print("lol") board = np.array(board).reshape((1,26)) - self.session.run(self.training_op, feed_dict = {self.x:board, self.value_next: v_next}) + self.session.run(self.training_op, feed_dict = { self.x: board, + self.value_next: v_next }) # while game isn't done: @@ -138,6 +152,85 @@ class Network: + def make_move(self, board, roll): + # print(Board.pretty(board)) + legal_moves = Board.calculate_legal_states(board, 1, roll) + moves_and_scores = [ (move, self.eval_state(np.array(move).reshape(1,26))) for move in legal_moves ] + scores = [ x[1] for x in moves_and_scores ] + best_score_index = np.array(scores).argmax() + best_move_pair = moves_and_scores[best_score_index] + #print("Found the best state, being:", np.array(move_scores).argmax()) + return best_move_pair + + + def train_model(self, episodes=1000, save_step_size = 100, trained_eps = 0): + start_time = time.time() + + def print_time_estimate(eps_completed): + cur_time = time.time() + time_diff = cur_time - start_time + eps_per_sec = eps_completed / time_diff + secs_per_ep = time_diff / eps_completed + eps_remaining = (episodes - eps_completed) + sys.stderr.write("[TRAIN] Averaging {per_sec} episodes per second\n".format(per_sec = round(eps_per_sec, 2))) + sys.stderr.write("[TRAIN] {eps_remaining} episodes remaining; approx. {time_remaining} seconds remaining\n".format(eps_remaining = eps_remaining, time_remaining = int(eps_remaining * secs_per_ep))) + + + sys.stderr.write("[TRAIN] Training {} episodes and save_step_size {}\n".format(episodes, save_step_size)) + outcomes = [] + for episode in range(1, episodes + 1): + sys.stderr.write("[TRAIN] Episode {}".format(episode + trained_eps)) + # TODO decide which player should be here + player = 1 + + roll = (random.randrange(1,7), random.randrange(1,7)) + prev_board, _ = self.make_move(Board.flip(Board.initial_state) if player == -1 else Board.initial_state, roll) + if player == -1: + prev_board = Board.flip(prev_board) + + # find the best move here, make this move, then change turn as the + # first thing inside of the while loop and then call + # best_move_and_score to get V_t+1 + + # i = 0 + while Board.outcome(prev_board) is None: + # print("-"*30) + # print(i) + # print(roll) + # print(Board.pretty(prev_board)) + # print("/"*30) + # i += 1 + + player *= -1 + roll = (random.randrange(1,7), random.randrange(1,7)) + + cur_board, cur_board_value = self.make_move(Board.flip(prev_board) if player == -1 else prev_board, roll) + if player == -1: + cur_board = Board.flip(cur_board) + + self.adjust_weights(prev_board, cur_board_value) + + prev_board = cur_board + + final_board = prev_board + sys.stderr.write("\t outcome {}".format(Board.outcome(final_board)[1])) + outcomes.append(Board.outcome(final_board)[1]) + final_score = np.array([ Board.outcome(final_board)[1] ]) + self.adjust_weights(prev_board, final_score.reshape((1, 1))) + + sys.stderr.write("\n") + + if episode % min(save_step_size, episodes) == 0: + sys.stderr.write("[TRAIN] Saving model...\n") + self.save_model(episode+trained_eps) + + if episode % 50 == 0: + print_time_estimate(episode) + + sys.stderr.write("[TRAIN] Saving model for final episode...\n") + self.save_model(episode+trained_eps) + + return outcomes # take turn, which finds the best state and picks it, based on the current network