diff --git a/network.py b/network.py index 381197b..f2c4bc9 100644 --- a/network.py +++ b/network.py @@ -93,7 +93,7 @@ class Network: :param decay_steps: The amount of steps between each decay :return: The result of the exponential decay performed on the learning rate """ - res = max_lr * decay_rate**(global_step // decay_steps) + res = max_lr * decay_rate ** (global_step // decay_steps) return res def do_backprop(self, prev_state, value_next): @@ -104,9 +104,9 @@ class Network: :return: Nothing, the calculation is performed on the model of the network """ self.learning_rate = tf.maximum(self.min_learning_rate, - self.exp_decay(self.max_learning_rate, self.global_step, 0.96, 50000), - name="learning_rate") - + self.exp_decay(self.max_learning_rate, self.global_step, 0.96, 50000), + name="learning_rate") + with tf.GradientTape() as tape: value = self.model(prev_state.reshape(1,-1)) grads = tape.gradient(value, self.model.variables) @@ -165,16 +165,14 @@ class Network: :param states: A number of states. The states have to be transformed before being given to this function. :return: """ - values = self.model.predict_on_batch(states) - return values + return self.model.predict_on_batch(states) def restore_model(self): """ Restore a model for a session, such that a trained model and either be further trained or used for evaluation - - :param sess: Current session + :return: Nothing. It's a side-effect that a model gets restored for the network. """ @@ -211,7 +209,6 @@ class Network: and then picking the best, by using the network to evaluate each state. This is 0-ply, ie. no look-ahead. The highest score is picked for the 1-player and the max(1-score) is picked for the -1-player. - :param sess: :param board: Current board :param roll: Current roll :param player: Current player @@ -224,10 +221,9 @@ class Network: transformed_scores = [x if np.sign(player) > 0 else 1 - x for x in scores] best_score_idx = np.argmax(np.array(transformed_scores)) - best_move = legal_moves[best_score_idx] - best_score = scores[best_score_idx] + best_move, best_score = legal_moves[best_score_idx], scores[best_score_idx] - return [best_move, best_score] + return (best_move, best_score) def make_move_1_ply(self, board, roll, player): """ @@ -237,9 +233,9 @@ class Network: :param player: :return: """ - # start = time.time() + start = time.time() best_pair = self.calculate_1_ply(board, roll, player) - # print(time.time() - start) + print(time.time() - start) return best_pair @@ -248,35 +244,30 @@ class Network: Find the best move based on a 1-ply look-ahead. First the x best moves are picked from a 0-ply and then all moves and scores are found for them. The expected score is then calculated for each of the boards from the 0-ply. - :param sess: + :param board: :param roll: The original roll :param player: The current player :return: Best possible move based on 1-ply look-ahead - """ # find all legal states from the given board and the given roll init_legal_states = Board.calculate_legal_states(board, player, roll) - legal_states = np.array([self.board_trans_func(state, player)[0] for state in init_legal_states]) - scores = self.calc_vals(legal_states) - scores = [score.numpy() for score in scores] + scores = [ score.numpy() + for score + in self.calc_vals(legal_states) ] moves_and_scores = list(zip(init_legal_states, scores)) - - sorted_moves_and_scores = sorted(moves_and_scores, key=itemgetter(1), reverse=player==1) - - best_boards = [x[0] for x in sorted_moves_and_scores[:10]] - - + sorted_moves_and_scores = sorted(moves_and_scores, key=itemgetter(1), reverse=(player == 1)) + best_boards = [ x[0] for x in sorted_moves_and_scores[:10] ] scores, trans_scores = self.do_ply(best_boards, player) best_score_idx = np.array(trans_scores).argmax() - return [best_boards[best_score_idx], scores[best_score_idx]] + return (best_boards[best_score_idx], scores[best_score_idx]) def do_ply(self, boards, player): """ @@ -285,7 +276,6 @@ class Network: allowing the function to search deeper, which could result in an even larger search space. If we wish to have more than 2-ply, this should be fixed, so we could extend this method to allow for 3-ply. - :param sess: :param boards: The boards to try all rolls on :param player: The player of the previous ply :return: An array of scores where each index describes one of the boards which was given as param @@ -305,11 +295,11 @@ class Network: length_list = [] test_list = [] # Prepping of data - start= time.time() + start = time.time() for board in boards: length = 0 for roll in all_rolls: - all_states = list(Board.calculate_legal_states(board, player*-1, roll)) + all_states = Board.calculate_legal_states(board, player*-1, roll) for state in all_states: state = np.array(self.board_trans_func(state, player*-1)[0]) test_list.append(state) @@ -318,148 +308,21 @@ class Network: # print(time.time() - start) - start = time.time() + # start = time.time() - all_scores_legit = self.model.predict_on_batch(np.array(test_list)) + all_scores = self.model.predict_on_batch(np.array(test_list)) split_scores = [] from_idx = 0 for length in length_list: - split_scores.append(all_scores_legit[from_idx:from_idx+length]) + split_scores.append(all_scores[from_idx:from_idx+length]) from_idx += length means_splits = [tf.reduce_mean(scores) for scores in split_scores] transformed_means_splits = [x if player == 1 else (1-x) for x in means_splits] # print(time.time() - start) - return ([means_splits, transformed_means_splits]) - - - def calc_n_ply(self, n_init, sess, board, player, roll): - """ - :param n_init: - :param sess: - :param board: - :param player: - :param roll: - :return: - """ - - # find all legal states from the given board and the given roll - init_legal_states = Board.calculate_legal_states(board, player, roll) - - # find all values for the above boards - zero_ply_moves_and_scores = [(move, self.eval_state(sess, self.board_trans_func(move, player))) for move in init_legal_states] - - # pythons reverse is in place and I can't call [:15] on it, without applying it to an object like so. Fuck. - sorted_moves_and_scores = sorted(zero_ply_moves_and_scores, key=itemgetter(1), reverse=player==1) - - - best_boards = [x[0] for x in sorted_moves_and_scores[:10]] - - best_move_score_pair = self.n_ply(n_init, sess, best_boards, player) - - return best_move_score_pair - - - def n_ply(self, n_init, sess, boards_init, player_init): - """ - :param n_init: - :param sess: - :param boards_init: - :param player_init: - :return: - """ - def ply(n, boards, player): - def calculate_possible_states(board): - possible_rolls = [ (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), - (1, 6), (2, 2), (2, 3), (2, 4), (2, 5), - (2, 6), (3, 3), (3, 4), (3, 5), (3, 6), - (4, 4), (4, 5), (4, 6), (5, 5), (5, 6), - (6, 6) ] - - # for roll in possible_rolls: - # print(len(Board.calculate_legal_states(board, player, roll))) - - return [ Board.calculate_legal_states(board, player, roll) - for roll - in possible_rolls ] - - def find_best_state_score(boards): - score_pairs = [ (board, self.eval_state(sess, self.board_trans_func(board, player))) - for board - in boards ] - scores = [ pair[1] - for pair - in score_pairs ] - best_score_pair = score_pairs[np.array(scores).argmax()] - - return best_score_pair - - def average_score(boards): - return sum(boards)/len(boards) - - def average_ply_score(board): - states_for_rolls = calculate_possible_states(board) - - best_state_score_for_each_roll = [ - find_best_state_score(states) - for states - in states_for_rolls ] - best_score_for_each_roll = [ x[1] - for x - in best_state_score_for_each_roll ] - - average_score_var = average_score(best_score_for_each_roll) - return average_score_var - - - if n == 1: - average_score_pairs = [ (board, average_ply_score(board)) - for board - in boards ] - return average_score_pairs - elif n > 1: # n != 1 - def average_for_score_pairs(score_pairs): - scores = [ pair[1] - for pair - in score_pairs ] - return sum(scores)/len(scores) - - def average_plain(scores): - return sum(scores)/len(scores) - - print("+"*20) - print(n) - print(type(boards)) - print(boards) - possible_states_for_boards = [ - (board, calculate_possible_states(board)) - for board - in boards ] - - average_score_pairs = [ - (inner_boards[0], average_plain([ average_for_score_pairs(ply(n - 1, inner_board, player * -1 if n == 1 else player)) - for inner_board - in inner_boards[1] ])) - for inner_boards - in possible_states_for_boards ] - - return average_score_pairs - - else: - assert False - - if n_init < 1: print("Unexpected argument n = {}".format(n_init)); exit() - - boards_with_scores = ply(n_init, boards_init, -1 * player_init) - #print("Boards with scores:",boards_with_scores) - scores = [ ( pair[1] if player_init == 1 else (1 - pair[1]) ) - for pair - in boards_with_scores ] - #print("All the scores:",scores) - best_score_pair = boards_with_scores[np.array(scores).argmax()] - return best_score_pair + return (means_splits, transformed_means_splits) def eval(self, episode_count, trained_eps = 0): @@ -477,7 +340,6 @@ class Network: """ Do the actual evaluation - :param sess: :param method: Either pubeval or dumbeval :param episodes: Amount of episodes to use in the evaluation :param trained_eps: @@ -509,11 +371,9 @@ class Network: board = Board.initial_state while Board.outcome(board) is None: roll = (random.randrange(1, 7), random.randrange(1, 7)) - board = (self.make_move(board, roll, 1))[0] roll = (random.randrange(1, 7), random.randrange(1, 7)) - board = Eval.make_pubeval_move(board, -1, roll)[0][0:26] sys.stderr.write("\t outcome {}".format(Board.outcome(board)[1])) @@ -532,11 +392,9 @@ class Network: board = Board.initial_state while Board.outcome(board) is None: roll = (random.randrange(1, 7), random.randrange(1, 7)) - board = (self.make_move(board, roll, 1))[0] roll = (random.randrange(1, 7), random.randrange(1, 7)) - board = Eval.make_dumbeval_move(board, -1, roll)[0][0:26] sys.stderr.write("\t outcome {}".format(Board.outcome(board)[1])) @@ -653,7 +511,7 @@ class Network: outcomes.append(Board.outcome(final_board)[1]) final_score = np.array([Board.outcome(final_board)[1]]) scaled_final_score = ((final_score + 2) / 4) - + self.do_backprop(self.board_trans_func(prev_board, player), scaled_final_score.reshape(1,1)) sys.stderr.write("\n") diff --git a/player.py b/player.py index 4208cdd..c8fda84 100644 --- a/player.py +++ b/player.py @@ -20,21 +20,22 @@ class Player: sets.append([Board.calculate_legal_states(board, player, [r,0]), r]) total += r sets.append([Board.calculate_legal_states(board, player, [total,0]), total]) + print(sets) return sets - def tmp_name(self, from_board, to_board, roll, player, total_moves): + def tmp_name(self, from_board, to_board, roll, player, total_moves, is_quad = False): sets = self.calc_move_sets(from_board, roll, player) return_board = from_board for idx, board_set in enumerate(sets): board_set[0] = list(board_set[0]) - print(to_board) - print(board_set) + # print(to_board) + # print(board_set) if to_board in board_set[0]: total_moves -= board_set[1] # if it's not the sum of the moves - if idx < 2: + if idx < (4 if is_quad else 2): roll[idx] = 0 else: roll = [0,0] @@ -43,8 +44,11 @@ class Player: return total_moves, roll, return_board def make_human_move(self, board, roll): - total_moves = roll[0] + roll[1] if roll[0] != roll[1] else int(roll[0])*4 - move = "" + is_quad = roll[0] == roll[1] + total_moves = roll[0] + roll[1] if not is_quad else int(roll[0])*4 + if is_quad: + roll = [roll[0]]*4 + while total_moves != 0: while True: print("You have {roll} left!".format(roll=total_moves)) @@ -60,6 +64,6 @@ class Player: print("The correct syntax is: 2/5 for a move from index 2 to 5.") to_board = Board.apply_moves_to_board(board, self.get_sym(), move) - total_moves, roll, board = self.tmp_name(board, to_board, list(roll), self.get_sym(), total_moves) + total_moves, roll, board = self.tmp_name(board, to_board, list(roll), self.get_sym(), total_moves, is_quad) print(Board.pretty(board)) return board \ No newline at end of file