diff --git a/main.py b/main.py index e2e8988..eb73a98 100644 --- a/main.py +++ b/main.py @@ -60,7 +60,8 @@ config = { 'bench_storage_path': 'bench', 'board_representation': args.board_rep, 'force_creation': args.force_creation, - 'use_baseline': args.use_baseline + 'use_baseline': args.use_baseline, + 'global_step': 0 } # Create models folder @@ -191,7 +192,7 @@ if __name__ == "__main__": episode_counts = [25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 20000] - def do_eval(sess): + def do_eval(): for eval_method in config['eval_methods']: result_path = os.path.join(config['bench_storage_path'], eval_method) + "-{}.log".format(int(time.time())) @@ -199,8 +200,7 @@ if __name__ == "__main__": for i in range(sample_count): start_time = time.time() # Evaluation measure to be benchmarked are described in `config` - outcomes = network.eval(episode_count = n, - tf_session = sess) + outcomes = network.eval(episode_count = n) time_diff = time.time() - start_time log_bench_eval_outcomes(outcomes, time = time_diff, @@ -210,8 +210,8 @@ if __name__ == "__main__": # CMM: oh no import tensorflow as tf - with tf.Session() as session: - network.restore_model(session) - do_eval(session) + + network.restore_model() + do_eval() diff --git a/network.py b/network.py index 818b886..d1d4a42 100644 --- a/network.py +++ b/network.py @@ -44,8 +44,6 @@ class Network: self.max_learning_rate = 0.1 self.min_learning_rate = 0.001 - self.global_step = tf.train.get_or_create_global_step() - #tf.train.get_or_create_global_step() # Restore trained episode count for model episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained") @@ -55,10 +53,18 @@ class Network: else: self.episodes_trained = 0 + global_step_path = os.path.join(self.checkpoint_path, "global_step") + if os.path.isfile(global_step_path): + with open(global_step_path, 'r') as f: + self.global_step = int(f.read()) + else: + self.global_step = 0 + + self.model = tf.keras.Sequential([ tf.keras.layers.Dense(40, activation="sigmoid", kernel_initializer=xavier_init, - input_shape=(1,30)), + input_shape=(1,self.input_size)), tf.keras.layers.Dense(1, activation="sigmoid", kernel_initializer=xavier_init) ]) @@ -72,11 +78,10 @@ class Network: def do_backprop(self, prev_state, value_next): self.learning_rate = tf.maximum(self.min_learning_rate, - self.exp_decay(self.max_learning_rate, self.episodes_trained, 0.96, 50000), + self.exp_decay(self.max_learning_rate, self.global_step, 0.96, 50000), name="learning_rate") - # self.learning_rate = 0.1 - print(tf.train.get_global_step()) + with tf.GradientTape() as tape: value = self.model(prev_state.reshape(1,-1)) grads = tape.gradient(value, self.model.variables) @@ -91,19 +96,24 @@ class Network: backprop_calc = self.learning_rate * difference_in_values * grad train_var.assign_add(backprop_calc) - print(self.episodes_trained) - def eval_state(self, sess, state): - return sess.run(self.value, feed_dict={self.x: state}) - def save_model(self, episode_count, global_step): - tfe.Saver(self.model.variables).save(os.path.join(self.checkpoint_path, 'model.ckpt'), global_step=self.global_step) + def eval_state(self, state): + return self.model(state.reshape(1,-1)) + + def save_model(self, episode_count): + tfe.Saver(self.model.variables).save(os.path.join(self.checkpoint_path, 'model.ckpt')) #self.saver.save(sess, os.path.join(self.checkpoint_path, 'model.ckpt'), global_step=global_step) with open(os.path.join(self.checkpoint_path, "episodes_trained"), 'w+') as f: print("[NETWK] ({name}) Saving model to:".format(name=self.name), os.path.join(self.checkpoint_path, 'model.ckpt')) f.write(str(episode_count) + "\n") + with open(os.path.join(self.checkpoint_path, "global_step"), 'w+') as f: + print("[NETWK] ({name}) Saving global step to:".format(name=self.name), + os.path.join(self.checkpoint_path, 'model.ckpt')) + f.write(str(self.global_step) + "\n") + def calc_vals(self, states): values = self.model.predict_on_batch(states) @@ -134,20 +144,14 @@ class Network: if os.path.isfile(episode_count_path): with open(episode_count_path, 'r') as f: self.config['start_episode'] = int(f.read()) - # else: - # latest_checkpoint = tf.train.latest_checkpoint("./") - # print("[NETWK] ({name}) Restoring model from:".format(name=self.name), - # str(latest_checkpoint)) - # tfe.Saver(self.model.variables).restore(latest_checkpoint) - #variables_names = [v.name for v in self.model.variables] + global_step_path = os.path.join(self.checkpoint_path, "global_step") + if os.path.isfile(global_step_path): + with open(global_step_path, 'r') as f: + self.config['global_step'] = int(f.read()) + + - # Restore trained episode count for model - #episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained") - #if os.path.isfile(episode_count_path): - # with open(episode_count_path, 'r') as f: - # self.config['start_episode'] = int(f.read()) - tf.train.get_or_create_global_step() def make_move(self, board, roll, player): """ @@ -161,10 +165,12 @@ class Network: :param player: Current player :return: A pair of the best state to go to, together with the score of that state """ - legal_states = list(Board.calculate_legal_states(board, player, roll)) - legal_states = [list(tmp) for tmp in legal_states] + legal_moves = list(Board.calculate_legal_states(board, player, roll)) + + legal_states = [list(tmp) for tmp in legal_moves] + legal_states = np.array([Board.board_features_quack_fat(tmp, player)[0] for tmp in legal_states]) - legal_moves = [self.board_trans_func(board, player) for board in Board.calculate_legal_states(board, player, roll)] + scores = self.model.predict_on_batch(legal_states) transformed_scores = [x if np.sign(player) > 0 else 1 - x for x in scores] @@ -172,7 +178,7 @@ class Network: best_score_idx = np.argmax(np.array(transformed_scores)) best_move = legal_moves[best_score_idx] best_score = scores[best_score_idx] - self.episodes_trained += 1 + return [best_move, best_score] def make_move_n_ply(self, sess, board, roll, player, n = 1): @@ -385,7 +391,7 @@ class Network: return all_rolls_scores - def eval(self, episode_count, trained_eps = 0, tf_session = None): + def eval(self, episode_count, trained_eps = 0): """ Used to evaluate a model. Can either use pubeval, a model playing at an intermediate level, or dumbeval a model which has been given random weights, so it acts deterministically random. @@ -396,7 +402,7 @@ class Network: :return: outcomes: The outcomes of the evaluation session """ - def do_eval(sess, method, episodes = 1000, trained_eps = 0): + def do_eval(method, episodes = 1000, trained_eps = 0): """ Do the actual evaluation @@ -433,7 +439,7 @@ class Network: while Board.outcome(board) is None: roll = (random.randrange(1, 7), random.randrange(1, 7)) - board = (self.make_move(sess, board, roll, 1))[0] + board = (self.make_move(board, roll, 1))[0] roll = (random.randrange(1, 7), random.randrange(1, 7)) @@ -456,7 +462,7 @@ class Network: while Board.outcome(board) is None: roll = (random.randrange(1, 7), random.randrange(1, 7)) - board = (self.make_move(sess, board, roll, 1))[0] + board = (self.make_move(board, roll, 1))[0] roll = (random.randrange(1, 7), random.randrange(1, 7)) @@ -475,40 +481,26 @@ class Network: sys.stderr.write("[EVAL ] Evaluation method '{}' is not defined\n".format(method)) return [0] - if tf_session == None: - with tf.Session() as session: - session.run(tf.global_variables_initializer()) - self.restore_model(session) - outcomes = [ (method, do_eval(session, - method, - episode_count, - trained_eps = trained_eps)) - for method - in self.config['eval_methods'] ] - return outcomes - else: - outcomes = [ (method, do_eval(tf_session, - method, - episode_count, - trained_eps = trained_eps)) - for method - in self.config['eval_methods'] ] - return outcomes + + outcomes = [ (method, do_eval(method, + episode_count, + trained_eps = trained_eps)) + for method + in self.config['eval_methods'] ] + return outcomes def train_model(self, episodes=1000, save_step_size=100, trained_eps=0): with tf.Session() as sess: difference_in_vals = 0 - writer = tf.summary.FileWriter("/tmp/log/tf", sess.graph) - sess.run(tf.global_variables_initializer()) - self.restore_model(sess) + self.restore_model() - variables_names = [v.name for v in tf.trainable_variables()] - values = sess.run(variables_names) - for k, v in zip(variables_names, values): - print("Variable: ", k) - print("Shape: ", v.shape) - print(v) + #variables_names = [v.name for v in tf.trainable_variables()] + #values = sess.run(variables_names) + #for k, v in zip(variables_names, values): + # print("Variable: ", k) + # print("Shape: ", v.shape) + # print(v) start_time = time.time() @@ -536,21 +528,21 @@ class Network: i = 0 while Board.outcome(prev_board) is None: i += 1 + self.global_step += 1 - cur_board, cur_board_value = self.make_move(sess, - prev_board, + + cur_board, cur_board_value = self.make_move(prev_board, (random.randrange(1, 7), random.randrange(1, 7)), player) - difference_in_vals += abs((cur_board_value - self.eval_state(sess, self.board_trans_func(prev_board, player)))) + difference_in_vals += abs((cur_board_value - self.eval_state(self.board_trans_func(prev_board, player)))) # adjust weights - sess.run(self.training_op, - feed_dict={self.x: self.board_trans_func(prev_board, player), - self.value_next: cur_board_value}) - - player *= -1 + #print(cur_board) + if Board.outcome(cur_board) is None: + self.do_backprop(self.board_trans_func(prev_board, player), cur_board_value) + player *= -1 prev_board = cur_board @@ -560,27 +552,23 @@ class Network: final_score = np.array([Board.outcome(final_board)[1]]) scaled_final_score = ((final_score + 2) / 4) - with tf.name_scope("final"): - merged = tf.summary.merge_all() - global_step, summary, _ = sess.run([self.global_step, merged, self.training_op], - feed_dict={self.x: self.board_trans_func(prev_board, player), - self.value_next: scaled_final_score.reshape((1, 1))}) - writer.add_summary(summary, episode + trained_eps) + + self.do_backprop(self.board_trans_func(prev_board, player), scaled_final_score.reshape(1,1)) + + sys.stderr.write("\n") if episode % min(save_step_size, episodes) == 0: sys.stderr.write("[TRAIN] Saving model...\n") - self.save_model(sess, episode + trained_eps, global_step) + self.save_model(episode + trained_eps) if episode % 50 == 0: print_time_estimate(episode) sys.stderr.write("[TRAIN] Saving model for final episode...\n") - self.save_model(sess, episode+trained_eps, global_step) - - writer.close() - + self.save_model(episode+trained_eps) + return outcomes, difference_in_vals[0][0]