Pownie/backgammon
1
0
Fork 0
Code Issues 6 Pull Requests Packages Projects Releases Wiki Activity

We should now be able to both train and eval as per usual.

Browse Source 
I've added a file "global_step", which works as the new global_step
counter, so we can use it for exp_decay.
This commit is contained in:
Alexander Munch-Hansen 2018-05-09 23:15:35 +02:00
parent cb7e7b519c
commit 6429e0732c
2 changed files with 73 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
main.py
View File

@ -60,7 +60,8 @@ config = {
'bench_storage_path': 'bench', 'bench_storage_path': 'bench',
'board_representation': args.board_rep, 'board_representation': args.board_rep,
'force_creation': args.force_creation, 'force_creation': args.force_creation,
'use_baseline': args.use_baseline 'use_baseline': args.use_baseline,
'global_step': 0
} }
# Create models folder # Create models folder
@ -191,7 +192,7 @@ if __name__ == "__main__":
episode_counts = [25, 50, 100, 250, 500, 1000, 2500, 5000, episode_counts = [25, 50, 100, 250, 500, 1000, 2500, 5000,
10000, 20000] 10000, 20000]
def do_eval(sess): def do_eval():
for eval_method in config['eval_methods']: for eval_method in config['eval_methods']:
result_path = os.path.join(config['bench_storage_path'], result_path = os.path.join(config['bench_storage_path'],
eval_method) + "-{}.log".format(int(time.time())) eval_method) + "-{}.log".format(int(time.time()))
@ -199,8 +200,7 @@ if __name__ == "__main__":
for i in range(sample_count): for i in range(sample_count):
start_time = time.time() start_time = time.time()
# Evaluation measure to be benchmarked are described in `config` # Evaluation measure to be benchmarked are described in `config`
outcomes = network.eval(episode_count = n, outcomes = network.eval(episode_count = n)
tf_session = sess)
time_diff = time.time() - start_time time_diff = time.time() - start_time
log_bench_eval_outcomes(outcomes, log_bench_eval_outcomes(outcomes,
time = time_diff, time = time_diff,
@ -210,8 +210,8 @@ if __name__ == "__main__":
# CMM: oh no # CMM: oh no
import tensorflow as tf import tensorflow as tf
with tf.Session() as session:
network.restore_model(session) network.restore_model()
do_eval(session) do_eval()

130
network.py
View File

@ -44,8 +44,6 @@ class Network:
self.max_learning_rate = 0.1 self.max_learning_rate = 0.1
self.min_learning_rate = 0.001 self.min_learning_rate = 0.001
self.global_step = tf.train.get_or_create_global_step()
#tf.train.get_or_create_global_step() #tf.train.get_or_create_global_step()
# Restore trained episode count for model # Restore trained episode count for model
episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained") episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained")
@ -55,10 +53,18 @@ class Network:
else: else:
self.episodes_trained = 0 self.episodes_trained = 0
global_step_path = os.path.join(self.checkpoint_path, "global_step")
if os.path.isfile(global_step_path):
with open(global_step_path, 'r') as f:
self.global_step = int(f.read())
else:
self.global_step = 0
self.model = tf.keras.Sequential([ self.model = tf.keras.Sequential([
tf.keras.layers.Dense(40, activation="sigmoid", kernel_initializer=xavier_init, tf.keras.layers.Dense(40, activation="sigmoid", kernel_initializer=xavier_init,
input_shape=(1,30)), input_shape=(1,self.input_size)),
tf.keras.layers.Dense(1, activation="sigmoid", kernel_initializer=xavier_init) tf.keras.layers.Dense(1, activation="sigmoid", kernel_initializer=xavier_init)
]) ])
@ -72,11 +78,10 @@ class Network:
def do_backprop(self, prev_state, value_next): def do_backprop(self, prev_state, value_next):
self.learning_rate = tf.maximum(self.min_learning_rate, self.learning_rate = tf.maximum(self.min_learning_rate,
self.exp_decay(self.max_learning_rate, self.episodes_trained, 0.96, 50000), self.exp_decay(self.max_learning_rate, self.global_step, 0.96, 50000),
name="learning_rate") name="learning_rate")
# self.learning_rate = 0.1
print(tf.train.get_global_step())
with tf.GradientTape() as tape: with tf.GradientTape() as tape:
value = self.model(prev_state.reshape(1,-1)) value = self.model(prev_state.reshape(1,-1))
grads = tape.gradient(value, self.model.variables) grads = tape.gradient(value, self.model.variables)
@ -91,19 +96,24 @@ class Network:
backprop_calc = self.learning_rate * difference_in_values * grad backprop_calc = self.learning_rate * difference_in_values * grad
train_var.assign_add(backprop_calc) train_var.assign_add(backprop_calc)
print(self.episodes_trained)
def eval_state(self, sess, state):
return sess.run(self.value, feed_dict={self.x: state})
def save_model(self, episode_count, global_step): def eval_state(self, state):
tfe.Saver(self.model.variables).save(os.path.join(self.checkpoint_path, 'model.ckpt'), global_step=self.global_step) return self.model(state.reshape(1,-1))
def save_model(self, episode_count):
tfe.Saver(self.model.variables).save(os.path.join(self.checkpoint_path, 'model.ckpt'))
#self.saver.save(sess, os.path.join(self.checkpoint_path, 'model.ckpt'), global_step=global_step) #self.saver.save(sess, os.path.join(self.checkpoint_path, 'model.ckpt'), global_step=global_step)
with open(os.path.join(self.checkpoint_path, "episodes_trained"), 'w+') as f: with open(os.path.join(self.checkpoint_path, "episodes_trained"), 'w+') as f:
print("[NETWK] ({name}) Saving model to:".format(name=self.name), print("[NETWK] ({name}) Saving model to:".format(name=self.name),
os.path.join(self.checkpoint_path, 'model.ckpt')) os.path.join(self.checkpoint_path, 'model.ckpt'))
f.write(str(episode_count) + "\n") f.write(str(episode_count) + "\n")
with open(os.path.join(self.checkpoint_path, "global_step"), 'w+') as f:
print("[NETWK] ({name}) Saving global step to:".format(name=self.name),
os.path.join(self.checkpoint_path, 'model.ckpt'))
f.write(str(self.global_step) + "\n")
def calc_vals(self, states): def calc_vals(self, states):
values = self.model.predict_on_batch(states) values = self.model.predict_on_batch(states)
@ -134,20 +144,14 @@ class Network:
if os.path.isfile(episode_count_path): if os.path.isfile(episode_count_path):
with open(episode_count_path, 'r') as f: with open(episode_count_path, 'r') as f:
self.config['start_episode'] = int(f.read()) self.config['start_episode'] = int(f.read())
# else:
# latest_checkpoint = tf.train.latest_checkpoint("./")
# print("[NETWK] ({name}) Restoring model from:".format(name=self.name),
# str(latest_checkpoint))
# tfe.Saver(self.model.variables).restore(latest_checkpoint)
#variables_names = [v.name for v in self.model.variables] global_step_path = os.path.join(self.checkpoint_path, "global_step")
if os.path.isfile(global_step_path):
with open(global_step_path, 'r') as f:
self.config['global_step'] = int(f.read())
# Restore trained episode count for model
#episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained")
#if os.path.isfile(episode_count_path):
# with open(episode_count_path, 'r') as f:
# self.config['start_episode'] = int(f.read())
tf.train.get_or_create_global_step()
def make_move(self, board, roll, player): def make_move(self, board, roll, player):
""" """
@ -161,10 +165,12 @@ class Network:
:param player: Current player :param player: Current player
:return: A pair of the best state to go to, together with the score of that state :return: A pair of the best state to go to, together with the score of that state
""" """
legal_states = list(Board.calculate_legal_states(board, player, roll)) legal_moves = list(Board.calculate_legal_states(board, player, roll))
legal_states = [list(tmp) for tmp in legal_states]
legal_states = [list(tmp) for tmp in legal_moves]
legal_states = np.array([Board.board_features_quack_fat(tmp, player)[0] for tmp in legal_states]) legal_states = np.array([Board.board_features_quack_fat(tmp, player)[0] for tmp in legal_states])
legal_moves = [self.board_trans_func(board, player) for board in Board.calculate_legal_states(board, player, roll)]
scores = self.model.predict_on_batch(legal_states) scores = self.model.predict_on_batch(legal_states)
transformed_scores = [x if np.sign(player) > 0 else 1 - x for x in scores] transformed_scores = [x if np.sign(player) > 0 else 1 - x for x in scores]
@ -172,7 +178,7 @@ class Network:
best_score_idx = np.argmax(np.array(transformed_scores)) best_score_idx = np.argmax(np.array(transformed_scores))
best_move = legal_moves[best_score_idx] best_move = legal_moves[best_score_idx]
best_score = scores[best_score_idx] best_score = scores[best_score_idx]
self.episodes_trained += 1
return [best_move, best_score] return [best_move, best_score]
def make_move_n_ply(self, sess, board, roll, player, n = 1): def make_move_n_ply(self, sess, board, roll, player, n = 1):
@ -385,7 +391,7 @@ class Network:
return all_rolls_scores return all_rolls_scores
def eval(self, episode_count, trained_eps = 0, tf_session = None): def eval(self, episode_count, trained_eps = 0):
""" """
Used to evaluate a model. Can either use pubeval, a model playing at an intermediate level, or dumbeval Used to evaluate a model. Can either use pubeval, a model playing at an intermediate level, or dumbeval
a model which has been given random weights, so it acts deterministically random. a model which has been given random weights, so it acts deterministically random.
@ -396,7 +402,7 @@ class Network:
:return: outcomes: The outcomes of the evaluation session :return: outcomes: The outcomes of the evaluation session
""" """
def do_eval(sess, method, episodes = 1000, trained_eps = 0): def do_eval(method, episodes = 1000, trained_eps = 0):
""" """
Do the actual evaluation Do the actual evaluation
@ -433,7 +439,7 @@ class Network:
while Board.outcome(board) is None: while Board.outcome(board) is None:
roll = (random.randrange(1, 7), random.randrange(1, 7)) roll = (random.randrange(1, 7), random.randrange(1, 7))
board = (self.make_move(sess, board, roll, 1))[0] board = (self.make_move(board, roll, 1))[0]
roll = (random.randrange(1, 7), random.randrange(1, 7)) roll = (random.randrange(1, 7), random.randrange(1, 7))
@ -456,7 +462,7 @@ class Network:
while Board.outcome(board) is None: while Board.outcome(board) is None:
roll = (random.randrange(1, 7), random.randrange(1, 7)) roll = (random.randrange(1, 7), random.randrange(1, 7))
board = (self.make_move(sess, board, roll, 1))[0] board = (self.make_move(board, roll, 1))[0]
roll = (random.randrange(1, 7), random.randrange(1, 7)) roll = (random.randrange(1, 7), random.randrange(1, 7))
@ -475,20 +481,8 @@ class Network:
sys.stderr.write("[EVAL ] Evaluation method '{}' is not defined\n".format(method)) sys.stderr.write("[EVAL ] Evaluation method '{}' is not defined\n".format(method))
return [0] return [0]
if tf_session == None:
with tf.Session() as session: outcomes = [ (method, do_eval(method,
session.run(tf.global_variables_initializer())
self.restore_model(session)
outcomes = [ (method, do_eval(session,
method,
episode_count,
trained_eps = trained_eps))
for method
in self.config['eval_methods'] ]
return outcomes
else:
outcomes = [ (method, do_eval(tf_session,
method,
episode_count, episode_count,
trained_eps = trained_eps)) trained_eps = trained_eps))
for method for method
@ -498,17 +492,15 @@ class Network:
def train_model(self, episodes=1000, save_step_size=100, trained_eps=0): def train_model(self, episodes=1000, save_step_size=100, trained_eps=0):
with tf.Session() as sess: with tf.Session() as sess:
difference_in_vals = 0 difference_in_vals = 0
writer = tf.summary.FileWriter("/tmp/log/tf", sess.graph)
sess.run(tf.global_variables_initializer()) self.restore_model()
self.restore_model(sess)
variables_names = [v.name for v in tf.trainable_variables()] #variables_names = [v.name for v in tf.trainable_variables()]
values = sess.run(variables_names) #values = sess.run(variables_names)
for k, v in zip(variables_names, values): #for k, v in zip(variables_names, values):
print("Variable: ", k) # print("Variable: ", k)
print("Shape: ", v.shape) # print("Shape: ", v.shape)
print(v) # print(v)
start_time = time.time() start_time = time.time()
@ -536,20 +528,20 @@ class Network:
i = 0 i = 0
while Board.outcome(prev_board) is None: while Board.outcome(prev_board) is None:
i += 1 i += 1
self.global_step += 1
cur_board, cur_board_value = self.make_move(sess,
prev_board, cur_board, cur_board_value = self.make_move(prev_board,
(random.randrange(1, 7), random.randrange(1, 7)), (random.randrange(1, 7), random.randrange(1, 7)),
player) player)
difference_in_vals += abs((cur_board_value - self.eval_state(sess, self.board_trans_func(prev_board, player)))) difference_in_vals += abs((cur_board_value - self.eval_state(self.board_trans_func(prev_board, player))))
# adjust weights # adjust weights
sess.run(self.training_op, #print(cur_board)
feed_dict={self.x: self.board_trans_func(prev_board, player), if Board.outcome(cur_board) is None:
self.value_next: cur_board_value}) self.do_backprop(self.board_trans_func(prev_board, player), cur_board_value)
player *= -1 player *= -1
prev_board = cur_board prev_board = cur_board
@ -560,26 +552,22 @@ class Network:
final_score = np.array([Board.outcome(final_board)[1]]) final_score = np.array([Board.outcome(final_board)[1]])
scaled_final_score = ((final_score + 2) / 4) scaled_final_score = ((final_score + 2) / 4)
with tf.name_scope("final"):
merged = tf.summary.merge_all() self.do_backprop(self.board_trans_func(prev_board, player), scaled_final_score.reshape(1,1))
global_step, summary, _ = sess.run([self.global_step, merged, self.training_op],
feed_dict={self.x: self.board_trans_func(prev_board, player),
self.value_next: scaled_final_score.reshape((1, 1))})
writer.add_summary(summary, episode + trained_eps)
sys.stderr.write("\n") sys.stderr.write("\n")
if episode % min(save_step_size, episodes) == 0: if episode % min(save_step_size, episodes) == 0:
sys.stderr.write("[TRAIN] Saving model...\n") sys.stderr.write("[TRAIN] Saving model...\n")
self.save_model(sess, episode + trained_eps, global_step) self.save_model(episode + trained_eps)
if episode % 50 == 0: if episode % 50 == 0:
print_time_estimate(episode) print_time_estimate(episode)
sys.stderr.write("[TRAIN] Saving model for final episode...\n") sys.stderr.write("[TRAIN] Saving model for final episode...\n")
self.save_model(sess, episode+trained_eps, global_step) self.save_model(episode+trained_eps)
writer.close()
return outcomes, difference_in_vals[0][0] return outcomes, difference_in_vals[0][0]