backgammon/network.py

import tensorflow as tf
from cup import Cup
import numpy as np
from board import Board
import os
import time
import sys
import random
from eval import Eval

class Network:
    hidden_size = 40
    input_size = 26
    output_size = 1
    # Can't remember the best learning_rate, look this up
    learning_rate = 0.05

    # TODO: Actually compile tensorflow properly
    #os.environ["TF_CPP_MIN_LOG_LEVEL"]="2"

    def custom_tanh(self, x, name=None):
        return tf.scalar_mul(tf.constant(2.00), tf.tanh(x, name))
    
    def __init__(self, config, name):
        self.config = config
        self.checkpoint_path = config['model_path']

        self.name = name

        # Restore trained episode count for model
        episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained")
        if os.path.isfile(episode_count_path):
            with open(episode_count_path, 'r') as f:
                self.episodes_trained = int(f.read())
        else:
            self.episodes_trained = 0
        
        # input = x
        self.x = tf.placeholder('float', [1, Network.input_size], name='input')
        self.value_next = tf.placeholder('float', [1, Network.output_size], name="value_next")

        xavier_init = tf.contrib.layers.xavier_initializer()
        
        W_1 = tf.get_variable("w_1", (Network.input_size, Network.hidden_size),
                              initializer=xavier_init)
        W_2 = tf.get_variable("w_2", (Network.hidden_size, Network.output_size),
                              initializer=xavier_init)

        b_1 = tf.get_variable("b_1", (Network.hidden_size,),
                              initializer=tf.zeros_initializer)
        b_2 = tf.get_variable("b_2", (Network.output_size,),
                              initializer=tf.zeros_initializer)

        normalized_input  = tf.nn.l2_normalize(self.x)
        value_after_input = tf.sigmoid(tf.matmul(normalized_input, W_1) + b_1, name='hidden_layer')

        self.value = tf.sigmoid(tf.matmul(value_after_input, W_2) + b_2, name='output_layer')

        # tf.reduce_sum basically finds the sum of its input, so this gives the
        # difference between the two values, in case they should be lists, which
        # they might be if our input changes

        # TODO: Alexander thinks that self.value will be computed twice (instead of once)
        difference_in_values = tf.reshape(tf.subtract(self.value_next, self.value, name='difference_in_values'), [])
        tf.summary.scalar("difference_in_values", tf.abs(difference_in_values))
        
        trainable_vars = tf.trainable_variables()
        gradients = tf.gradients(self.value, trainable_vars)
        
        apply_gradients = []
        
        with tf.variable_scope('apply_gradients'):
            for gradient, trainable_var in zip(gradients, trainable_vars):
                # Hopefully this is Δw_t = α(V_t+1 - V_t)▿_wV_t.
                backprop_calc = Network.learning_rate * difference_in_values * gradient
                grad_apply = trainable_var.assign_add(backprop_calc)
                apply_gradients.append(grad_apply)
            
            self.training_op = tf.group(*apply_gradients, name='training_op')

        self.saver = tf.train.Saver(max_to_keep=1)
        
    def eval_state(self, sess, state):
        # Run state through a network

        # Remember to create placeholders for everything because wtf tensorflow
        # and graphs

        # Remember to create the dense layers

        # Figure out a way of giving a layer a custom activiation function (we
        # want something which gives [-2,2]. Naively tahn*2, however I fell this
        # is wrong.

        # tf.group, groups a bunch of actions, so calculate the different
        # gradients for the different weights, by using tf.trainable_variables()
        # to find all variables and tf.gradients(current_value,
        # trainable_variables) to find all the gradients. We can then loop
        # through this and calculate the trace for each gradient and variable
        # pair (note, zip can be used to combine the two lists found before),
        # and then we can calculate the overall change in weights, based on the
        # formula listed in tesauro (learning_rate * difference_in_values *
        # trace), this calculation can be assigned to a tf variable and put in a
        # list and then this can be grouped into a single operation, essentially
        # building our own backprop function.

        # Grouping them is done by
        # tf.group(*the_gradients_from_before_we_want_to_apply,
        # name="training_op")

        # If we remove the eligibily trace to begin with, we only have to
        # implement learning_rate * (difference_in_values) * gradients (the
        # before-mentioned calculation.

        
        # print("Network is evaluating")
        #print("eval ({})".format(self.name), state, val, sep="\n")
        return sess.run(self.value, feed_dict={self.x: state})


    def save_model(self, sess, episode_count):
        self.saver.save(sess, os.path.join(self.checkpoint_path, 'model.ckpt'))
        with open(os.path.join(self.checkpoint_path, "episodes_trained"), 'w+') as f:
            print("[NETWK] ({name}) Saving model to:".format(name = self.name),
                  os.path.join(self.checkpoint_path, 'model.ckpt'))
            f.write(str(episode_count) + "\n")
    
    def restore_model(self, sess):
        if os.path.isfile(os.path.join(self.checkpoint_path, 'model.ckpt.index')):
            latest_checkpoint = tf.train.latest_checkpoint(self.checkpoint_path)
            print("[NETWK] ({name}) Restoring model from:".format(name = self.name),
                  str(latest_checkpoint))
            self.saver.restore(sess, latest_checkpoint)
            variables_names = [v.name for v in tf.trainable_variables()]
            values = sess.run(variables_names)
            for k, v in zip(variables_names, values):
                print("Variable: ", k)
                print("Shape: ", v.shape)
                print(v)

            # Restore trained episode count for model
            episode_count_path = os.path.join(self.checkpoint_path, "episodes_trained")
            if os.path.isfile(episode_count_path):
                with open(episode_count_path, 'r') as f:
                    self.config['start_episode'] = int(f.read())

    def make_move(self, sess, board, roll):
        # print(Board.pretty(board))
        legal_moves = Board.calculate_legal_states(board, 1, roll)
        moves_and_scores = [ (move, self.eval_state(sess, np.array(move).reshape(1,26))) for move in legal_moves ]
        scores = [ x[1] for x in moves_and_scores ]
        best_score_index = np.array(scores).argmax()
        best_move_pair = moves_and_scores[best_score_index]
        #print("Found the best state, being:", np.array(move_scores).argmax())
        return best_move_pair
        
                
    def train_model(self, episodes=1000, save_step_size = 100, trained_eps = 0):
        with tf.Session() as sess:
            writer = tf.summary.FileWriter("/tmp/log/tf", sess.graph)
            
            sess.run(tf.global_variables_initializer())
            self.restore_model(sess)
            
            variables_names = [v.name for v in tf.trainable_variables()]
            values = sess.run(variables_names)
            for k, v in zip(variables_names, values):
                print("Variable: ", k)
                print("Shape: ", v.shape)
                print(v)

            start_time = time.time()

            def print_time_estimate(eps_completed):
                cur_time      = time.time()
                time_diff     = cur_time - start_time
                eps_per_sec   = eps_completed / time_diff
                secs_per_ep   = time_diff / eps_completed
                eps_remaining = (episodes - eps_completed)
                sys.stderr.write("[TRAIN] Averaging {per_sec} episodes per second\n".format(per_sec = round(eps_per_sec, 2)))
                sys.stderr.write("[TRAIN] {eps_remaining} episodes remaining; approx. {time_remaining} seconds remaining\n".format(eps_remaining = eps_remaining, time_remaining = int(eps_remaining * secs_per_ep)))

        
            sys.stderr.write("[TRAIN] Training {} episodes and save_step_size {}\n".format(episodes, save_step_size))
            outcomes = []
            for episode in range(1, episodes + 1):
                sys.stderr.write("[TRAIN] Episode {}".format(episode + trained_eps))
                # TODO decide which player should be here
                player = 1
                
                roll = (random.randrange(1,7), random.randrange(1,7))
                prev_board, _ = self.make_move(sess, Board.flip(Board.initial_state) if player == -1 else Board.initial_state, roll)
                if player == -1:
                    prev_board = Board.flip(prev_board)
            
                # find the best move here, make this move, then change turn as the
                # first thing inside of the while loop and then call
                # best_move_and_score to get V_t+1

                # i = 0
                while Board.outcome(prev_board) is None:
                    # print("-"*30)
                    # print(i)
                    # print(roll)
                    # print(Board.pretty(prev_board))
                    # print("/"*30)
                    # i += 1
                
                    player *= -1
                    roll = (random.randrange(1,7), random.randrange(1,7))

                    cur_board, cur_board_value = self.make_move(sess, Board.flip(prev_board) if player == -1 else prev_board, roll)
                    if player == -1:
                        cur_board  = Board.flip(cur_board)

                    # print("cur_board_value:", cur_board_value)
                    
                    # adjust weights
                    sess.run(self.training_op,
                             feed_dict = { self.x: np.array(prev_board).reshape((1,26)),
                                           self.value_next: cur_board_value })
                    prev_board = cur_board

                final_board = prev_board
                sys.stderr.write("\t outcome {}".format(Board.outcome(final_board)[1]))
                outcomes.append(Board.outcome(final_board)[1])
                final_score = np.array([ Board.outcome(final_board)[1] ])
                scaled_final_score = ((final_score + 2) / 4)

                # print("scaled_final_score",scaled_final_score)

                with tf.name_scope("final"):
                    merged = tf.summary.merge_all()
                    summary, _ = sess.run([merged, self.training_op],
                                          feed_dict = { self.x: np.array(prev_board).reshape((1,26)),
                                                        self.value_next: scaled_final_score.reshape((1, 1)) })
                    writer.add_summary(summary, episode + trained_eps)
                    
                sys.stderr.write("\n")
            
                if episode % min(save_step_size, episodes) == 0:
                    sys.stderr.write("[TRAIN] Saving model...\n")
                    self.save_model(sess, episode+trained_eps)

                if episode % 50 == 0:
                    print_time_estimate(episode)

            sys.stderr.write("[TRAIN] Saving model for final episode...\n")
            self.save_model(sess, episode+trained_eps)
            
            writer.close()
        
            return outcomes

            
                # take turn, which finds the best state and picks it, based on the current network
                # save current state
                # run training operation (session.run(self.training_op, {x:x, value_next, value_next})), (something which does the backprop, based on the state after having taken a turn, found before, and the state we saved in the beginning and from now we'll save it at the end of the turn
                # save the current state again, so we can continue running backprop based on the "previous" turn.

        # NOTE: We need to make a method so that we can take a single turn or at least just pick the next best move, so we know how to evaluate according to TD-learning. Right now, our game just continues in a while loop without nothing to stop it!
        


    def eval(self, trained_eps = 0):
        def do_eval(sess, method, episodes = 1000, trained_eps = 0):
            start_time = time.time()

            def print_time_estimate(eps_completed):
                cur_time      = time.time()
                time_diff     = cur_time - start_time
                eps_per_sec   = eps_completed / time_diff
                secs_per_ep   = time_diff / eps_completed
                eps_remaining = (episodes - eps_completed)
                sys.stderr.write("[EVAL ] Averaging {per_sec} episodes per second\n".format(per_sec = round(eps_per_sec, 2)))
                sys.stderr.write("[EVAL ] {eps_remaining} episodes remaining; approx. {time_remaining} seconds remaining\n".format(eps_remaining = eps_remaining, time_remaining = int(eps_remaining * secs_per_ep)))

            sys.stderr.write("[EVAL ] Evaluating {eps} episode(s) with method '{method}'\n".format(eps=episodes, method=method))
            
            if method == 'random':
                outcomes = []
                for i in range(1, episodes + 1):
                    sys.stderr.write("[EVAL ] Episode {}".format(i))
                    board = Board.initial_state
                    while Board.outcome(board) is None:
                        roll = (random.randrange(1,7), random.randrange(1,7))
                        board = (self.p1.make_move(sess, board, self.p1.get_sym(), roll))[0]
                        roll = (random.randrange(1,7), random.randrange(1,7))
                        board = Board.flip(Eval.make_random_move(Board.flip(board), 1, roll))
                    sys.stderr.write("\t outcome {}".format(Board.outcome(board)[1]))
                    outcomes.append(Board.outcome(board)[1])
                    sys.stderr.write("\n")

                    if i % 50 == 0:
                        print_time_estimate(i)
                return outcomes
            elif method == 'pubeval':
                outcomes = []
                # Add the evaluation code for pubeval, the bot has a method make_pubeval_move(board, sym, roll), which can be used to get the best move according to pubeval
                for i in range(1, episodes + 1):
                    sys.stderr.write("[EVAL ] Episode {}".format(i))
                    board = Board.initial_state
                    #print("init:", board, sep="\n")
                    while Board.outcome(board) is None:
                        #print("-"*30)
                        roll = (random.randrange(1,7), random.randrange(1,7))
                        #print(roll)

                        prev_board = tuple(board)
                        board = (self.make_move(sess, board, roll))[0]
                        #print("post p1:", board, sep="\n")

                        #print("."*30)
                        roll = (random.randrange(1,7), random.randrange(1,7))
                        #print(roll)
                        
                        prev_board = tuple(board)
                        board = Eval.make_pubeval_move(board, -1, roll)[0][0:26]
                        #print("post pubeval:", board, sep="\n")

                        
                    #print("*"*30)
                    #print(board)
                    #print("+"*30)
                    sys.stderr.write("\t outcome {}".format(Board.outcome(board)[1]))
                    outcomes.append(Board.outcome(board)[1])
                    sys.stderr.write("\n")

                    if i % 10 == 0:
                        print_time_estimate(i)
                    
                return outcomes
            # elif method == 'dumbmodel':
            #     config_prime = self.config.copy()
            #     config_prime['model_path'] = os.path.join(config_prime['model_storage_path'], 'dumbmodel')
            #     eval_bot = Bot(1, config = config_prime, name = "dumbmodel")
            #     #print(self.config, "\n", config_prime)
            #     outcomes = []
            #     for i in range(1, episodes + 1):
            #         sys.stderr.write("[EVAL ] Episode {}".format(i))
            #         board = Board.initial_state
            #         while Board.outcome(board) is None:
            #             roll = (random.randrange(1,7), random.randrange(1,7))
            #             board = (self.make_move(board, self.p1.get_sym(), roll))[0]
                        
            #             roll = (random.randrange(1,7), random.randrange(1,7))
            #             board = Board.flip(eval_bot.make_move(Board.flip(board), self.p1.get_sym(), roll)[0])
            #         sys.stderr.write("\t outcome {}".format(Board.outcome(board)[1]))
            #         outcomes.append(Board.outcome(board)[1])
            #         sys.stderr.write("\n")

            #         if i % 50 == 0:
            #             print_time_estimate(i)
            #     return outcomes
            else:
                sys.stderr.write("[EVAL ] Evaluation method '{}' is not defined\n".format(method))
                return [0]

        with tf.Session() as session:
            session .run(tf.global_variables_initializer())
            self.restore_model(session)
            outcomes = [ (method, do_eval(session,
                                          method,
                                          self.config['episode_count'],
                                          trained_eps = trained_eps))
                         for method
                         in self.config['eval_methods'] ]
        return outcomes