126 lines
5.8 KiB
Python
126 lines
5.8 KiB
Python
import tensorflow as tf
|
||
from cup import Cup
|
||
import numpy as np
|
||
from board import Board
|
||
#from game import Game
|
||
import os
|
||
|
||
class Config():
|
||
hidden_size = 40
|
||
input_size = 26
|
||
output_size = 1
|
||
# Can't remember the best learning_rate, look this up
|
||
learning_rate = 0.3
|
||
checkpoint_path = "/tmp/"
|
||
|
||
|
||
class Network:
|
||
|
||
# TODO: Actually compile tensorflow properly
|
||
#os.environ["TF_CPP_MIN_LOG_LEVEL"]="2"
|
||
|
||
def custom_tanh(self, x, name=None):
|
||
a = tf.Variable(2.00, tf.float32)
|
||
return tf.scalar_mul(a, tf.tanh(x, name))
|
||
|
||
|
||
def __init__(self, session):
|
||
self.session = session
|
||
self.config = Config
|
||
input_size = self.config.input_size
|
||
hidden_size = self.config.hidden_size
|
||
output_size = self.config.output_size
|
||
learning_rate = self.config.learning_rate
|
||
self.checkpoint_path = self.config.checkpoint_path
|
||
|
||
# input = x
|
||
self.x = tf.placeholder('float', [1,input_size], name='x')
|
||
self.value_next = tf.placeholder('float', [1,output_size], name="value_next")
|
||
|
||
xavier_init = tf.contrib.layers.xavier_initializer()
|
||
|
||
W_1 = tf.Variable(xavier_init((input_size, hidden_size)))
|
||
W_2 = tf.Variable(xavier_init((hidden_size, output_size)))
|
||
|
||
b_1 = tf.zeros(hidden_size,)
|
||
b_2 = tf.zeros(output_size,)
|
||
|
||
value_after_input = self.custom_tanh(tf.matmul(self.x, W_1) + b_1, name='hidden_layer')
|
||
|
||
# TODO: Remember to make this tanh * 2
|
||
self.value = self.custom_tanh(tf.matmul(value_after_input, W_2) + b_2, name='output_layer')
|
||
|
||
# tf.reduce_sum basically finds the sum of it's input, so this gives the difference between the two values, in case they should be lists, which they might be if our input changes
|
||
difference_in_values = tf.reduce_sum(self.value_next - self.value, name='difference')
|
||
|
||
trainable_vars = tf.trainable_variables()
|
||
gradients = tf.gradients(self.value, trainable_vars)
|
||
|
||
apply_gradients = []
|
||
|
||
with tf.variable_scope('apply_gradients'):
|
||
for gradient, trainable_var in zip(gradients, trainable_vars):
|
||
# Hopefully this is Δw_t = α(V_t+1 - V_t)▿_wV_t.
|
||
backprop_calc = learning_rate * difference_in_values * gradient
|
||
grad_apply = trainable_var.assign_add(backprop_calc)
|
||
apply_gradients.append(grad_apply)
|
||
|
||
self.training_op = tf.group(*apply_gradients, name='training_op')
|
||
|
||
self.saver = tf.train.Saver(max_to_keep=1)
|
||
self.session.run(tf.global_variables_initializer())
|
||
|
||
def eval_state(self, state):
|
||
# Run state through a network
|
||
|
||
# Remember to create placeholders for everything because wtf tensorflow and graphs
|
||
|
||
# Remember to create the dense layers
|
||
|
||
# Figure out a way of giving a layer a custom activiation function (we want something which gives [-2,2]. Naively tahn*2, however I fell this is wrong.
|
||
|
||
# tf.group, groups a bunch of actions, so calculate the different gradients for the different weights, by using tf.trainable_variables() to find all variables and tf.gradients(current_value, trainable_variables) to find all the gradients. We can then loop through this and calculate the trace for each gradient and variable pair (note, zip can be used to combine the two lists found before), and then we can calculate the overall change in weights, based on the formula listed in tesauro (learning_rate * difference_in_values * trace), this calculation can be assigned to a tf variable and put in a list and then this can be grouped into a single operation, essentially building our own backprop function.
|
||
# Grouping them is done by tf.group(*the_gradients_from_before_we_want_to_apply, name="training_op")
|
||
|
||
# If we remove the eligibily trace to begin with, we only have
|
||
# to implement learning_rate * (difference_in_values) * gradients (the before-mentioned calculation.
|
||
|
||
|
||
# print("Network is evaluating")
|
||
val = self.session.run(self.value, feed_dict={self.x: state})
|
||
|
||
return val
|
||
|
||
def save_model(self):
|
||
self.saver.save(self.session, self.checkpoint_path + 'model.ckpt')
|
||
|
||
def restore_model(self):
|
||
if os.path.isfile(self.checkpoint_path):
|
||
latest_checkpoint = tf.train.latest_checkpoint(self.checkpoint_path)
|
||
self.saver.restore(self.session, latest_checkpoint)
|
||
|
||
# Have a circular dependency, #fuck, need to rewrite something
|
||
def train(self, x, v_next):
|
||
# print("lol")
|
||
x = np.array(x).reshape((1,26))
|
||
self.session.run(self.training_op, feed_dict = {self.x:x, self.value_next: v_next})
|
||
|
||
|
||
# while game isn't done:
|
||
#x_next = g.next_move()
|
||
#value_next = network.eval_state(x_next)
|
||
#self.session.run(self.training_op, feed_dict={self.x: x, self.value_next: value_next})
|
||
#x = x_next
|
||
|
||
|
||
|
||
|
||
|
||
# take turn, which finds the best state and picks it, based on the current network
|
||
# save current state
|
||
# run training operation (session.run(self.training_op, {x:x, value_next, value_next})), (something which does the backprop, based on the state after having taken a turn, found before, and the state we saved in the beginning and from now we'll save it at the end of the turn
|
||
# save the current state again, so we can continue running backprop based on the "previous" turn.
|
||
|
||
# NOTE: We need to make a method so that we can take a single turn or at least just pick the next best move, so we know how to evaluate according to TD-learning. Right now, our game just continues in a while loop without nothing to stop it!
|
||
|