Tensorflow - How to implement random search hyperparameters?

Question

Consider this simple definition of a schedule + session. Suppose I want to configure hyperparameters (learning speed and exclude probability) in a random search? What is the recommended way to implement it?

graph = tf.Graph()
with graph.as_default():

    # Placeholders
    data = tf.placeholder(tf.float32,shape=(None,  img_h, img_w, num_channels),name='data')
    labels = ...
    dropout_keep_prob = tf.placeholder(tf.float32, name='keep_prob')
    learning_rate = tf.placeholder(tf.float32, name='learning_rate')

    # model architecture...

with tf.Session(graph=graph) as session:
    tf.initialize_all_variables().run()
    for step in range(num_steps):
        offset = (step * batch_size) % (train_length.shape[0] - batch_size)
        # Generate a minibatch.
        batch_data = train_images[offset:(offset + batch_size), :]
        #...
        feed_train = {data: batch_data, 
                      #...
                      learning_rate: 0.001,
                      keep_prob : 0.7
                     }

I tried putting everything inside a function

def run_model(learning_rate,keep_prob):
    graph = tf.Graph()
    with graph.as_default():
    # graph here...

    with tf.Session(graph=graph) as session:
        tf.initialize_all_variables().run()
        # session here...

But I ran into area problems (I'm not very familiar with areas in Python / Tensoflow). Is there any best practice to achieve this?

Answer 1

, . , , . , , . . .

def generate_random_hyperparams(lr_min, lr_max, kp_min, kp_max):
    '''generate random learning rate and keep probability'''
    # random search through log space for learning rate
    random_learng_rate = 10**np.random.uniform(lr_min, lr_max)
    random_keep_prob = np.random.uniform(kp_min, kp_max)
    return random_learning_rate, random_keep_prob

, , ( , ) ... , run_model .

def run_model(random_learning_rate,random_keep_prob):
    # Note that the arguments is named differently from the placeholders in the graph
    graph = tf.Graph()
    with graph.as_default():
        # graph here...
        learning_rate = tf.placeholder(tf.float32, name='learning_rate')
        keep_prob = tf.placeholder(tf.float32, name='keep_prob')
        # other operation ...

    with tf.Session(graph=graph) as session:
        tf.initialize_all_variables().run()
        # session here...
        feed_train = {data: batch_data, 
                  #placeholder variable names as dict key, python value variables as dict value
                  learning_rate: random_learning_rate,
                  keep_prob : random_keep_prob
                 }
        # evaluate performance with random_learning_rate and random_keep_prob
        performance = session.run([...], feed_dict = feed_train)
    return performance

, tf.placeholders , python.

:

performance_records = {}
for i in range(10): # random search hyper-parameter space 10 times
    random_learning_rate, random_keep_prob = generate_random_hyperparams(-5, -1, 0.2, 0.8)
    performance = run_model(random_learning_rate, random_keep_prob)
    performance_records[(random_learning_rate, random_keep_prob)] = performance