Problem
When I start my training, my preprocessing examples are successfully created, however my training does not begin. Much stranger is the fact that when I analyze my TensorBoard graph, I see some additional conditional nodes that are not in the code. I want to know where and why these additional nodes are included in the picture, and that is why training does not begin. The following is a systematic description of the situation:
TensorFlow Graph The
following TensorBoard diagram shows my graph:
The code that this graph creates is below
def getconv2drelu(inputtensor, kernelsize, strides, padding, convname,
imagesummaries=False):
weights = tf.get_variable("weights", shape=kernelsize, dtype=tf.float32,
initializer=tf.truncated_normal_initializer(0,
0.01),
regularizer=tf.nn.l2_loss)
biases = tf.get_variable("biases", shape=kernelsize[3], dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
conv = tf.nn.conv2d(input=inputtensor, filter=weights, strides=strides,
padding=padding, name=convname)
response = tf.nn.bias_add(conv, biases)
if imagesummaries:
filters = (weights - tf.reduce_min(weights)) / (tf.reduce_max(
weights) - tf.reduce_min(weights))
filters = tf.transpose(filters, [3, 0, 1, 2])
tf.summary.image(convname + " filters", filters,
max_outputs=kernelsize[3])
response = tf.nn.relu(response)
activation_summary(response)
return response
def getfullyconnected(inputtensor, numinput, numoutput):
weights = tf.get_variable("weights", shape=[numinput, numoutput],
dtype=tf.float32,
initializer=
tf.truncated_normal_initializer(0, 0.01))
biases = tf.get_variable("biases", shape=[numoutput], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(
0, 0.01))
response = tf.add(tf.matmul(inputtensor, weights), biases)
response = tf.nn.relu(response)
activation_summary(response)
return response
def inference(inputs):
with tf.variable_scope("layer1"):
conv = getconv2drelu(inputtensor=inputs, kernelsize=[7, 7, 3, 96],
strides=[1, 2, 2, 1], padding="VALID",
convname="conv1", imagesummaries=True)
pool = tf.nn.max_pool(conv, [1, 3, 3, 1], strides=[1, 3, 3, 1],
padding="SAME", name="pool1")
with tf.variable_scope("layer2"):
conv = getconv2drelu(inputtensor=pool, kernelsize=[7, 7, 96, 256],
strides=[1, 1, 1, 1], padding="VALID",
convname="conv2", imagesummaries=False)
pool = tf.nn.max_pool(conv, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding="SAME", name="pool2")
with tf.variable_scope("layer3"):
conv = getconv2drelu(inputtensor=pool, kernelsize=[7, 7, 256, 512],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv3", imagesummaries=False)
with tf.variable_scope("layer4"):
conv = getconv2drelu(inputtensor=conv, kernelsize=[3, 3, 512, 512],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv4", imagesummaries=False)
with tf.variable_scope("layer5"):
conv = getconv2drelu(inputtensor=conv, kernelsize=[3, 3, 512, 1024],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv5", imagesummaries=False)
with tf.variable_scope("layer6"):
conv = getconv2drelu(inputtensor=conv, kernelsize=[3, 3, 1024, 1024],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv6", imagesummaries=False)
pool = tf.nn.max_pool(conv, [1, 3, 3, 1], strides=[1, 3, 3, 1],
padding="SAME", name="pool1")
pool = tf.contrib.layers.flatten(pool)
with tf.variable_scope("fc1"):
fc = getfullyconnected(pool, 5 * 5 * 1024, 4096)
drop = tf.nn.dropout(fc, keep_prob=0.5)
with tf.variable_scope("fc2"):
fc = getfullyconnected(drop, 4096, 4096)
drop = tf.nn.dropout(fc, keep_prob=0.5)
with tf.variable_scope("fc3"):
logits = getfullyconnected(drop, 4096, 1000)
return logits
The full TensorBoard schedule is shown below:
The drawing is too small, but you can see a row of pink nodes on the left. Below is an expanded version of such a segment:
( !!) :
. [221, 221, 3].
, isVariableInitialized. . , , . , - tf.get_variable(), .
, . , , . ?
:
with tf.control_dependencies(putops):
train_op = tf.group(apply_gradient_op, variables_averages_op)
sess.run(train_op) # tf.Session() as been defined before sess.run()
putops [], :
cpu_compute_stage = data_flow_ops.StagingArea(
[tf.float32, tf.int32],
shapes=[images_shape, labels_shape]
)
cpu_compute_stage_op = gpu_copy_stage.put(
[host_images, host_labels])
putops.append(gpu_copy_stage_op)
with tf.device('/gpu:%d' % i):
with tf.name_scope('%s_%d' % (TOWER_NAME, i)) as scope:
gpu_compute_stage = data_flow_ops.StagingArea(
[tf.float32, tf.int32],
shapes=[images_shape, labels_shape]
)
gpu_compute_stage_op = gpu_compute_stage.put(
[host_images, host_labels]
)
putops.append(gpu_compute_stage_op)
, , , .