I was training DeepSpeech with Common Voice and received the following error:
D Starting batch…
2018-05-03 10:45:58.253682: W tensorflow/core/framework/op_kernel.cc:1202] OP_REQUIRES failed at ctc_loss_op.cc:166 : Invalid argument: Labels length is zero in batch 0
E Labels length is zero in batch 0
E [[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
E
E Caused by op ‘tower_0/CTCLoss’, defined at:
E File “DeepSpeech.py”, line 1838, in
E tf.app.run()
E File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/platform/app.py”, line 126, in run
E _sys.exit(main(argv))
E File “DeepSpeech.py”, line 1795, in main
E train()
E File “DeepSpeech.py”, line 1501, in train
E results_tuple, gradients, mean_edit_distance, loss = get_tower_results(model_feeder, optimizer)
E File “DeepSpeech.py”, line 640, in get_tower_results
E calculate_mean_edit_distance_and_loss(model_feeder, i, no_dropout if optimizer is None else dropout_rates)
E File “DeepSpeech.py”, line 527, in calculate_mean_edit_distance_and_loss
E total_loss = tf.nn.ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len)
E File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/ops/ctc_ops.py”, line 158, in ctc_loss
E ignore_longer_outputs_than_inputs=ignore_longer_outputs_than_inputs)
E File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/ops/gen_ctc_ops.py”, line 212, in _ctc_loss
E name=name)
E File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py”, line 787, in _apply_op_helper
E op_def=op_def)
E File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py”, line 3271, in create_op
E op_def=op_def)
E File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py”, line 1650, in init
E self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
E
E InvalidArgumentError (see above for traceback): Labels length is zero in batch 0
E [[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
E
Traceback (most recent call last):
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1361, in _do_call
return fn(*args)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1340, in _run_fn
target_list, status, run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py”, line 516, in exit
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: Labels length is zero in batch 0
[[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “DeepSpeech.py”, line 1637, in train
_, current_step, batch_loss, batch_report = session.run([train_op, global_step, loss, report_params], **extra_params)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 546, in run
run_metadata=run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1022, in run
run_metadata=run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1113, in run
raise six.reraise(*original_exc_info)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/six.py”, line 693, in reraise
raise value
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1098, in run
return self._sess.run(*args, **kwargs)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1170, in run
run_metadata=run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 950, in run
return self._sess.run(*args, **kwargs)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 905, in run
run_metadata_ptr)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1137, in _run
feed_dict_tensor, options, run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1355, in _do_run
options, run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1374, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Labels length is zero in batch 0
[[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
Caused by op ‘tower_0/CTCLoss’, defined at:
File “DeepSpeech.py”, line 1838, in
tf.app.run()
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/platform/app.py”, line 126, in run
_sys.exit(main(argv))
File “DeepSpeech.py”, line 1795, in main
train()
File “DeepSpeech.py”, line 1501, in train
results_tuple, gradients, mean_edit_distance, loss = get_tower_results(model_feeder, optimizer)
File “DeepSpeech.py”, line 640, in get_tower_results
calculate_mean_edit_distance_and_loss(model_feeder, i, no_dropout if optimizer is None else dropout_rates)
File “DeepSpeech.py”, line 527, in calculate_mean_edit_distance_and_loss
total_loss = tf.nn.ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/ops/ctc_ops.py”, line 158, in ctc_loss
ignore_longer_outputs_than_inputs=ignore_longer_outputs_than_inputs)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/ops/gen_ctc_ops.py”, line 212, in _ctc_loss
name=name)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py”, line 787, in _apply_op_helper
op_def=op_def)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py”, line 3271, in create_op
op_def=op_def)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py”, line 1650, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Labels length is zero in batch 0
[[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
D Closing queues…
D Queues closed.
Traceback (most recent call last):
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1361, in _do_call
return fn(*args)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1340, in _run_fn
target_list, status, run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py”, line 516, in exit
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: Labels length is zero in batch 0
[[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “DeepSpeech.py”, line 1637, in train
_, current_step, batch_loss, batch_report = session.run([train_op, global_step, loss, report_params], **extra_params)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 546, in run
run_metadata=run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1022, in run
run_metadata=run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1113, in run
raise six.reraise(*original_exc_info)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/six.py”, line 693, in reraise
raise value
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1098, in run
return self._sess.run(*args, **kwargs)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 1170, in run
run_metadata=run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py”, line 950, in run
return self._sess.run(*args, **kwargs)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 905, in run
run_metadata_ptr)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1137, in _run
feed_dict_tensor, options, run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1355, in _do_run
options, run_metadata)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/client/session.py”, line 1374, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Labels length is zero in batch 0
[[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
Caused by op ‘tower_0/CTCLoss’, defined at:
File “DeepSpeech.py”, line 1838, in
tf.app.run()
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/platform/app.py”, line 126, in run
_sys.exit(main(argv))
File “DeepSpeech.py”, line 1795, in main
train()
File “DeepSpeech.py”, line 1501, in train
results_tuple, gradients, mean_edit_distance, loss = get_tower_results(model_feeder, optimizer)
File “DeepSpeech.py”, line 640, in get_tower_results
calculate_mean_edit_distance_and_loss(model_feeder, i, no_dropout if optimizer is None else dropout_rates)
File “DeepSpeech.py”, line 527, in calculate_mean_edit_distance_and_loss
total_loss = tf.nn.ctc_loss(labels=batch_y, inputs=logits, sequence_length=batch_seq_len)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/ops/ctc_ops.py”, line 158, in ctc_loss
ignore_longer_outputs_than_inputs=ignore_longer_outputs_than_inputs)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/ops/gen_ctc_ops.py”, line 212, in _ctc_loss
name=name)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py”, line 787, in _apply_op_helper
op_def=op_def)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py”, line 3271, in create_op
op_def=op_def)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py”, line 1650, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Labels length is zero in batch 0
[[Node: tower_0/CTCLoss = CTCLoss[ctc_merge_repeated=true, ignore_longer_outputs_than_inputs=false, preprocess_collapse_repeated=false, _device="/job:localhost/replica:0/task:0/device:CPU:0"](tower_0/logits/_827, tower_0/ToInt64/_829, tower_0/Gather, tower_0/Gather_DequeueMany:1)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “DeepSpeech.py”, line 1838, in
tf.app.run()
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/platform/app.py”, line 126, in run
_sys.exit(main(argv))
File “DeepSpeech.py”, line 1795, in main
train()
File “DeepSpeech.py”, line 1666, in train
hook.end(session)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/basic_session_run_hooks.py”, line 463, in end
self._save(session, last_step)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/basic_session_run_hooks.py”, line 474, in _save
self._get_saver().save(session, self._save_path, global_step=step)
File “/data/venv/deepspeech-venv/lib/python3.6/site-packages/tensorflow/python/training/saver.py”, line 1646, in save
raise TypeError("‘sess’ must be a Session; %s" % sess)
TypeError: ‘sess’ must be a Session; <tensorflow.python.training.monitored_session.MonitoredSession object at 0x7fb59936aac8>
My configuration is:
python3 DeepSpeech.py --n_hidden 2048 --initialize_from_frozen_model models/output_graph.pb --checkpoint_dir fine_tuning_checkpoints_cv --epoch 1 --train_files data/CV/cv-valid-train.csv,data/CV/cv-other-train.csv --dev_files data/CV/cv-valid-dev.csv --test_files data/CV/cv-valid-test.csv --validation_step 1 summary_dir summary —summary_secs 120
I used import_cv.py to import common voice. Can somebody let me know what wrong I’m doing.