Evaluation Error

[impulsecorp]

I tried it on 2 different Linux servers, and both give the same error during the evaluation phase:

INFO:tensorflow:Saving checkpoints for 153980 into results/1/model.ckpt.
INFO:tensorflow:Loss for final step: 2.1530373.
Training complete. Evaluating...
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2019-02-10-12:14:47
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from results/1/model.ckpt-153980
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Simplification of result y0 failed. Saving representations of non-simplified formula.

Error: results/1/graph_y0: syntax error in line 11 near ''

---

OutOfRangeError Traceback (most recent call last)
/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1333 try:
-> 1334 return fn(*args)
1335 except errors.OpError as e:

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1318 return self._call_tf_sessionrun(
-> 1319 options, feed_dict, fetch_list, target_list, run_metadata)
1320

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1406 self._session, options, feed_dict, fetch_list, target_list,
-> 1407 run_metadata)
1408

OutOfRangeError: End of sequence
[[{{node IteratorGetNext}} = IteratorGetNextoutput_shapes=[[?,138], [?,1]], output_types=[DT_FLOAT, DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

During handling of the above exception, another exception occurred:

OutOfRangeError Traceback (most recent call last)
/usr/local/lib/python3.6/site-packages/tensorflow/python/training/evaluation.py in _evaluate_once(checkpoint_path, master, scaffold, eval_ops, feed_dict, final_ops, final_ops_feed_dict, hooks, config)
273 while not session.should_stop():
--> 274 session.run(eval_ops, feed_dict)
275

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in run(self, fetches, feed_dict, options, run_metadata)
670 options=options,
--> 671 run_metadata=run_metadata)
672

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in run(self, fetches, feed_dict, options, run_metadata)
1155 options=options,
-> 1156 run_metadata=run_metadata)
1157 except _PREEMPTION_ERRORS as e:

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in run(self, *args, **kwargs)
1254 else:
-> 1255 raise six.reraise(*original_exc_info)
1256

/usr/local/lib/python3.6/site-packages/six.py in reraise(tp, value, tb)
692 raise value.with_traceback(tb)
--> 693 raise value
694 finally:

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in run(self, *args, **kwargs)
1239 try:
-> 1240 return self._sess.run(*args, **kwargs)
1241 except _PREEMPTION_ERRORS:

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in run(self, fetches, feed_dict, options, run_metadata)
1311 options=options,
-> 1312 run_metadata=run_metadata)
1313

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in run(self, *args, **kwargs)
1075 def run(self, *args, **kwargs):
-> 1076 return self._sess.run(*args, **kwargs)
1077

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
928 result = self._run(None, fetches, feed_dict, options_ptr,
--> 929 run_metadata_ptr)
930 if run_metadata:

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1151 results = self._do_run(handle, final_targets, final_fetches,
-> 1152 feed_dict_tensor, options, run_metadata)
1153 else:

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1327 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1328 run_metadata)
1329 else:

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1347 message = error_interpolation.interpolate(message, self._graph)
-> 1348 raise type(e)(node_def, op, message)
1349

OutOfRangeError: End of sequence
[[node IteratorGetNext (defined at /home/ubuntu/new/automl/earthquakes2019/EQL_Tensorflow/data_utils.py:110) = IteratorGetNextoutput_shapes=[[?,138], [?,1]], output_types=[DT_FLOAT, DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

Caused by op 'IteratorGetNext', defined at:
File "/usr/local/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/usr/local/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in
app.launch_new_instance()
File "/usr/local/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 505, in start
self.io_loop.start()
File "/usr/local/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 132, in start
self.asyncio_loop.run_forever()
File "/usr/local/lib/python3.6/asyncio/base_events.py", line 438, in run_forever
self._run_once()
File "/usr/local/lib/python3.6/asyncio/base_events.py", line 1451, in _run_once
handle._run()
File "/usr/local/lib/python3.6/asyncio/events.py", line 145, in _run
self._callback(*self._args)
File "/usr/local/lib/python3.6/site-packages/tornado/ioloop.py", line 758, in _run_callback
ret = callback()
File "/usr/local/lib/python3.6/site-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1233, in inner
self.run()
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1147, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 370, in dispatch_queue
yield self.process_one()
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 346, in wrapper
runner = Runner(result, future, yielded)
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1080, in init
self.run()
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1147, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 357, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 326, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 267, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 326, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 534, in execute_request
user_expressions, allow_stdin,
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 326, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 294, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2819, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2845, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.6/site-packages/IPython/core/async_helpers.py", line 67, in _pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3020, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3185, in run_ast_nodes
if (yield from self.run_code(code, result)):
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3267, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 23, in
val_results = eqlearner.evaluate(input_fn=val_input, name='validation', hooks=[evaluation_hook])
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 478, in evaluate
return _evaluate()
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 460, in _evaluate
self._evaluate_build_graph(input_fn, hooks, checkpoint_path))
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1484, in _evaluate_build_graph
self._call_model_fn_eval(input_fn, self.config))
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1517, in _call_model_fn_eval
input_fn, model_fn_lib.ModeKeys.EVAL)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1075, in _get_features_and_labels_from_input_fn
self._call_input_fn(input_fn, mode))
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 1162, in _call_input_fn
return input_fn(**kwargs)
File "/home/ubuntu/new/automl/earthquakes2019/EQL_Tensorflow/data_utils.py", line 152, in
val_input = lambda: input_from_data(data=val_data, batch_size=batch_size, repeats=1)
File "/home/ubuntu/new/automl/earthquakes2019/EQL_Tensorflow/data_utils.py", line 110, in input_from_data
xs, ys = ds.make_one_shot_iterator().get_next()
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/data/ops/iterator_ops.py", line 421, in get_next
name=name)), self._output_types,
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gen_dataset_ops.py", line 2069, in iterator_get_next
output_shapes=output_shapes, name=name)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1770, in init
self._traceback = tf_stack.extract_stack()

OutOfRangeError (see above for traceback): End of sequence
[[node IteratorGetNext (defined at /home/ubuntu/new/automl/earthquakes2019/EQL_Tensorflow/data_utils.py:110) = IteratorGetNextoutput_shapes=[[?,138], [?,1]], output_types=[DT_FLOAT, DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

During handling of the above exception, another exception occurred:

CalledProcessError Traceback (most recent call last)
in
21 eqlearner.train(input_fn=train_input, hooks=[logging_hook])
22 print('Training complete. Evaluating...')
---> 23 val_results = eqlearner.evaluate(input_fn=val_input, name='validation', hooks=[evaluation_hook])
24 results = dict(val_error=val_results['loss'], complexity=evaluation_hook.get_complexity())
25 if test_input is not None: # test_input function is only provided if extrapolation data is given

/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py in evaluate(self, input_fn, steps, hooks, checkpoint_path, name)
476 return _evaluate()
477 else:
--> 478 return _evaluate()
479
480 def _convert_eval_steps_to_hooks(self, steps):

/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py in _evaluate()
465 eval_dict=eval_dict,
466 all_hooks=all_hooks,
--> 467 output_dir=self.eval_dir(name))
468
469 with ops.Graph().as_default():

/usr/local/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py in _evaluate_run(self, checkpoint_path, scaffold, update_op, eval_dict, all_hooks, output_dir)
1589 final_ops=eval_dict,
1590 hooks=all_hooks,
-> 1591 config=self._session_config)
1592
1593 current_global_step = eval_results[ops.GraphKeys.GLOBAL_STEP]

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/evaluation.py in _evaluate_once(checkpoint_path, master, scaffold, eval_ops, feed_dict, final_ops, final_ops_feed_dict, hooks, config)
272 if eval_ops is not None:
273 while not session.should_stop():
--> 274 session.run(eval_ops, feed_dict)
275
276 logging.info('Finished evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S',

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in exit(self, exception_type, exception_value, traceback)
781 if exception_type in [errors.OutOfRangeError, StopIteration]:
782 exception_type = None
--> 783 self._close_internal(exception_type)
784 # exit should return True to suppress an exception.
785 return exception_type is None

/usr/local/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py in _close_internal(self, exception_type)
814 if not exception_type:
815 for h in self._hooks:
--> 816 h.end(self._coordinated_creator.tf_sess)
817 finally:
818 try:

~/new/automl/earthquakes2019/EQL_Tensorflow/evaluation.py in end(self, session)
62 self.complexity = calculate_complexity(kernels, biases, self.fns_list, self.thresh)
63 if self.generate_symbolic_expr:
---> 64 save_symbolic_expression(kernels, biases, self.fns_list, self.store_path, self.round_decimals)
65
66 def init_network_structure(self, model, params):

~/new/automl/earthquakes2019/EQL_Tensorflow/evaluation.py in save_symbolic_expression(kernels, biases, fns_list, save_path, round_decimals)
173 print('Simplification of result y%i failed. Saving representations of non-simplified formula.' % i)
174 expr_to_latex_png(res, path.join(save_path, 'latex_y' + str(i) + '.png'))
--> 175 expression_graph_as_png(result, path.join(save_path, 'graph_y' + str(i) + '.png'), view=False)
176
177

~/new/automl/earthquakes2019/EQL_Tensorflow/evaluation.py in expression_graph_as_png(expr, output_file, view)
130 graph = Source(dotprint(expr))
131 graph.format = 'png'
--> 132 graph.render(output_file.rpartition('.png')[0], view=view, cleanup=True)
133
134

/usr/local/lib/python3.6/site-packages/graphviz/files.py in render(self, filename, directory, view, cleanup, format, renderer, formatter)
186 format = self._format
187
--> 188 rendered = backend.render(self._engine, format, filepath, renderer, formatter)
189
190 if cleanup:

/usr/local/lib/python3.6/site-packages/graphviz/backend.py in render(engine, format, filepath, renderer, formatter, quiet)
181 """
182 cmd, rendered = command(engine, format, filepath, renderer, formatter)
--> 183 run(cmd, capture_output=True, check=True, quiet=quiet)
184 return rendered
185

/usr/local/lib/python3.6/site-packages/graphviz/backend.py in run(cmd, input, capture_output, check, quiet, **kwargs)
157 stderr_write_bytes(err, flush=True)
158 if check and proc.returncode:
--> 159 raise CalledProcessError(proc.returncode, cmd, output=out, stderr=err)
160
161 return out, err

CalledProcessError: Command '['dot', '-Tpng', '-O', 'results/1/graph_y0']' returned non-zero exit status 1.

[mrolinek]

Thanks!

We might have an idea where this is coming from.

We will be issuing a fix shortly!