You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
ERROR:tensorflow:Error recorded from training_loop: Checksum does not match: stored 405604006 vs. calculated on the restored bytes 2393749446
[[node checkpoint_initializer_101 (defined at home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
Original stack trace for 'checkpoint_initializer_101':
File "data/roberta_zh/run_classifier.py", line 1059, in
tf.app.run()
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/absl/app.py", line 299, in run
_run_main(main, args)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/absl/app.py", line 250, in _run_main
sys.exit(main(argv))
File "data/roberta_zh/run_classifier.py", line 949, in main
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 3030, in train
saving_listeners=saving_listeners)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 370, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1161, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1191, in _train_model_default
features, labels, ModeKeys.TRAIN, self.config)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 2857, in _call_model_fn
config)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1149, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 3126, in _model_fn
features, labels, is_export_mode=is_export_mode)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 1663, in call_without_tpu
return self._call_model_fn(features, labels, is_export_mode=is_export_mode)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 1994, in _call_model_fn
estimator_spec = self._model_fn(features=features, **kwargs)
File "data/roberta_zh/run_classifier.py", line 721, in model_fn
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 291, in init_from_checkpoint
init_from_checkpoint_fn)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 1940, in merge_call
return self._merge_call(merge_fn, args, kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 1947, in _merge_call
return merge_fn(self._strategy, *args, **kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 286, in
ckpt_dir_or_file, assignment_map)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 334, in _init_from_checkpoint
_set_variable_or_list_initializer(var, ckpt_file, tensor_name_in_ckpt)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 458, in _set_variable_or_list_initializer
_set_checkpoint_initializer(variable_or_list, ckpt_file, tensor_name, "")
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 412, in _set_checkpoint_initializer
ckpt_file, [tensor_name], [slice_spec], [base_type], name=name)[0]
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_io_ops.py", line 1696, in restore_v2
name=name)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py", line 1748, in init
self._traceback = tf_stack.extract_stack()
E0514 14:59:55.399166 140287376860992 error_handling.py:75] Error recorded from training_loop: Checksum does not match: stored 405604006 vs. calculated on the restored bytes 2393749446
[[node checkpoint_initializer_101 (defined at home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
The text was updated successfully, but these errors were encountered:
ERROR:tensorflow:Error recorded from training_loop: Checksum does not match: stored 405604006 vs. calculated on the restored bytes 2393749446
[[node checkpoint_initializer_101 (defined at home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
Original stack trace for 'checkpoint_initializer_101':
File "data/roberta_zh/run_classifier.py", line 1059, in
tf.app.run()
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/absl/app.py", line 299, in run
_run_main(main, args)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/absl/app.py", line 250, in _run_main
sys.exit(main(argv))
File "data/roberta_zh/run_classifier.py", line 949, in main
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 3030, in train
saving_listeners=saving_listeners)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 370, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1161, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1191, in _train_model_default
features, labels, ModeKeys.TRAIN, self.config)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 2857, in _call_model_fn
config)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/estimator.py", line 1149, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 3126, in _model_fn
features, labels, is_export_mode=is_export_mode)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 1663, in call_without_tpu
return self._call_model_fn(features, labels, is_export_mode=is_export_mode)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/tpu/tpu_estimator.py", line 1994, in _call_model_fn
estimator_spec = self._model_fn(features=features, **kwargs)
File "data/roberta_zh/run_classifier.py", line 721, in model_fn
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 291, in init_from_checkpoint
init_from_checkpoint_fn)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 1940, in merge_call
return self._merge_call(merge_fn, args, kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/distribute/distribute_lib.py", line 1947, in _merge_call
return merge_fn(self._strategy, *args, **kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 286, in
ckpt_dir_or_file, assignment_map)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 334, in _init_from_checkpoint
_set_variable_or_list_initializer(var, ckpt_file, tensor_name_in_ckpt)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 458, in _set_variable_or_list_initializer
_set_checkpoint_initializer(variable_or_list, ckpt_file, tensor_name, "")
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/training/checkpoint_utils.py", line 412, in _set_checkpoint_initializer
ckpt_file, [tensor_name], [slice_spec], [base_type], name=name)[0]
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_io_ops.py", line 1696, in restore_v2
name=name)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py", line 1748, in init
self._traceback = tf_stack.extract_stack()
E0514 14:59:55.399166 140287376860992 error_handling.py:75] Error recorded from training_loop: Checksum does not match: stored 405604006 vs. calculated on the restored bytes 2393749446
[[node checkpoint_initializer_101 (defined at home/work/anaconda3/envs/tf1.x/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py:1748) ]]
The text was updated successfully, but these errors were encountered: