From a2b620b43fd20908a391fedd4bcd7b99e1290827 Mon Sep 17 00:00:00 2001 From: gouzi <530971494@qq.com> Date: Thu, 23 Nov 2023 00:12:08 +0800 Subject: [PATCH 1/5] add `test_legacy_and_pir_exe_and_pir_api` --- test/dygraph_to_static/test_deepcopy.py | 9 +- test/dygraph_to_static/test_loop.py | 58 ++++---- .../dygraph_to_static/test_partial_program.py | 133 +++++++++--------- test/dygraph_to_static/test_ptb_lm_v2.py | 21 +-- .../test_set_dynamic_shape.py | 5 +- 5 files changed, 118 insertions(+), 108 deletions(-) diff --git a/test/dygraph_to_static/test_deepcopy.py b/test/dygraph_to_static/test_deepcopy.py index 5d281ba8ea213..3dc77879447b6 100644 --- a/test/dygraph_to_static/test_deepcopy.py +++ b/test/dygraph_to_static/test_deepcopy.py @@ -16,7 +16,10 @@ from copy import deepcopy import numpy as np -from dygraph_to_static_utils_new import Dy2StTestBase, test_legacy_and_pir +from dygraph_to_static_utils_new import ( + Dy2StTestBase, + test_legacy_and_pir_exe_and_pir_api, +) from test_rollback import Net, foo import paddle @@ -24,7 +27,7 @@ class TestDeepCopy(Dy2StTestBase): - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test_net(self): net = Net() net = paddle.jit.to_static(net) @@ -40,7 +43,7 @@ def test_net(self): self.assertTrue(id(copy_net), id(copy_net.forward.__self__)) np.testing.assert_array_equal(src_out.numpy(), copy_out.numpy()) - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test_func(self): st_foo = paddle.jit.to_static(foo) x = paddle.randn([3, 4]) diff --git a/test/dygraph_to_static/test_loop.py b/test/dygraph_to_static/test_loop.py index 3aefa231d6d27..297660e38981c 100644 --- a/test/dygraph_to_static/test_loop.py +++ b/test/dygraph_to_static/test_loop.py @@ -13,10 +13,15 @@ # limitations under the License. import inspect +import tempfile import unittest import numpy as np -from dygraph_to_static_utils_new import Dy2StTestBase +from dygraph_to_static_utils_new import ( + Dy2StTestBase, + test_legacy_and_pir, + test_legacy_and_pir_exe_and_pir_api, +) import paddle import paddle.nn.functional as F @@ -248,6 +253,7 @@ def setUp(self): self.nested_for_loop_func = nested_for_loop_dyfunc + @test_legacy_and_pir_exe_and_pir_api def test_loop_vars(self): for i in range(len(self.loop_funcs)): func = self.loop_funcs[i] @@ -263,6 +269,7 @@ def test_loop_vars(self): self.assertEqual(loop_var_names, self.loop_var_names[i]) self.assertEqual(create_var_names, self.create_var_names[i]) + @test_legacy_and_pir_exe_and_pir_api def test_nested_loop_vars(self): func = self.nested_for_loop_func test_func = inspect.getsource(func) @@ -303,9 +310,9 @@ def test_nested_loop_vars(self): class TestTransformWhileLoop(Dy2StTestBase): def setUp(self): self.place = ( - base.CUDAPlace(0) - if base.is_compiled_with_cuda() - else base.CPUPlace() + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() ) self.x = np.zeros(shape=(1), dtype=np.int32) self._init_dyfunc() @@ -320,18 +327,18 @@ def _run_dygraph(self): return self._run(to_static=False) def _run(self, to_static): - with base.dygraph.guard(self.place): - # Set the input of dyfunc to Tensor - tensor_x = base.dygraph.to_variable(self.x, zero_copy=False) - if to_static: - ret = paddle.jit.to_static(self.dyfunc)(tensor_x) - else: - ret = self.dyfunc(tensor_x) - if hasattr(ret, "numpy"): - return ret.numpy() - else: - return ret + # Set the input of dyfunc to Tensor + tensor_x = base.dygraph.to_variable(self.x, zero_copy=False) + if to_static: + ret = paddle.jit.to_static(self.dyfunc)(tensor_x) + else: + ret = self.dyfunc(tensor_x) + if hasattr(ret, "numpy"): + return ret.numpy() + else: + return ret + @test_legacy_and_pir def test_ast_to_func(self): static_numpy = self._run_static() dygraph_numpy = self._run_dygraph() @@ -382,9 +389,9 @@ def _init_dyfunc(self): class TestTransformForLoop(Dy2StTestBase): def setUp(self): self.place = ( - base.CUDAPlace(0) - if base.is_compiled_with_cuda() - else base.CPUPlace() + paddle.CUDAPlace(0) + if paddle.is_compiled_with_cuda() + else paddle.CPUPlace() ) self.len = 100 self._init_dyfunc() @@ -399,12 +406,11 @@ def _run_dygraph(self): return self._run(to_static=False) def _run(self, to_static): - with base.dygraph.guard(self.place): - if to_static: - ret = paddle.jit.to_static(self.dyfunc)(self.len) - else: - ret = self.dyfunc(self.len) - return ret.numpy() + if to_static: + ret = paddle.jit.to_static(self.dyfunc)(self.len) + else: + ret = self.dyfunc(self.len) + return ret.numpy() def test_ast_to_func(self): np.testing.assert_allclose( @@ -472,7 +478,9 @@ def test_start(self): ) ], ) - paddle.jit.save(model, "./inference/inference") + temp_dir = tempfile.TemporaryDirectory() + paddle.jit.save(model, temp_dir.name) + temp_dir.cleanup() if __name__ == '__main__': diff --git a/test/dygraph_to_static/test_partial_program.py b/test/dygraph_to_static/test_partial_program.py index cc3c5678c4843..ad26be3c60cdb 100644 --- a/test/dygraph_to_static/test_partial_program.py +++ b/test/dygraph_to_static/test_partial_program.py @@ -19,12 +19,12 @@ Dy2StTestBase, test_ast_only, test_legacy_and_pir, + test_legacy_and_pir_exe_and_pir_api, ) from test_fetch_feed import Linear import paddle from paddle import base -from paddle.jit.api import to_static SEED = 2020 @@ -76,20 +76,19 @@ def fake_input(self): ] def _run(self, to_static): - with base.dygraph.guard(): - if self.x is None or self.y is None: - self.fake_input() + if self.x is None or self.y is None: + self.fake_input() - if to_static: - out = paddle.jit.to_static(nested_input, full_graph=True)( - self.x, self.y - ) - else: - out = nested_input(self.x, self.y) + if to_static: + out = paddle.jit.to_static(nested_input, full_graph=True)( + self.x, self.y + ) + else: + out = nested_input(self.x, self.y) return out.numpy() - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test_nest(self): dygraph_res = self._run(to_static=False) static_res = self._run(to_static=True) @@ -102,21 +101,20 @@ def setUp(self): self.y = None def _run(self, to_static): - with base.dygraph.guard(): - if self.x is None or self.y is None: - self.x = fake_data([10, 16]) - self.y = fake_data([10, 16]) - - if to_static: - out = paddle.jit.to_static(nested_output, full_graph=True)( - self.x, self.y - ) - else: - out = nested_output(self.x, self.y) + if self.x is None or self.y is None: + self.x = fake_data([10, 16]) + self.y = fake_data([10, 16]) + + if to_static: + out = paddle.jit.to_static(nested_output, full_graph=True)( + self.x, self.y + ) + else: + out = nested_output(self.x, self.y) return out - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test_nest(self): dygraph_res = self._run(to_static=False) dygraph_res = paddle.utils.flatten(dygraph_res) @@ -139,53 +137,51 @@ class TestWithTrainAndEval(Dy2StTestBase): @test_ast_only @test_legacy_and_pir def test_switch_eval_and_train(self): - with base.dygraph.guard(): - linear_net = Linear() - linear_net = paddle.jit.to_static(linear_net, full_graph=True) - x_data = np.random.random((4, 10)).astype('float32') - x = base.dygraph.to_variable(x_data) - linear_net(x) - - _, train_partial_layer = linear_net.forward.program_cache.last()[-1] - # check default mode is for training - self.assertEqual( - train_partial_layer.program, train_partial_layer._train_program - ) + linear_net = Linear() + linear_net = paddle.jit.to_static(linear_net, full_graph=True) + x_data = np.random.random((4, 10)).astype('float32') + x = base.dygraph.to_variable(x_data) + linear_net(x) + + _, train_partial_layer = linear_net.forward.program_cache.last()[-1] + # check default mode is for training + self.assertEqual( + train_partial_layer.program, train_partial_layer._train_program + ) - # switch to run test program after `eval()` - linear_net.eval() - linear_net(x) - _, eval_partial_layer = linear_net.forward.program_cache.last()[-1] - self.assertEqual( - eval_partial_layer.program, eval_partial_layer._infer_program - ) + # switch to run test program after `eval()` + linear_net.eval() + linear_net(x) + _, eval_partial_layer = linear_net.forward.program_cache.last()[-1] + self.assertEqual( + eval_partial_layer.program, eval_partial_layer._infer_program + ) - # switch back into training - linear_net.train() - linear_net(x) - self.assertEqual( - train_partial_layer.program, train_partial_layer._train_program - ) + # switch back into training + linear_net.train() + linear_net(x) + self.assertEqual( + train_partial_layer.program, train_partial_layer._train_program + ) class TestWithNoGrad(Dy2StTestBase): @test_ast_only @test_legacy_and_pir def test_with_no_grad(self): - with base.dygraph.guard(): - linear_net = Linear() - linear_net = paddle.jit.to_static(linear_net, full_graph=True) - x_data = np.random.random((5, 10)).astype('float32') - x = base.dygraph.to_variable(x_data) - - with paddle.no_grad(): - linear_net.train() - linear_net(x) - # BUG: 我们希望这里 是 ASTStaticFunction(StaticFunction): - _, partial_layer = linear_net.forward.program_cache.last()[-1] - self.assertEqual( - partial_layer.program, partial_layer._train_program - ) + linear_net = Linear() + linear_net = paddle.jit.to_static(linear_net, full_graph=True) + x_data = np.random.random((5, 10)).astype('float32') + x = base.dygraph.to_variable(x_data) + + with paddle.no_grad(): + linear_net.train() + linear_net(x) + # BUG: 我们希望这里 是 ASTStaticFunction(StaticFunction): + _, partial_layer = linear_net.forward.program_cache.last()[-1] + self.assertEqual( + partial_layer.program, partial_layer._train_program + ) class GPT2LMHeadModel(paddle.nn.Layer): @@ -197,7 +193,6 @@ def __init__(self): np.random.rand(2, 3).astype('float32') ) - @to_static(full_graph=True) def forward(self, x): x = paddle.reshape(x, shape=[-1, 6]) x1, x2, x3 = paddle.split(x=x, axis=1, num_or_sections=3) @@ -209,13 +204,11 @@ class TestPruneUnusedParamInProgram(Dy2StTestBase): def test_prune(self): input_ids = np.array([[15, 11, 6, 3, 18, 13]]).astype("float32") - place = base.CPUPlace() - with base.dygraph.guard(place): - model = GPT2LMHeadModel() - model.eval() - input_ids = paddle.to_tensor(input_ids) - out = model(input_ids) - np.testing.assert_array_equal(out.numpy(), [[15, 11]]) + model = paddle.jit.to_static(GPT2LMHeadModel()) + model.eval() + input_ids = paddle.to_tensor(input_ids) + out = model(input_ids) + np.testing.assert_array_equal(out.numpy(), [[15, 11]]) if __name__ == '__main__': diff --git a/test/dygraph_to_static/test_ptb_lm_v2.py b/test/dygraph_to_static/test_ptb_lm_v2.py index abc351d17f1ec..8173cf9b4d037 100644 --- a/test/dygraph_to_static/test_ptb_lm_v2.py +++ b/test/dygraph_to_static/test_ptb_lm_v2.py @@ -17,7 +17,9 @@ import unittest import numpy as np -from dygraph_to_static_utils_new import Dy2StTestBase +from dygraph_to_static_utils_new import ( + Dy2StTestBase, +) import paddle @@ -180,7 +182,6 @@ def __init__( def build_once(self, input, label, init_hidden, init_cell): pass - @paddle.jit.to_static def forward(self, input, label, init_hidden, init_cell): init_h = paddle.reshape( init_hidden, shape=[self.num_layers, -1, self.hidden_size] @@ -235,13 +236,15 @@ def train(place): paddle.disable_static(place) paddle.seed(SEED) paddle.framework.random._manual_program_seed(SEED) - ptb_model = PtbModel( - hidden_size=hidden_size, - vocab_size=vocab_size, - num_layers=num_layers, - num_steps=num_steps, - init_scale=init_scale, - dropout=dropout, + ptb_model = paddle.jit.to_static( + PtbModel( + hidden_size=hidden_size, + vocab_size=vocab_size, + num_layers=num_layers, + num_steps=num_steps, + init_scale=init_scale, + dropout=dropout, + ) ) sgd = paddle.optimizer.SGD( diff --git a/test/dygraph_to_static/test_set_dynamic_shape.py b/test/dygraph_to_static/test_set_dynamic_shape.py index 3a3843846a9a4..bf0a12c18ee89 100644 --- a/test/dygraph_to_static/test_set_dynamic_shape.py +++ b/test/dygraph_to_static/test_set_dynamic_shape.py @@ -14,7 +14,10 @@ import unittest -from dygraph_to_static_utils_new import Dy2StTestBase, test_ast_only +from dygraph_to_static_utils_new import ( + Dy2StTestBase, + test_ast_only, +) import paddle From 8e39f8b6631d4650430405a68e3254ebe28362bb Mon Sep 17 00:00:00 2001 From: SigureMo Date: Thu, 23 Nov 2023 12:11:45 +0000 Subject: [PATCH 2/5] fix some tests --- test/dygraph_to_static/test_loop.py | 2 - .../dygraph_to_static/test_partial_program.py | 57 +++++++++++++++++-- test/dygraph_to_static/test_ptb_lm_v2.py | 4 ++ 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/test/dygraph_to_static/test_loop.py b/test/dygraph_to_static/test_loop.py index 297660e38981c..e0e065d8723bc 100644 --- a/test/dygraph_to_static/test_loop.py +++ b/test/dygraph_to_static/test_loop.py @@ -19,7 +19,6 @@ import numpy as np from dygraph_to_static_utils_new import ( Dy2StTestBase, - test_legacy_and_pir, test_legacy_and_pir_exe_and_pir_api, ) @@ -338,7 +337,6 @@ def _run(self, to_static): else: return ret - @test_legacy_and_pir def test_ast_to_func(self): static_numpy = self._run_static() dygraph_numpy = self._run_dygraph() diff --git a/test/dygraph_to_static/test_partial_program.py b/test/dygraph_to_static/test_partial_program.py index ad26be3c60cdb..705f063beb532 100644 --- a/test/dygraph_to_static/test_partial_program.py +++ b/test/dygraph_to_static/test_partial_program.py @@ -20,6 +20,7 @@ test_ast_only, test_legacy_and_pir, test_legacy_and_pir_exe_and_pir_api, + test_pir_api_only, ) from test_fetch_feed import Linear @@ -125,7 +126,7 @@ def test_nest(self): self.assertTrue(len(dygraph_res) == len(static_res)) for dy_var, st_var in zip(dygraph_res, static_res): - if isinstance(dy_var, base.core.eager.Tensor): + if isinstance(dy_var, paddle.Tensor): np.testing.assert_allclose( dy_var.numpy(), st_var.numpy(), rtol=1e-05 ) @@ -136,7 +137,8 @@ def test_nest(self): class TestWithTrainAndEval(Dy2StTestBase): @test_ast_only @test_legacy_and_pir - def test_switch_eval_and_train(self): + def test_legacy_ir_switch_eval_and_train(self): + # TODO(cleanup-legacy-ir): Remove this test case linear_net = Linear() linear_net = paddle.jit.to_static(linear_net, full_graph=True) x_data = np.random.random((4, 10)).astype('float32') @@ -164,15 +166,47 @@ def test_switch_eval_and_train(self): train_partial_layer.program, train_partial_layer._train_program ) + @test_ast_only + @test_pir_api_only + def test_switch_eval_and_train(self): + linear_net = Linear() + linear_net = paddle.jit.to_static(linear_net, full_graph=True) + x_data = np.random.random((4, 10)).astype('float32') + x = paddle.to_tensor(x_data) + linear_net(x) + + _, train_partial_layer = linear_net.forward.program_cache.last()[-1] + # check default mode is for training + self.assertEqual( + train_partial_layer.program, + train_partial_layer.train_program, + ) + + # switch to run test program after `eval()` + linear_net.eval() + linear_net(x) + _, eval_partial_layer = linear_net.forward.program_cache.last()[-1] + self.assertEqual( + eval_partial_layer.program, eval_partial_layer.infer_program + ) + + # switch back into training + linear_net.train() + linear_net(x) + self.assertEqual( + train_partial_layer.program, train_partial_layer.train_program + ) + class TestWithNoGrad(Dy2StTestBase): @test_ast_only @test_legacy_and_pir - def test_with_no_grad(self): + def test_legacy_ir_with_no_grad(self): + # TODO(cleanup-legacy-ir): Remove this test case linear_net = Linear() linear_net = paddle.jit.to_static(linear_net, full_graph=True) x_data = np.random.random((5, 10)).astype('float32') - x = base.dygraph.to_variable(x_data) + x = paddle.to_tensor(x_data) with paddle.no_grad(): linear_net.train() @@ -183,6 +217,21 @@ def test_with_no_grad(self): partial_layer.program, partial_layer._train_program ) + @test_ast_only + @test_pir_api_only + def test_with_no_grad(self): + linear_net = Linear() + linear_net = paddle.jit.to_static(linear_net, full_graph=True) + x_data = np.random.random((5, 10)).astype('float32') + x = paddle.to_tensor(x_data) + + with paddle.no_grad(): + linear_net.train() + linear_net(x) + # BUG: 我们希望这里 是 ASTStaticFunction(StaticFunction): + _, partial_layer = linear_net.forward.program_cache.last()[-1] + self.assertEqual(partial_layer.program, partial_layer.train_program) + class GPT2LMHeadModel(paddle.nn.Layer): def __init__(self): diff --git a/test/dygraph_to_static/test_ptb_lm_v2.py b/test/dygraph_to_static/test_ptb_lm_v2.py index 8173cf9b4d037..997ad5990a11a 100644 --- a/test/dygraph_to_static/test_ptb_lm_v2.py +++ b/test/dygraph_to_static/test_ptb_lm_v2.py @@ -19,6 +19,8 @@ import numpy as np from dygraph_to_static_utils_new import ( Dy2StTestBase, + test_ast_only, + test_pir_api_only, ) import paddle @@ -334,6 +336,8 @@ def setUp(self): else paddle.CPUPlace() ) + @test_ast_only + @test_pir_api_only def test_check_result(self): loss_1, hidden_1, cell_1 = train_static(self.place) loss_2, hidden_2, cell_2 = train_dygraph(self.place) From db0dad27c2df88468d7d2e5ba243d23ccb4ff753 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Mon, 27 Nov 2023 08:11:53 +0000 Subject: [PATCH 3/5] restore test_ptb_lm_v2 --- test/dygraph_to_static/test_ptb_lm_v2.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/dygraph_to_static/test_ptb_lm_v2.py b/test/dygraph_to_static/test_ptb_lm_v2.py index 997ad5990a11a..8173cf9b4d037 100644 --- a/test/dygraph_to_static/test_ptb_lm_v2.py +++ b/test/dygraph_to_static/test_ptb_lm_v2.py @@ -19,8 +19,6 @@ import numpy as np from dygraph_to_static_utils_new import ( Dy2StTestBase, - test_ast_only, - test_pir_api_only, ) import paddle @@ -336,8 +334,6 @@ def setUp(self): else paddle.CPUPlace() ) - @test_ast_only - @test_pir_api_only def test_check_result(self): loss_1, hidden_1, cell_1 = train_static(self.place) loss_2, hidden_2, cell_2 = train_dygraph(self.place) From 5c995b7df30d0f0c27f5f3b62145ba7a4d7c8178 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Mon, 27 Nov 2023 08:47:21 +0000 Subject: [PATCH 4/5] update build strategy --- test/dygraph_to_static/test_build_strategy.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/dygraph_to_static/test_build_strategy.py b/test/dygraph_to_static/test_build_strategy.py index 18796eebe2c86..deaed3ca6d896 100644 --- a/test/dygraph_to_static/test_build_strategy.py +++ b/test/dygraph_to_static/test_build_strategy.py @@ -19,6 +19,7 @@ Dy2StTestBase, test_ast_only, test_legacy_and_pir, + test_legacy_and_pir_exe_and_pir_api, ) from test_resnet import ResNetHelper @@ -90,6 +91,7 @@ def test_in_static_mode_mkldnn(self): class TestError(Dy2StTestBase): + @test_legacy_and_pir_exe_and_pir_api def test_type_error(self): def foo(x): out = x + 1 From 41b9648c3c07f7f610a596947123dacb5181f855 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Mon, 27 Nov 2023 08:51:05 +0000 Subject: [PATCH 5/5] Revert "update build strategy" This reverts commit 5c995b7df30d0f0c27f5f3b62145ba7a4d7c8178. --- test/dygraph_to_static/test_build_strategy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/dygraph_to_static/test_build_strategy.py b/test/dygraph_to_static/test_build_strategy.py index deaed3ca6d896..18796eebe2c86 100644 --- a/test/dygraph_to_static/test_build_strategy.py +++ b/test/dygraph_to_static/test_build_strategy.py @@ -19,7 +19,6 @@ Dy2StTestBase, test_ast_only, test_legacy_and_pir, - test_legacy_and_pir_exe_and_pir_api, ) from test_resnet import ResNetHelper @@ -91,7 +90,6 @@ def test_in_static_mode_mkldnn(self): class TestError(Dy2StTestBase): - @test_legacy_and_pir_exe_and_pir_api def test_type_error(self): def foo(x): out = x + 1