From 469d5a631d77d135f055d3aa012ac852a0ef0856 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sat, 17 Sep 2022 23:06:57 +0300 Subject: test_bf16 --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index bae26de..05da6ed 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -253,7 +253,7 @@ for c in req_grad: transpose = [(False, True), (False, False)] str_transpose = ["NT", "NN"] -dtype = [torch.float16] +dtype = [torch.float16, torch.bfloat16] has_fp16_weights = [True, False] has_bias = [True, False] values = list( -- cgit v1.2.3 From a9c7953e0a68a934a18a9495b20deeed9665b2a6 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sat, 17 Sep 2022 23:10:21 +0300 Subject: cast to half before double_quant --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 05da6ed..636fe86 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -354,7 +354,7 @@ def test_matmullt( state.SCB, SCBt, coo_tensorB, - ) = bnb.functional.double_quant(B2) + ) = bnb.functional.double_quant(B2.half()) B2 = state.CB if not transpose[0] and transpose[1]: -- cgit v1.2.3 From 140cdbe8767247bb9b8ea510755cceaa304b6859 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sat, 17 Sep 2022 23:12:58 +0300 Subject: check dtypes first --- tests/test_autograd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 636fe86..083d465 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -354,7 +354,7 @@ def test_matmullt( state.SCB, SCBt, coo_tensorB, - ) = bnb.functional.double_quant(B2.half()) + ) = bnb.functional.double_quant(B2.to(torch.float16)) B2 = state.CB if not transpose[0] and transpose[1]: @@ -367,6 +367,8 @@ def test_matmullt( if has_bias: out_torch += bias + assert out_bnb.dtype == torch.dtype + n = out_bnb.numel() err = torch.abs(out_bnb - out_torch).mean().item() # print(f'abs error {err:.4f}') -- cgit v1.2.3 From 9379df85d223dff18f0fa4adbaf60770700b262a Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sat, 17 Sep 2022 23:13:23 +0300 Subject: check dtypes first --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 083d465..c47754b 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -367,7 +367,7 @@ def test_matmullt( if has_bias: out_torch += bias - assert out_bnb.dtype == torch.dtype + assert out_bnb.dtype == out_torch.dtype n = out_bnb.numel() err = torch.abs(out_bnb - out_torch).mean().item() -- cgit v1.2.3 From e29c5f5c41627668c650a2849e29599cd4f0bf3a Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sat, 17 Sep 2022 23:22:04 +0300 Subject: clearer assertions --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index c47754b..5171c4f 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -367,7 +367,7 @@ def test_matmullt( if has_bias: out_torch += bias - assert out_bnb.dtype == out_torch.dtype + assert out_bnb.dtype == A.dtype, f"bnb matmullt received {A.dtype} but returned {out_bnb.dtype}" n = out_bnb.numel() err = torch.abs(out_bnb - out_torch).mean().item() -- cgit v1.2.3 From 7facedda38da928843e9ed0de1810d45ce1b9224 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sat, 17 Sep 2022 23:41:40 +0300 Subject: copypaste tolerances --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 5171c4f..28d9259 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -427,4 +427,4 @@ def test_matmullt( ) if req_grad[2]: - torch.testing.assert_allclose(gradBias1, gradBias2) + torch.testing.assert_allclose(gradBias1, gradBias2, atol=0.18, rtol=0.3) -- cgit v1.2.3 From d9ca0ed9051a21295e9be80ec08a6589ebd98222 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sat, 17 Sep 2022 23:44:28 +0300 Subject: un-fuse bias --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 28d9259..5171c4f 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -427,4 +427,4 @@ def test_matmullt( ) if req_grad[2]: - torch.testing.assert_allclose(gradBias1, gradBias2, atol=0.18, rtol=0.3) + torch.testing.assert_allclose(gradBias1, gradBias2) -- cgit v1.2.3 From e35e2c665a69647d829c48e22fba0230180c11e7 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sun, 18 Sep 2022 00:35:03 +0300 Subject: cast properly --- tests/test_autograd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 5171c4f..4e4282a 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -372,8 +372,10 @@ def test_matmullt( n = out_bnb.numel() err = torch.abs(out_bnb - out_torch).mean().item() # print(f'abs error {err:.4f}') + out_error_rate = 0.0175 if dtype == torch.float16 else 0.02 + idx = torch.isclose(out_bnb, out_torch, atol=0.01, rtol=0.1) - assert (idx == 0).sum().item() <= n * 0.0175 + assert (idx == 0).sum().item() <= n * out_error_rate idx = torch.isclose(out_bnb, out_torch, atol=0.035, rtol=0.2) assert (idx == 0).sum().item() <= n * 0.001 -- cgit v1.2.3 From fa8e07c7c5186e18d9e2d45042814fe6e8d76d5a Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sun, 18 Sep 2022 00:38:02 +0300 Subject: more lenient threshold --- tests/test_autograd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 4e4282a..0150fbb 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -372,10 +372,9 @@ def test_matmullt( n = out_bnb.numel() err = torch.abs(out_bnb - out_torch).mean().item() # print(f'abs error {err:.4f}') - out_error_rate = 0.0175 if dtype == torch.float16 else 0.02 idx = torch.isclose(out_bnb, out_torch, atol=0.01, rtol=0.1) - assert (idx == 0).sum().item() <= n * out_error_rate + assert (idx == 0).sum().item() <= n * (0.0175 if dtype == torch.float16 else 0.02) idx = torch.isclose(out_bnb, out_torch, atol=0.035, rtol=0.2) assert (idx == 0).sum().item() <= n * 0.001 -- cgit v1.2.3 From f6670329fb9b26dc5547cfef6da73bea75c548ca Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sun, 18 Sep 2022 00:42:23 +0300 Subject: bump threshold to 0.21 --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 0150fbb..cb82898 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -374,7 +374,7 @@ def test_matmullt( # print(f'abs error {err:.4f}') idx = torch.isclose(out_bnb, out_torch, atol=0.01, rtol=0.1) - assert (idx == 0).sum().item() <= n * (0.0175 if dtype == torch.float16 else 0.02) + assert (idx == 0).sum().item() <= n * (0.0175 if dtype == torch.float16 else 0.021) idx = torch.isclose(out_bnb, out_torch, atol=0.035, rtol=0.2) assert (idx == 0).sum().item() <= n * 0.001 -- cgit v1.2.3 From 76ce9aa6da7d68d2463f0f3e99532ab5b6db58a8 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Tue, 20 Sep 2022 06:51:25 +0300 Subject: try fp32 --- tests/test_autograd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_autograd.py') diff --git a/tests/test_autograd.py b/tests/test_autograd.py index cb82898..40bb441 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -253,7 +253,7 @@ for c in req_grad: transpose = [(False, True), (False, False)] str_transpose = ["NT", "NN"] -dtype = [torch.float16, torch.bfloat16] +dtype = [torch.float16, torch.bfloat16, torch.float32] has_fp16_weights = [True, False] has_bias = [True, False] values = list( -- cgit v1.2.3