diff --git a/deepspeed/module_inject/layers.py b/deepspeed/module_inject/layers.py
index 5f04594b052a..ad398be9f67f 100644
--- a/deepspeed/module_inject/layers.py
+++ b/deepspeed/module_inject/layers.py
@@ -222,7 +222,8 @@ def forward(ctx: Any, group: dist.ProcessGroup, input: torch.Tensor, is_inferenc
         if is_inference_mode:
             dist.inference_all_reduce(input, group=group)
         else:
-            dist.all_reduce(input.contiguous(), group=group)
+            input = input.contiguous()
+            dist.all_reduce(input, group=group)
         return input
 
     @staticmethod
@@ -276,8 +277,8 @@ def forward(ctx: Any, group: dist.ProcessGroup, input: torch.Tensor, weight, bia
     def backward(ctx: Any, grad_output: torch.Tensor) -> Tuple[None, torch.Tensor]:
 
         input, weight = ctx.saved_tensors
-        grad_input = grad_output.matmul(weight)
-        handle = dist.all_reduce(grad_input.contiguous(), group=ctx.group, async_op=True)
+        grad_input = grad_output.matmul(weight).contiguous()
+        handle = dist.all_reduce(grad_input, group=ctx.group, async_op=True)
         grad_weight = grad_output.view(-1, grad_output.shape[-1]).t().matmul(input.view(-1, input.shape[-1]))
         grad_bias = grad_output.sum(0) if ctx.use_bias else None
         handle.wait()
@@ -310,7 +311,8 @@ def backward(ctx: Any, grad_output: torch.Tensor) -> Tuple[None, torch.Tensor]:
         if not pg_utils.is_parallel(ctx.group):
             return None, grad_output
 
-        dist.all_reduce(grad_output.contiguous(), group=ctx.group)
+        grad_output = grad_output.contiguous()
+        dist.all_reduce(grad_output, group=ctx.group)
         return None, grad_output
 
 
diff --git a/tests/unit/module_inject/test_tp_noncontiguous_reduce.py b/tests/unit/module_inject/test_tp_noncontiguous_reduce.py
new file mode 100644
index 000000000000..4a2e85418afd
--- /dev/null
+++ b/tests/unit/module_inject/test_tp_noncontiguous_reduce.py
@@ -0,0 +1,39 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+import torch
+
+import deepspeed.comm as dist
+from deepspeed.accelerator import get_accelerator
+from deepspeed.module_inject.layers import ColumnParallel, RowParallel
+
+from unit.common import DistributedTest
+
+
+class TestTensorParallelNonContiguousReduce(DistributedTest):
+    """The TP all-reduce must reach the returned tensor even when it is not contiguous.
+
+    Reducing a temporary from ``x.contiguous()`` silently drops the result for non-contiguous ``x``.
+    """
+    world_size = 2
+
+    def test_row_parallel_forward(self):
+        rank = dist.get_rank()
+        x = torch.full((3, 2), rank + 1.0, device=get_accelerator().device_name()).t()
+        assert not x.is_contiguous()
+
+        out = RowParallel.apply(dist.new_group([0, 1]), x, False)
+
+        assert torch.all(out == 3.0)
+
+    def test_column_parallel_backward(self):
+        group = dist.new_group([0, 1])
+        rank = dist.get_rank()
+        x = torch.ones(3, 2, device=get_accelerator().device_name(), requires_grad=True)
+
+        # The transpose makes the incoming gradient non-contiguous.
+        out = ColumnParallel.apply(group, x).t()
+        (out * (rank + 1)).sum().backward()
+
+        assert torch.all(x.grad == 3.0)