diff --git a/mathics/eval/tensors.py b/mathics/eval/tensors.py
index 406f7fe12..5bcceec64 100644
--- a/mathics/eval/tensors.py
+++ b/mathics/eval/tensors.py
@@ -5,7 +5,7 @@
 from sympy.utilities.iterables import permutations
 
 from mathics.core.atoms import Integer, Integer0, Integer1, String
-from mathics.core.convert.matrix import to_sympy_array, to_sympy_matrix
+from mathics.core.convert.matrix import to_sympy_array
 from mathics.core.convert.python import from_python
 from mathics.core.convert.sympy import from_sympy_matrix
 from mathics.core.evaluation import Evaluation
@@ -339,6 +339,9 @@ def eval_Transpose(m, dimensions: int) -> Optional[Expression]:
         p = PermuteDims(sympy_m, (1, 0, 2))
         return from_sympy_matrix(p.as_explicit())
 
-    assert dimensions == 2
-    sympy_m = to_sympy_matrix(m)
-    return None if sympy_m is None else from_sympy_matrix(sympy_m.T)
+    assert dimensions == 2, "eval_Transpose only applies to 2- and 3-dimensional array."
+    # Rearrange the elements directly. Going through a SymPy matrix
+    # would turn a result with a single column, such as the transpose
+    # of {{1, 2, 3}}, into a vector.
+    rows = [row.elements for row in m.elements]
+    return ListExpression(*(ListExpression(*column) for column in zip(*rows)))
diff --git a/test/builtin/test_tensor.py b/test/builtin/test_tensor.py
index 853d7806f..43d0c2e78 100644
--- a/test/builtin/test_tensor.py
+++ b/test/builtin/test_tensor.py
@@ -11,8 +11,18 @@
 @pytest.mark.parametrize(
     ("str_expr", "msgs", "str_expected", "fail_msg"),
     [
-        ("Dimensions[{}]", None, "{0}", None),
-        ("Dimensions[{{}}]", None, "{1, 0}", None),
+        (
+            "Dimensions[{}]",
+            None,
+            "{0}",
+            None,
+        ),
+        (
+            "Dimensions[{{}}]",
+            None,
+            "{1, 0}",
+            None,
+        ),
         ## Issue #670
         (
             "A = {{ b ^ ( -1 / 2), 0}, {a * b ^ ( -1 / 2 ), b ^ ( 1 / 2 )}}",
@@ -23,7 +33,50 @@
         ("A . Inverse[A]", None, "{{1, 0}, {0, 1}}", None),
         ("A", None, "{{1 / Sqrt[b], 0}, {a / Sqrt[b], Sqrt[b]}}", None),
         # Transpose
-        ("Transpose[x]", None, "Transpose[x]", None),
+        (
+            "Transpose[x]",
+            None,
+            "Transpose[x]",
+            None,
+        ),
+        # A matrix with a single row or column stays a matrix.
+        (
+            "Transpose[{{1, 2, 3}}]",
+            None,
+            "{{1}, {2}, {3}}",
+            "Transpose of a single row turns into a single column",
+        ),
+        (
+            "Transpose[{{1}, {2}, {3}}]",
+            None,
+            "{{1, 2, 3}}",
+            "Transpose of a single column turns into a single row",
+        ),
+        (
+            "Transpose[{{a}}]",
+            None,
+            "{{a}}",
+            "Transpose of a single element",
+        ),
+        (
+            "Transpose[Transpose[{{a, b}}]]",
+            None,
+            "{{a, b}}",
+            None,
+        ),
+        (
+            "Dimensions[Transpose[{{1, 2, 3}}]]",
+            None,
+            "{3, 1}",
+            "Check the shape of a Transpose of a single row",
+        ),
+        # Elements are moved as they are, without a round trip through SymPy.
+        (
+            "Transpose[{{x + y, f[x]}, {1/2, Sin[x]}}]",
+            None,
+            "{{x + y, 1 / 2}, {f[x], Sin[x]}}",
+            None,
+        ),
     ],
 )
 def test_tensor(str_expr, msgs, str_expected, fail_msg):