Merge remote-tracking branch 'origin/maint/2.0'

Author: dimitri-yatsenko

src/datajoint/expression.py

@@ -1019,7 +1019,10 @@ def to_arrays(self, *attrs, include_key=False, order_by=None, limit=None, offset
                     arr = np.array(values)
                 except ValueError:
                     # Variable-size data (e.g., arrays of different shapes)
-                    arr = np.array(values, dtype=object)
+                    # Must assign individually to avoid numpy broadcasting issues
+                    arr = np.empty(len(values), dtype=object)
+                    for i, v in enumerate(values):
+                        arr[i] = v
                 result_arrays.append(arr)
 
             if include_key:

src/datajoint/jobs.py

@@ -13,7 +13,7 @@
 import platform
 import subprocess
 
-from .condition import AndList
+from .condition import AndList, Not
 from .errors import DataJointError, DuplicateError
 from .heading import Heading
 from .table import Table
@@ -370,8 +370,6 @@ def refresh(
 
         # Keys that need jobs: in key_source, not in target, not in jobs
         # Disable semantic_check for Job table (self) because its attributes may not have matching lineage
-        from .condition import Not
-
         new_keys = (key_source - self._target).restrict(Not(self), semantic_check=False).proj()
         new_key_list = new_keys.keys()
 
@@ -396,8 +394,10 @@ def refresh(
         # 2. Re-pend success jobs if keep_completed=True
         if config.jobs.keep_completed:
             # Success jobs whose keys are in key_source but not in target
-            # Disable semantic_check for Job table operations
-            success_to_repend = self.completed.restrict(key_source, semantic_check=False) - self._target
+            # Disable semantic_check for Job table operations (job table PK has different lineage than target)
+            success_to_repend = self.completed.restrict(key_source, semantic_check=False).restrict(
+                Not(self._target), semantic_check=False
+            )
             repend_keys = success_to_repend.keys()
             for key in repend_keys:
                 (self & key).delete_quick()

tests/integration/test_fetch.py

@@ -397,3 +397,63 @@ def test_to_arrays_without_include_key(lang):
     names, langs = result
     assert isinstance(names, np.ndarray)
     assert isinstance(langs, np.ndarray)
+
+
+def test_to_arrays_inhomogeneous_shapes(schema_any):
+    """Test to_arrays handles arrays of different shapes correctly.
+
+    Regression test for https://github.com/datajoint/datajoint-python/issues/1380
+    """
+    table = schema.Longblob()
+    table.delete()
+
+    # Insert arrays with different shapes that numpy would try to broadcast
+    table.insert(
+        [
+            {"id": 0, "data": np.random.randn(100)},  # shape (100,)
+            {"id": 1, "data": np.random.randn(100, 1)},  # shape (100, 1)
+            {"id": 2, "data": np.random.randn(100, 2)},  # shape (100, 2)
+        ]
+    )
+
+    # This should not raise ValueError
+    data = table.to_arrays("data", order_by="id")
+
+    # Should return object array with 3 elements
+    assert data.dtype == object
+    assert len(data) == 3
+
+    # Each element should preserve its original shape
+    assert data[0].shape == (100,)
+    assert data[1].shape == (100, 1)
+    assert data[2].shape == (100, 2)
+
+
+def test_to_arrays_inhomogeneous_shapes_second_axis(schema_any):
+    """Test to_arrays handles arrays differing on second axis.
+
+    Regression test for https://github.com/datajoint/datajoint-python/issues/1380
+    """
+    table = schema.Longblob()
+    table.delete()
+
+    # Insert arrays with different shapes on second axis
+    table.insert(
+        [
+            {"id": 0, "data": np.random.randn(100)},  # shape (100,)
+            {"id": 1, "data": np.random.randn(1, 100)},  # shape (1, 100)
+            {"id": 2, "data": np.random.randn(2, 100)},  # shape (2, 100)
+        ]
+    )
+
+    # This should not raise ValueError
+    data = table.to_arrays("data", order_by="id")
+
+    # Should return object array with 3 elements
+    assert data.dtype == object
+    assert len(data) == 3
+
+    # Each element should preserve its original shape
+    assert data[0].shape == (100,)
+    assert data[1].shape == (1, 100)
+    assert data[2].shape == (2, 100)

tests/integration/test_jobs.py

@@ -158,3 +158,52 @@ def test_long_error_stack(clean_jobs, subject, experiment):
     experiment.jobs.error(key, "error message", long_error_stack)
     error_stack = experiment.jobs.errors.fetch1("error_stack")
     assert error_stack == long_error_stack, "error stacks do not agree"
+
+
+def test_populate_reserve_jobs_with_keep_completed(clean_jobs, subject, experiment):
+    """Test populate(reserve_jobs=True) with keep_completed=True.
+
+    Regression test for https://github.com/datajoint/datajoint-python/issues/1379
+    The bug was that the `-` operator in jobs.refresh() didn't pass semantic_check=False,
+    causing a DataJointError about different lineages when keep_completed=True.
+    """
+    # Clear experiment data to ensure there's work to do
+    experiment.delete()
+
+    with dj.config.override(jobs={"keep_completed": True, "add_job_metadata": True}):
+        # Should not raise DataJointError about semantic matching
+        experiment.populate(reserve_jobs=True)
+
+        # Verify jobs completed successfully
+        assert len(experiment) > 0, "No data was populated"
+        assert len(experiment.jobs.errors) == 0, "Unexpected errors during populate"
+
+        # With keep_completed=True, completed jobs should be retained
+        assert len(experiment.jobs.completed) > 0, "Completed jobs not retained"
+
+
+def test_jobs_refresh_with_keep_completed(clean_jobs, subject, experiment):
+    """Test that jobs.refresh() works with keep_completed=True.
+
+    Regression test for https://github.com/datajoint/datajoint-python/issues/1379
+    """
+    # Clear experiment data and jobs
+    experiment.delete()
+    experiment.jobs.delete()
+
+    with dj.config.override(jobs={"keep_completed": True, "add_job_metadata": True}):
+        # Refresh should create pending jobs without semantic matching error
+        experiment.jobs.refresh()
+        pending_before = len(experiment.jobs.pending)
+        assert pending_before > 0, "No pending jobs created"
+
+        # Manually reserve and complete a job
+        key = experiment.jobs.pending.keys(limit=1)[0]
+        experiment.jobs.reserve(key)
+        experiment.jobs.complete(key)
+
+        # Job should now be completed
+        assert len(experiment.jobs.completed) == 1, "Job not marked as completed"
+
+        # Calling refresh again should not raise semantic matching error
+        experiment.jobs.refresh()  # This was failing before the fix