fix: Add separate config options for filepath checksum on insert vs fetch

Adds two new config options with fallback to the existing one:
- `filepath_checksum_size_limit_insert`: skip checksum computation on insert
- `filepath_checksum_size_limit_fetch`: skip checksum verification on fetch
- `filepath_checksum_size_limit`: fallback for both if specific ones not set

This allows teams to:
- Compute checksums on insert (for auditing) but skip verification on fetch
- Skip checksums entirely for large files
- Configure different size limits for insert vs fetch operations

When checksum is skipped on insert:
- A warning is logged
- contents_hash is stored as NULL
- Existing file verification is bypassed

Fixes #1386

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: dimitri-yatsenko

datajoint/external.py

@@ -276,13 +276,27 @@ def upload_filepath(self, local_filepath):
         uuid = uuid_from_buffer(
             init_string=relative_filepath
         )  # hash relative path, not contents
-        contents_hash = uuid_from_file(local_filepath)
+
+        # Check if checksum should be skipped based on file size limit
+        file_size = Path(local_filepath).stat().st_size
+        size_limit = config.get("filepath_checksum_size_limit_insert") or config.get(
+            "filepath_checksum_size_limit"
+        )
+        skip_checksum = size_limit is not None and file_size > size_limit
+
+        if skip_checksum:
+            contents_hash = None
+            logger.warning(
+                f"Skipping checksum for '{relative_filepath}' ({file_size} bytes > {size_limit} byte limit)"
+            )
+        else:
+            contents_hash = uuid_from_file(local_filepath)
 
         # check if the remote file already exists and verify that it matches
         check_hash = (self & {"hash": uuid}).fetch("contents_hash")
         if check_hash.size:
             # the tracking entry exists, check that it's the same file as before
-            if contents_hash != check_hash[0]:
+            if not skip_checksum and contents_hash != check_hash[0]:
                 raise DataJointError(
                     f"A different version of '{relative_filepath}' has already been placed."
                 )
@@ -291,15 +305,15 @@ def upload_filepath(self, local_filepath):
             self._upload_file(
                 local_filepath,
                 self._make_external_filepath(relative_filepath),
-                metadata={"contents_hash": str(contents_hash)},
+                metadata={"contents_hash": str(contents_hash) if contents_hash else ""},
             )
             self.connection.query(
                 "INSERT INTO {tab} (hash, size, filepath, contents_hash) VALUES (%s, {size}, '{filepath}', %s)".format(
                     tab=self.full_table_name,
-                    size=Path(local_filepath).stat().st_size,
+                    size=file_size,
                     filepath=relative_filepath,
                 ),
-                args=(uuid.bytes, contents_hash.bytes),
+                args=(uuid.bytes, contents_hash.bytes if contents_hash else None),
             )
         return uuid
 
@@ -312,7 +326,9 @@ def download_filepath(self, filepath_hash):
         """
 
         def _need_checksum(local_filepath, expected_size):
-            limit = config.get("filepath_checksum_size_limit")
+            limit = config.get("filepath_checksum_size_limit_fetch") or config.get(
+                "filepath_checksum_size_limit"
+            )
             actual_size = Path(local_filepath).stat().st_size
             if expected_size != actual_size:
                 # this should never happen without outside interference

datajoint/settings.py

@@ -49,8 +49,13 @@
         "database.use_tls": None,
         "enable_python_native_blobs": True,  # python-native/dj0 encoding support
         "add_hidden_timestamp": False,
-        # file size limit for when to disable checksums
+        # file size limits for when to disable checksums (in bytes)
+        # filepath_checksum_size_limit: fallback for both insert and fetch if specific ones not set
         "filepath_checksum_size_limit": None,
+        # filepath_checksum_size_limit_insert: skip checksum computation on insert for large files
+        "filepath_checksum_size_limit_insert": None,
+        # filepath_checksum_size_limit_fetch: skip checksum verification on fetch for large files
+        "filepath_checksum_size_limit_fetch": None,
     }
 )