release-testing (#889)

* make script shebangs more flexible

* move location of python tests

This change allows us to run the test suite against a release candidate
hosted on test.pypi.org.

```
pip install --extra-index-url https://test.pypi.org/simple/ datafusion==40.0.0

pytest --import-mode=importlib python/tests
```

* add documentation for testing against release candidate

* add dev script for cleaning maturin build artifacts

* update ruff lint config for new test location

Author: Michael-J-Ward

dev/clean.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This cleans up the project by removing build artifacts and other generated files.
+
+# Function to remove a directory and print the action
+remove_dir() {
+    if [ -d "$1" ]; then
+        echo "Removing directory: $1"
+        rm -rf "$1"
+    fi
+}
+
+# Function to remove a file and print the action
+remove_file() {
+    if [ -f "$1" ]; then
+        echo "Removing file: $1"
+        rm -f "$1"
+    fi
+}
+
+# Remove .pytest_cache directory
+remove_dir .pytest_cache/
+
+# Remove target directory
+remove_dir target/
+
+# Remove any __pycache__ directories
+find python/ -type d -name "__pycache__" -print | while read -r dir; do
+    remove_dir "$dir"
+done
+
+# Remove pytest-coverage.lcov file
+# remove_file .coverage
+# remove_file pytest-coverage.lcov
+
+# Remove rust-coverage.lcov file
+# remove_file rust-coverage.lcov
+
+# Remove pyo3 files
+find python/ -type f -name '_internal.*.so' -print | while read -r file; do
+    remove_file "$file"
+done
+
+echo "Cleanup complete."

dev/python_lint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

dev/release/README.md

@@ -159,10 +159,29 @@ Send the email to start the vote.
 
 ## Verifying a Release
 
-Install the release from testpypi:
+Running the unit tests against a testpypi release candidate:
 
 ```bash
-pip install --extra-index-url https://test.pypi.org/simple/ datafusion==0.7.0
+# clone a fresh repo
+git clone https://github.com/apache/datafusion-python.git
+cd datafusion-python
+
+# checkout the release commit
+git fetch --tags
+git checkout 40.0.0-rc1
+
+# create the env
+python3 -m venv venv
+source venv/bin/activate
+
+# install release candidate
+pip install --extra-index-url https://test.pypi.org/simple/ datafusion==40.0.0
+
+# only dep needed to run tests is pytest
+pip install pytest
+
+# run the tests
+pytest --import-mode=importlib python/tests
 ```
 
 Try running one of the examples from the top-level README, or write some custom Python code to query some available

dev/release/verify-release-candidate.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

dev/rust_lint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docs/build.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

pyproject.toml

@@ -78,7 +78,7 @@ max-doc-length = 88
 
 # Disable docstring checking for these directories
 [tool.ruff.lint.per-file-ignores]
-"python/datafusion/tests/*" = ["D"]
+"python/tests/*" = ["D"]
 "examples/*" = ["D", "W505"]
 "dev/*" = ["D"]
 "benchmarks/*" = ["D", "F"]