Skip to content

Commit d215649

Browse files
millerjpmillerjp
committed
bulk setup
1 parent f5155ff commit d215649

File tree

200 files changed

+58858
-9
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

200 files changed

+58858
-9
lines changed

‎.github/workflows/ci-monorepo.yml‎

Lines changed: 354 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,354 @@
1+
name: Monorepo CI Base
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
package:
7+
description: 'Package to test (async-cassandra or async-cassandra-bulk)'
8+
required: true
9+
type: string
10+
run-integration-tests:
11+
description: 'Run integration tests'
12+
required: false
13+
type: boolean
14+
default: false
15+
run-full-suite:
16+
description: 'Run full test suite'
17+
required: false
18+
type: boolean
19+
default: false
20+
21+
env:
22+
PACKAGE_DIR: libs/${{ inputs.package }}
23+
24+
jobs:
25+
lint:
26+
runs-on: ubuntu-latest
27+
name: Lint ${{ inputs.package }}
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
32+
- name: Set up Python 3.12
33+
uses: actions/setup-python@v5
34+
with:
35+
python-version: '3.12'
36+
37+
- name: Install dependencies
38+
run: |
39+
cd ${{ env.PACKAGE_DIR }}
40+
python -m pip install --upgrade pip
41+
pip install -e ".[dev]"
42+
43+
- name: Run linting checks
44+
run: |
45+
cd ${{ env.PACKAGE_DIR }}
46+
echo "=== Running ruff ==="
47+
ruff check src/ tests/
48+
echo "=== Running black ==="
49+
black --check src/ tests/
50+
echo "=== Running isort ==="
51+
isort --check-only src/ tests/
52+
echo "=== Running mypy ==="
53+
mypy src/
54+
55+
security:
56+
runs-on: ubuntu-latest
57+
needs: lint
58+
name: Security ${{ inputs.package }}
59+
60+
steps:
61+
- uses: actions/checkout@v4
62+
63+
- name: Set up Python 3.12
64+
uses: actions/setup-python@v5
65+
with:
66+
python-version: '3.12'
67+
68+
- name: Install security tools
69+
run: |
70+
python -m pip install --upgrade pip
71+
pip install bandit[toml] safety pip-audit
72+
73+
- name: Run Bandit security scan
74+
run: |
75+
cd ${{ env.PACKAGE_DIR }}
76+
echo "=== Running Bandit security scan ==="
77+
# Run bandit with config file and capture exit code
78+
bandit -c ../../.bandit -r src/ -f json -o bandit-report.json || BANDIT_EXIT=$?
79+
# Show the detailed issues found
80+
echo "=== Bandit Detailed Results ==="
81+
bandit -c ../../.bandit -r src/ -v || true
82+
# For low severity issues, we'll just warn but not fail
83+
if [ "${BANDIT_EXIT:-0}" -eq 1 ]; then
84+
echo "⚠️ Bandit found low-severity issues (see above)"
85+
# Check if there are medium or high severity issues
86+
if bandit -c ../../.bandit -r src/ -lll &>/dev/null; then
87+
echo "✅ No medium or high severity issues found - continuing"
88+
exit 0
89+
else
90+
echo "❌ Medium or high severity issues found - failing"
91+
exit 1
92+
fi
93+
fi
94+
exit ${BANDIT_EXIT:-0}
95+
96+
- name: Check dependencies with Safety
97+
run: |
98+
cd ${{ env.PACKAGE_DIR }}
99+
echo "=== Checking dependencies with Safety ==="
100+
pip install -e ".[dev,test]"
101+
# Using the new 'scan' command as 'check' is deprecated
102+
safety scan --json || SAFETY_EXIT=$?
103+
# Safety scan exits with 64 if vulnerabilities found
104+
if [ "${SAFETY_EXIT:-0}" -eq 64 ]; then
105+
echo "❌ Vulnerabilities found in dependencies"
106+
exit 1
107+
fi
108+
109+
- name: Run pip-audit
110+
run: |
111+
cd ${{ env.PACKAGE_DIR }}
112+
echo "=== Running pip-audit ==="
113+
# Skip the local package as it's not on PyPI yet
114+
pip-audit --skip-editable
115+
116+
- name: Upload security reports
117+
uses: actions/upload-artifact@v4
118+
if: always()
119+
with:
120+
name: security-reports-${{ inputs.package }}
121+
path: |
122+
${{ env.PACKAGE_DIR }}/bandit-report.json
123+
124+
unit-tests:
125+
runs-on: ubuntu-latest
126+
needs: lint
127+
name: Unit Tests ${{ inputs.package }}
128+
129+
steps:
130+
- uses: actions/checkout@v4
131+
132+
- name: Set up Python 3.12
133+
uses: actions/setup-python@v5
134+
with:
135+
python-version: '3.12'
136+
137+
- name: Install dependencies
138+
run: |
139+
cd ${{ env.PACKAGE_DIR }}
140+
python -m pip install --upgrade pip
141+
pip install -e ".[test]"
142+
143+
- name: Run unit tests with coverage
144+
run: |
145+
cd ${{ env.PACKAGE_DIR }}
146+
pytest tests/unit/ -v --cov=${{ inputs.package == 'async-cassandra' && 'async_cassandra' || 'async_cassandra_bulk' }} --cov-report=html --cov-report=xml || echo "No unit tests found (expected for new packages)"
147+
148+
build:
149+
runs-on: ubuntu-latest
150+
needs: [lint, security, unit-tests]
151+
name: Build ${{ inputs.package }}
152+
153+
steps:
154+
- uses: actions/checkout@v4
155+
156+
- name: Set up Python 3.12
157+
uses: actions/setup-python@v5
158+
with:
159+
python-version: '3.12'
160+
161+
- name: Install build dependencies
162+
run: |
163+
python -m pip install --upgrade pip
164+
pip install build twine
165+
166+
- name: Build package
167+
run: |
168+
cd ${{ env.PACKAGE_DIR }}
169+
echo "=== Building package ==="
170+
python -m build
171+
echo "=== Package contents ==="
172+
ls -la dist/
173+
174+
- name: Check package with twine
175+
run: |
176+
cd ${{ env.PACKAGE_DIR }}
177+
echo "=== Checking package metadata ==="
178+
twine check dist/*
179+
180+
- name: Display package info
181+
run: |
182+
cd ${{ env.PACKAGE_DIR }}
183+
echo "=== Wheel contents ==="
184+
python -m zipfile -l dist/*.whl | head -20
185+
echo "=== Package metadata ==="
186+
pip show --verbose ${{ inputs.package }} || true
187+
188+
- name: Upload build artifacts
189+
uses: actions/upload-artifact@v4
190+
with:
191+
name: python-package-distributions-${{ inputs.package }}
192+
path: ${{ env.PACKAGE_DIR }}/dist/
193+
retention-days: 7
194+
195+
integration-tests:
196+
runs-on: ubuntu-latest
197+
needs: [lint, security, unit-tests]
198+
if: ${{ inputs.package == 'async-cassandra' && (inputs.run-integration-tests || inputs.run-full-suite) }}
199+
name: Integration Tests ${{ inputs.package }}
200+
201+
strategy:
202+
fail-fast: false
203+
matrix:
204+
test-suite:
205+
- name: "Integration Tests"
206+
command: "pytest tests/integration -v -m 'not stress'"
207+
- name: "FastAPI Integration"
208+
command: "pytest tests/fastapi_integration -v"
209+
- name: "BDD Tests"
210+
command: "pytest tests/bdd -v"
211+
- name: "Example App"
212+
command: "cd ../../examples/fastapi_app && pytest tests/ -v"
213+
214+
services:
215+
cassandra:
216+
image: cassandra:5
217+
ports:
218+
- 9042:9042
219+
options: >-
220+
--health-cmd "nodetool status"
221+
--health-interval 30s
222+
--health-timeout 10s
223+
--health-retries 10
224+
--memory=4g
225+
--memory-reservation=4g
226+
env:
227+
CASSANDRA_CLUSTER_NAME: TestCluster
228+
CASSANDRA_DC: datacenter1
229+
CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch
230+
HEAP_NEWSIZE: 512M
231+
MAX_HEAP_SIZE: 3G
232+
JVM_OPTS: "-XX:+UseG1GC -XX:G1RSetUpdatingPauseTimePercent=5 -XX:MaxGCPauseMillis=300"
233+
234+
steps:
235+
- uses: actions/checkout@v4
236+
237+
- name: Set up Python 3.12
238+
uses: actions/setup-python@v5
239+
with:
240+
python-version: '3.12'
241+
242+
- name: Install dependencies
243+
run: |
244+
cd ${{ env.PACKAGE_DIR }}
245+
python -m pip install --upgrade pip
246+
pip install -e ".[test,dev]"
247+
248+
- name: Verify Cassandra is ready
249+
run: |
250+
echo "Installing cqlsh to verify Cassandra..."
251+
pip install cqlsh
252+
echo "Testing Cassandra connection..."
253+
cqlsh localhost 9042 -e "DESC CLUSTER" | head -10
254+
echo "✅ Cassandra is ready and responding to CQL"
255+
256+
- name: Run ${{ matrix.test-suite.name }}
257+
env:
258+
CASSANDRA_HOST: localhost
259+
CASSANDRA_PORT: 9042
260+
run: |
261+
cd ${{ env.PACKAGE_DIR }}
262+
echo "=== Running ${{ matrix.test-suite.name }} ==="
263+
${{ matrix.test-suite.command }}
264+
265+
stress-tests:
266+
runs-on: ubuntu-latest
267+
needs: [lint, security, unit-tests]
268+
if: ${{ inputs.package == 'async-cassandra' && inputs.run-full-suite }}
269+
name: Stress Tests ${{ inputs.package }}
270+
271+
strategy:
272+
fail-fast: false
273+
matrix:
274+
test-suite:
275+
- name: "Stress Tests"
276+
command: "pytest tests/integration -v -m stress"
277+
278+
services:
279+
cassandra:
280+
image: cassandra:5
281+
ports:
282+
- 9042:9042
283+
options: >-
284+
--health-cmd "nodetool status"
285+
--health-interval 30s
286+
--health-timeout 10s
287+
--health-retries 10
288+
--memory=4g
289+
--memory-reservation=4g
290+
env:
291+
CASSANDRA_CLUSTER_NAME: TestCluster
292+
CASSANDRA_DC: datacenter1
293+
CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch
294+
HEAP_NEWSIZE: 512M
295+
MAX_HEAP_SIZE: 3G
296+
JVM_OPTS: "-XX:+UseG1GC -XX:G1RSetUpdatingPauseTimePercent=5 -XX:MaxGCPauseMillis=300"
297+
298+
steps:
299+
- uses: actions/checkout@v4
300+
301+
- name: Set up Python 3.12
302+
uses: actions/setup-python@v5
303+
with:
304+
python-version: '3.12'
305+
306+
- name: Install dependencies
307+
run: |
308+
cd ${{ env.PACKAGE_DIR }}
309+
python -m pip install --upgrade pip
310+
pip install -e ".[test,dev]"
311+
312+
- name: Verify Cassandra is ready
313+
run: |
314+
echo "Installing cqlsh to verify Cassandra..."
315+
pip install cqlsh
316+
echo "Testing Cassandra connection..."
317+
cqlsh localhost 9042 -e "DESC CLUSTER" | head -10
318+
echo "✅ Cassandra is ready and responding to CQL"
319+
320+
- name: Run ${{ matrix.test-suite.name }}
321+
env:
322+
CASSANDRA_HOST: localhost
323+
CASSANDRA_PORT: 9042
324+
run: |
325+
cd ${{ env.PACKAGE_DIR }}
326+
echo "=== Running ${{ matrix.test-suite.name }} ==="
327+
${{ matrix.test-suite.command }}
328+
329+
test-summary:
330+
name: Test Summary ${{ inputs.package }}
331+
runs-on: ubuntu-latest
332+
needs: [lint, security, unit-tests, build]
333+
if: always()
334+
steps:
335+
- name: Summary
336+
run: |
337+
echo "## Test Results Summary for ${{ inputs.package }}" >> $GITHUB_STEP_SUMMARY
338+
echo "" >> $GITHUB_STEP_SUMMARY
339+
echo "### Core Tests" >> $GITHUB_STEP_SUMMARY
340+
echo "- Lint: ${{ needs.lint.result }}" >> $GITHUB_STEP_SUMMARY
341+
echo "- Security: ${{ needs.security.result }}" >> $GITHUB_STEP_SUMMARY
342+
echo "- Unit Tests: ${{ needs.unit-tests.result }}" >> $GITHUB_STEP_SUMMARY
343+
echo "- Build: ${{ needs.build.result }}" >> $GITHUB_STEP_SUMMARY
344+
echo "" >> $GITHUB_STEP_SUMMARY
345+
346+
if [ "${{ needs.lint.result }}" != "success" ] || \
347+
[ "${{ needs.security.result }}" != "success" ] || \
348+
[ "${{ needs.unit-tests.result }}" != "success" ] || \
349+
[ "${{ needs.build.result }}" != "success" ]; then
350+
echo "❌ Some tests failed" >> $GITHUB_STEP_SUMMARY
351+
exit 1
352+
else
353+
echo "✅ All tests passed" >> $GITHUB_STEP_SUMMARY
354+
fi

‎.github/workflows/full-test.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Full Test Suite
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
package:
7+
description: 'Package to test (async-cassandra, async-cassandra-bulk, or both)'
8+
required: true
9+
default: 'both'
10+
type: choice
11+
options:
12+
- async-cassandra
13+
- async-cassandra-bulk
14+
- both
15+
16+
jobs:
17+
async-cassandra:
18+
if: ${{ github.event.inputs.package == 'async-cassandra' || github.event.inputs.package == 'both' }}
19+
uses: ./.github/workflows/ci-monorepo.yml
20+
with:
21+
package: async-cassandra
22+
run-integration-tests: true
23+
run-full-suite: true
24+
25+
async-cassandra-bulk:
26+
if: ${{ github.event.inputs.package == 'async-cassandra-bulk' || github.event.inputs.package == 'both' }}
27+
uses: ./.github/workflows/ci-monorepo.yml
28+
with:
29+
package: async-cassandra-bulk
30+
run-integration-tests: false
31+
run-full-suite: false

0 commit comments

Comments
 (0)