From 629bc9878869c7e54fc4b62799fcbb74dbb39504 Mon Sep 17 00:00:00 2001 From: hemildesai Date: Wed, 23 Dec 2020 02:54:12 +0530 Subject: [PATCH 1/9] Add contents of Python.gitignore to gitignore Signed-off-by: hemildesai --- .gitignore | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/.gitignore b/.gitignore index 62a43479..7c16b260 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,144 @@ python/replicate/bin # scratch notebooks Untitled.ipynb .ipynb_checkpoints + +#Taken from https://github.com/github/gitignore/blob/master/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ From 8c7fcf9458672fb51b531ea98c7154cfb01d2a74 Mon Sep 17 00:00:00 2001 From: hemildesai Date: Wed, 23 Dec 2020 02:59:43 +0530 Subject: [PATCH 2/9] Update vscode settings to automatically select venv and discover tests Signed-off-by: hemildesai --- .vscode/settings.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 0aaf99d1..365f99ee 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,13 @@ { "files.exclude": { "go": true - } + }, + "python.pythonPath": "${workspaceFolder}/venv/bin/python3", + "python.testing.pytestArgs": [ + "python/tests", + "end-to-end-test/end_to_end_test", + ], + "python.testing.unittestEnabled": false, + "python.testing.nosetestsEnabled": false, + "python.testing.pytestEnabled": true } From 88233c1fd9535960ce576f54f96c96e8e55a034f Mon Sep 17 00:00:00 2001 From: hemildesai Date: Wed, 23 Dec 2020 03:00:35 +0530 Subject: [PATCH 3/9] Setup and use a virtual env for Python in makefiles Signed-off-by: hemildesai --- Makefile | 18 +++++++++++++----- end-to-end-test/Makefile | 7 +++++-- python/Makefile | 27 +++++++++++++++++---------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 8f883e45..dc6056f6 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,15 @@ ENVIRONMENT := development +ROOTDIR=./ +VENV=$(ROOTDIR)venv/bin/ + +.PHONY: venv +venv: + test -d venv || python -m venv $(ROOTDIR)venv OS := $(shell uname -s) .PHONY: build -build: verify-dev-env +build: verify-dev-env venv cd go && $(MAKE) build-all ENVIRONMENT=$(ENVIRONMENT) cd python && $(MAKE) build @@ -18,14 +24,16 @@ else endif .PHONY: develop -develop: verify-dev-env +develop: verify-dev-env venv cd go && $(MAKE) build cd go && $(MAKE) install - cd python && python setup.py develop + cd python && $(MAKE) develop + @printf "\nPython venv created at $(ROOTDIR)venv. Run `tput bold`source $(ROOTDIR)venv/bin/activate`tput sgr0` to activate the venv in your shell.\n" + @printf "If you're using vscode, select this venv as the Python interpretor in the bottom left.\n" .PHONY: install-test-dependencies -install-test-dependencies: - pip install -r requirements-test.txt +install-test-dependencies: venv + $(VENV)pip install -r requirements-test.txt .PHONY: test test: install-test-dependencies develop diff --git a/end-to-end-test/Makefile b/end-to-end-test/Makefile index 585aed2c..2c5a9785 100644 --- a/end-to-end-test/Makefile +++ b/end-to-end-test/Makefile @@ -1,7 +1,10 @@ +ROOTDIR=../ +VENV=$(ROOTDIR)venv/bin/ + .PHONY: test test: - pytest -m "not external" + $(VENV)pytest -m "not external" .PHONY: test-external test-external: - pytest + $(VENV)pytest diff --git a/python/Makefile b/python/Makefile index cc633391..132f1f3a 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,25 +1,32 @@ +ROOTDIR=../ +VENV=$(ROOTDIR)venv/bin/ + .PHONY: pre-commit pre-commit: lint .PHONY: fmt fmt: - black . + $(VENV)black . + +.PHONY: develop +develop: + $(VENV)python setup.py develop # --import-mode makes pytest import from globally installed package by default instead of local replicate/ module. # This assumes you've run `make develop` before running tests # https://docs.pytest.org/en/stable/pythonpath.html#pythonpath .PHONY: test test: lint - pytest --import-mode=importlib -m "not external" + $(VENV)pytest --import-mode=importlib -m "not external" .PHONY: test-external test-external: lint - pytest --import-mode=importlib + $(VENV)pytest --import-mode=importlib .PHONY: lint lint: - mypy . - black --check . + $(VENV)mypy . + $(VENV)black --check . .PHONY: clean clean: @@ -42,9 +49,9 @@ clean: # https://docs.python.org/3/distutils/apiref.html#distutils.util.get_platform .PHONY: build build: clean - pip install wheel - python setup.py bdist_wheel --plat-name manylinux1_x86_64 - python setup.py bdist_wheel --plat-name macosx_10_9_x86_64 + $(VENV)pip install wheel + $(VENV)python setup.py bdist_wheel --plat-name manylinux1_x86_64 + $(VENV)python setup.py bdist_wheel --plat-name macosx_10_9_x86_64 .PHONY: targets targets: @@ -52,5 +59,5 @@ targets: .PHONY: vendor vendor: - pip install vendoring - vendoring sync -v . + $(VENV)pip install vendoring + $(VENV)vendoring sync -v . From 1b71d571dc4255ec43723cb603e5927f73e3c7b1 Mon Sep 17 00:00:00 2001 From: hemildesai Date: Wed, 23 Dec 2020 03:01:21 +0530 Subject: [PATCH 4/9] Use the python path from virtualenv in end to end tests Signed-off-by: hemildesai --- end-to-end-test/end_to_end_test/test_checkout.py | 5 +++-- end-to-end-test/end_to_end_test/test_delete.py | 4 ++-- end-to-end-test/end_to_end_test/test_list.py | 8 +++++--- end-to-end-test/end_to_end_test/utils.py | 1 + 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/end-to-end-test/end_to_end_test/test_checkout.py b/end-to-end-test/end_to_end_test/test_checkout.py index 072317a5..f8d4d972 100644 --- a/end-to-end-test/end_to_end_test/test_checkout.py +++ b/end-to-end-test/end_to_end_test/test_checkout.py @@ -5,7 +5,8 @@ import os import subprocess import pytest # type: ignore -from .utils import get_env + +from .utils import get_env, PYTHON_PATH @pytest.mark.parametrize( @@ -69,7 +70,7 @@ def main(): ) env = get_env() - cmd = ["python", "train.py"] + cmd = [PYTHON_PATH, "train.py"] subprocess.run(cmd, cwd=tmpdir, env=env, check=True) experiments = json.loads( diff --git a/end-to-end-test/end_to_end_test/test_delete.py b/end-to-end-test/end_to_end_test/test_delete.py index ce5328a4..f4fda9ec 100644 --- a/end-to-end-test/end_to_end_test/test_delete.py +++ b/end-to-end-test/end_to_end_test/test_delete.py @@ -4,7 +4,7 @@ from pathlib import Path import pytest # type: ignore -from .utils import path_exists, get_env +from .utils import path_exists, get_env, PYTHON_PATH @pytest.mark.parametrize( @@ -57,7 +57,7 @@ def main(): ) env = get_env() - cmd = ["python", "train.py", "--foo"] + cmd = [PYTHON_PATH, "train.py", "--foo"] subprocess.run(cmd, cwd=tmpdir, env=env, check=True) experiments = json.loads( diff --git a/end-to-end-test/end_to_end_test/test_list.py b/end-to-end-test/end_to_end_test/test_list.py index 44388d9d..b9deb858 100644 --- a/end-to-end-test/end_to_end_test/test_list.py +++ b/end-to-end-test/end_to_end_test/test_list.py @@ -4,7 +4,7 @@ import pytest # type: ignore from dateutil.parser import parse as parse_date -from .utils import get_env +from .utils import get_env, PYTHON_PATH @pytest.mark.parametrize( @@ -58,7 +58,7 @@ def main(): env = get_env() - subprocess.run(["python", "train.py", "--foo"], cwd=tmpdir, env=env, check=True) + subprocess.run([PYTHON_PATH, "train.py", "--foo"], cwd=tmpdir, env=env, check=True) experiments = json.loads( subprocess.run( @@ -106,7 +106,9 @@ def main(): experiment.checkpoint(path=".", step=3) """ ) - subprocess.run(["python", "train2.py", exp["id"]], cwd=tmpdir, env=env, check=True) + subprocess.run( + [PYTHON_PATH, "train2.py", exp["id"]], cwd=tmpdir, env=env, check=True + ) experiments = json.loads( subprocess.run( ["replicate", "--verbose", "list", "--json"], diff --git a/end-to-end-test/end_to_end_test/utils.py b/end-to-end-test/end_to_end_test/utils.py index 2ce95663..8b6ca1c4 100644 --- a/end-to-end-test/end_to_end_test/utils.py +++ b/end-to-end-test/end_to_end_test/utils.py @@ -5,6 +5,7 @@ from google.cloud import storage as google_storage ROOT_DIRECTORY = Path(__file__).parent.parent.parent +PYTHON_PATH = os.path.abspath(os.path.join(ROOT_DIRECTORY, "venv/bin/python")) def get_env(): From d83cc47057cdcfcfb3ffd2bb2d0aed87ac7f6471 Mon Sep 17 00:00:00 2001 From: hemildesai Date: Wed, 23 Dec 2020 03:23:56 +0530 Subject: [PATCH 5/9] Fix make install Signed-off-by: hemildesai --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dc6056f6..40a6b067 100644 --- a/Makefile +++ b/Makefile @@ -16,9 +16,9 @@ build: verify-dev-env venv .PHONY: install install: build ifeq ($(OS),Linux) - pip install python/dist/replicate-*-py3-none-manylinux1_x86_64.whl + $(VENV)pip install python/dist/replicate-*-py3-none-manylinux1_x86_64.whl else ifeq ($(OS),Darwin) - pip install python/dist/replicate-*-py3-none-macosx_*.whl + $(VENV)pip install python/dist/replicate-*-py3-none-macosx_*.whl else @echo Unknown OS: $(OS) endif From aecea77fc36a0495c45edecdd48a309aa626597b Mon Sep 17 00:00:00 2001 From: hemildesai Date: Wed, 23 Dec 2020 04:12:07 +0530 Subject: [PATCH 6/9] Copy go binary seperately in make install Signed-off-by: hemildesai --- Makefile | 1 + go/Makefile | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 40a6b067..c5bfed29 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ else ifeq ($(OS),Darwin) else @echo Unknown OS: $(OS) endif + cd go && $(MAKE) copy-binary .PHONY: develop develop: verify-dev-env venv diff --git a/go/Makefile b/go/Makefile index 9789477b..f5ed8513 100644 --- a/go/Makefile +++ b/go/Makefile @@ -51,14 +51,17 @@ build-all: # install without sudo if the install path exists and is writeable, # or if it doesn't exist and its directory is writeable -.PHONY: install -install: build +.PHONY: copy-binary +copy-binary: if [[ (-f "$(INSTALL_PATH)" && -w "$(INSTALL_PATH)") || (! -f "$(INSTALL_PATH)" && -w $$(dirname "$(INSTALL_PATH)")) ]]; then \ cp $(BINARY) $(INSTALL_PATH); \ else \ sudo cp $(BINARY) $(INSTALL_PATH); \ fi +.PHONY: install +install: build copy-binary + .PHONY: clean clean: rm -rf $(RELEASE_DIR) From b36645fea6e321fd95705febe6f4ed668dd6f335 Mon Sep 17 00:00:00 2001 From: hemildesai Date: Fri, 25 Dec 2020 00:30:24 +0530 Subject: [PATCH 7/9] Upgrade pip in ci Signed-off-by: hemildesai --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bd76559a..c2d7c556 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,6 +107,8 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true + - name: Upgrade pip + run: pip install --upgrade pip - name: Install run: | (cd ../ && make install-test-dependencies) @@ -171,6 +173,8 @@ jobs: run: | echo "${{ secrets.SSH_PRIVATE_KEY }}" > /tmp/private_key.pem chmod 400 /tmp/private_key.pem + - name: Upgrade pip + run: pip install --upgrade pip - name: Install run: | (cd ../ && make install-test-dependencies) From c661fbd7c52491c45cc547a66ce0200fa759a047 Mon Sep 17 00:00:00 2001 From: hemildesai Date: Fri, 25 Dec 2020 00:36:51 +0530 Subject: [PATCH 8/9] Revert "Upgrade pip in ci" This reverts commit b36645fea6e321fd95705febe6f4ed668dd6f335. Signed-off-by: hemildesai --- .github/workflows/ci.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2d7c556..bd76559a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -107,8 +107,6 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - - name: Upgrade pip - run: pip install --upgrade pip - name: Install run: | (cd ../ && make install-test-dependencies) @@ -173,8 +171,6 @@ jobs: run: | echo "${{ secrets.SSH_PRIVATE_KEY }}" > /tmp/private_key.pem chmod 400 /tmp/private_key.pem - - name: Upgrade pip - run: pip install --upgrade pip - name: Install run: | (cd ../ && make install-test-dependencies) From 37e8d355c1e0059d0fc24d2ef9a6cd91020c5e63 Mon Sep 17 00:00:00 2001 From: hemildesai Date: Fri, 25 Dec 2020 00:38:26 +0530 Subject: [PATCH 9/9] Upgrade pip when creating virtualenv Signed-off-by: hemildesai --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c5bfed29..06a5a8b4 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ VENV=$(ROOTDIR)venv/bin/ .PHONY: venv venv: - test -d venv || python -m venv $(ROOTDIR)venv + test -d venv || python -m venv $(ROOTDIR)venv && $(VENV)pip install --upgrade pip OS := $(shell uname -s)