diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index adbbb0b..0000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[html] -directory = docs/coverage_htlm diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ebc5a6..5bae08b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,8 +15,12 @@ repos: - id: check-yaml - id: detect-private-key - id: end-of-file-fixer + - id: fix-encoding-pragma + args: [--remove] - id: pretty-format-json args: [--autofix,--no-sort-keys] + - id: requirements-txt-fixer + - id: trailing-whitespace - repo: https://github.com/psf/black rev: 19.10b0 hooks: @@ -27,3 +31,8 @@ repos: hooks: - id: blacken-docs additional_dependencies: [black==19.10b0] +- repo: https://github.com/asottile/reorder_python_imports + rev: v2.4.0 + hooks: + - id: reorder-python-imports + args: ['--application-directories=.:src', --py36-plus] diff --git a/.readthedocs.yml b/.readthedocs.yml index c53ae54..244df35 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -10,7 +10,9 @@ sphinx: configuration: src/pyDataverse/docs/source/conf.py # Optionally build your docs in additional formats such as PDF and ePub -formats: all +formats: + - epub + - pdf # Optionally set the version of Python and requirements required to build your docs python: diff --git a/.travis.yml b/.travis.yml index ec21750..31c57c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,6 @@ matrix: env: TOXENV=py38 - python: 3.6 env: TOXENV=docs - - python: 3.6 - env: TOXENV=coverage - - python: 3.6 - env: TOXENV=coveralls branches: only: @@ -26,9 +22,9 @@ before_install: - echo $TRAVIS_PYTHON_VERSION install: + - pip install --upgrade pip - pip install tox-travis - - pip install coverage - - pip install coveralls + - pip install codecov - virtualenv --version - easy_install --version - pip --version @@ -38,7 +34,7 @@ script: - tox after_success: - - coveralls + - codecov notifications: email: diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 51d4143..41a9868 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -229,7 +229,7 @@ you created the branch, check the section on :ref:`updating a PR `. -From here, you now can move forward to +From here, you now can move forward to - contribute to the documentation (see below) - contribute to the :ref:`code base ` @@ -267,7 +267,7 @@ Some other important things to know about the docs: - The pyDataverse documentation consists of two parts: - the docstrings in the code itself and - - the docs in the folder ``src/pyDataverse/doc/`` + - the docs in the folder ``src/pyDataverse/docs/`` - The docstrings provide a clear explanation of the usage of the individual functions, while the documentation consists of tutorial-like overviews per topic together with some other information (what’s new, installation, this page you're viewing right now, etc). - The docstrings follow the `Numpy Docstring Standard `_. @@ -871,6 +871,3 @@ You can find the full release history at :ref:`community_history` and on **Versioning** For pyDataverse, `Semantic versioning `_ is used for releases. - - -.. _contributing_resources: diff --git a/README.md b/README.md index 0b02de6..2216137 100644 --- a/README.md +++ b/README.md @@ -5,4 +5,4 @@ pyDataverse is a Python module for [Dataverse](http://dataverse.org). It helps to access the Dataverse [API's](http://guides.dataverse.org/en/latest/api/index.html) and manipulate, validate, import and export all Dataverse data-types (Dataverse, Dataset, Datafile). -**Find out more: [Read the Docs](https://pydataverse.readthedocs.io/en/latest/)** +**Find out more: [pyDataverse Documentation](https://pydataverse.readthedocs.io/en/latest/)** diff --git a/requirements/common.txt b/requirements/common.txt index 3e37022..1e6a7f2 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -1,2 +1,2 @@ -requests>=2.12.0 jsonschema>=3.2.0 +requests>=2.12.0 diff --git a/requirements/development.txt b/requirements/development.txt index e633bf3..f15824d 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,8 +1,7 @@ -# Requirements for development. -r common.txt --r lint.txt -r docs.txt +-r lint.txt +-r packaging.txt -r tests.txt -wheel pre-commit -twine +wheel diff --git a/requirements/docs.txt b/requirements/docs.txt index 7b376cd..03708d4 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,9 +1,7 @@ -# Requirements for documentation test (tox) --r common.txt -sphinx==1.7.* -git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme +git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme pydocstyle -restructuredtext_lint pygments +restructuredtext_lint rstcheck +sphinx==1.7.* diff --git a/requirements/lint.txt b/requirements/lint.txt index 3968095..3bec37f 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -1,18 +1,17 @@ -# Requirements for linting tests (tox) +autopep8 black==19.10b0 flake8==3.7.4 -flake8-docstrings==1.5.0 -flake8-rst-docstrings==v0.0.13 -flake8-blind-except==v0.1.1 -flake8-builtins==1.5.3 flake8-bandit==v2.1.0 +flake8-blind-except==v0.1.1 flake8-breakpoint==v1.1.0 flake8-bugbear==20.1.4 +flake8-builtins==1.5.3 flake8-comprehensions==3.2.3 -flake8-requirements==v1.3.2 -flake8-return==0.3.0 +flake8-docstrings==1.5.0 flake8-pytest-style==v1.2.1 +flake8-requirements==v1.3.2 +flake8-rst-docstrings==v0.0.13 +mypy==0.790 +Pygments pylint radon -mypy==0.790 -autopep8 diff --git a/requirements/packaging.txt b/requirements/packaging.txt index 9906383..af996cf 100644 --- a/requirements/packaging.txt +++ b/requirements/packaging.txt @@ -1,2 +1 @@ -# Requirements for packaging test (tox) twine diff --git a/requirements/tests.txt b/requirements/tests.txt index 45b26c5..9c15a43 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,6 +1,5 @@ -# Requirements for tests (pytest) --r common.txt +codecov pytest pytest-cov -tox selenium==3.141.0 +tox diff --git a/setup.cfg b/setup.cfg index e4cec31..fd9d770 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,32 +3,36 @@ license_file = LICENSE.txt [tool:pytest] testpaths = tests - -[coverage:run] -source = wheel -omit = .tox/* - -[coverage:report] -show_missing = true - -[radon] -cc_min = B +addopts = -v [flake8] max-line-length = 88 ignore = E129,E203,E501,W503,S101 exclude = - .tox - .egg + src/pyDataverse/docs + src/pyDataverse/schemas + src/pyDataverse/templates conf.py conftest.py + __pycache__ + .mypy_cache + .tox + .egg [pylint] max-line-length = 88 -[pylama:pycodestyle] -max_line_length = 88 +[coverage:run] +source=pyDataverse + +[coverage:report] +show_missing = true + +[coverage:html] +directory = docs/coverage/html + +[coverage:xml] +output = coverage.xml -[pylama:pylint] -max_line_length = 88 -disable = R +[radon] +cc_min = B diff --git a/setup.py b/setup.py index 7c07d8b..8ba5698 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,11 @@ import codecs import os import re -from setuptools.command.test import test as TestCommand +import sys + from setuptools import find_packages from setuptools import setup -import sys +from setuptools.command.test import test as TestCommand ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) diff --git a/src/pyDataverse/__init__.py b/src/pyDataverse/__init__.py index 965617b..6ff93c7 100644 --- a/src/pyDataverse/__init__.py +++ b/src/pyDataverse/__init__.py @@ -4,8 +4,6 @@ Licensed under the MIT License. """ -from __future__ import absolute_import - from requests.packages import urllib3 urllib3.disable_warnings() # noqa diff --git a/src/pyDataverse/api.py b/src/pyDataverse/api.py index 22aeefe..18bb67c 100644 --- a/src/pyDataverse/api.py +++ b/src/pyDataverse/api.py @@ -2,16 +2,19 @@ import json import subprocess as sp -from requests import ConnectionError, Response, delete, get, post, put +from requests import ConnectionError +from requests import delete +from requests import get +from requests import post +from requests import put +from requests import Response -from pyDataverse.exceptions import ( - ApiAuthorizationError, - ApiUrlError, - DatasetNotFoundError, - DataverseNotEmptyError, - DataverseNotFoundError, - OperationFailedError, -) +from pyDataverse.exceptions import ApiAuthorizationError +from pyDataverse.exceptions import ApiUrlError +from pyDataverse.exceptions import DatasetNotFoundError +from pyDataverse.exceptions import DataverseNotEmptyError +from pyDataverse.exceptions import DataverseNotFoundError +from pyDataverse.exceptions import OperationFailedError class Api: @@ -1639,7 +1642,7 @@ def get_datafile_metadata( # CHECK: Its not really clear, if the version query can also be done via ID. return self.get_request(url, auth=auth) - def upload_datafile(self, identifier, filename, json_str=None, is_pid=True): + def upload_datafile(self, identifier, filename, json_str=None, is_pid=True, content_type='application/octet-stream'): """Add file to a dataset. Add a file to an existing Dataset. Description and tags are optional: @@ -1666,6 +1669,8 @@ def upload_datafile(self, identifier, filename, json_str=None, is_pid=True): Metadata as JSON string. is_pid : bool ``True`` to use persistent identifier. ``False``, if not. + content_type : str + MIME type. Defaults to ``application/octet-stream``; this will prompt Dataverse to attempt to identify the MIME type of the file more accurately. Returns ------- @@ -1680,7 +1685,7 @@ def upload_datafile(self, identifier, filename, json_str=None, is_pid=True): else: url += "/datasets/{0}/add".format(identifier) - files = {"file": open(filename, "rb")} + files = {"file": (filename, open(filename, "rb"), content_type)} return self.post_request( url, data={"jsonData": json_str}, files=files, auth=True ) @@ -2083,6 +2088,7 @@ def get_children( - Unify tree and models """ + # print(self.base_url_api) children = [] if children_types is None: @@ -2181,7 +2187,7 @@ def get_user(self): https://guides.dataverse.org/en/latest/api/native-api.html#get-user-information-in-json-format """ - url = f"{self.base_url}/users/:me" + url = f"{self.base_url_api_native}/users/:me" return self.get_request(url, auth=True) def redetect_file_type( diff --git a/src/pyDataverse/docs/source/conf.py b/src/pyDataverse/docs/source/conf.py index b62a143..02050d7 100644 --- a/src/pyDataverse/docs/source/conf.py +++ b/src/pyDataverse/docs/source/conf.py @@ -3,21 +3,18 @@ # This file does only contain a selection of the most common options. For a # full list see the documentation: # http://www.sphinx-doc.org/en/master/config - # -- Path setup -------------------------------------------------------------- - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # sys.path.insert(0, os.path.abspath('../../')) - # -- Project information ----------------------------------------------------- - -import pyDataverse -from datetime import date import os import sys +from datetime import date + +import pyDataverse sys.path.insert(0, os.path.abspath("../../..")) diff --git a/src/pyDataverse/docs/source/index.rst b/src/pyDataverse/docs/source/index.rst index 181f0b7..9ab9fa1 100644 --- a/src/pyDataverse/docs/source/index.rst +++ b/src/pyDataverse/docs/source/index.rst @@ -39,7 +39,7 @@ Release v\ |version|. .. _homepage_description: -**pyDataverse** is a Python module for `Dataverse `_ you can use for: +**pyDataverse** is a Python module for `Dataverse `_ you can use for: - accessing the Dataverse `API's `_ - manipulating and using the Dataverse (meta)data - Dataverses, Datasets, Datafiles diff --git a/src/pyDataverse/docs/source/reference.rst b/src/pyDataverse/docs/source/reference.rst index d4787ef..1e92724 100644 --- a/src/pyDataverse/docs/source/reference.rst +++ b/src/pyDataverse/docs/source/reference.rst @@ -10,6 +10,8 @@ Where pyDataverse depends on external libraries, we document the most important right here and provide links to the canonical documentation outside of scope. +.. _reference_api: + API Interface ----------------------------- @@ -19,6 +21,8 @@ Access all of Dataverse APIs. :members: +.. _reference_models: + Models Interface ----------------------------- @@ -29,6 +33,8 @@ and `Datafile`). This includes import, export and manipulation. :inherited-members: +.. _reference_utils: + Utils Interface ----------------------------- @@ -38,6 +44,8 @@ Helper functions. :members: +.. _reference_exceptions: + Exceptions ----------------------------- diff --git a/src/pyDataverse/docs/source/user/advanced-usage.rst b/src/pyDataverse/docs/source/user/advanced-usage.rst index 6c98545..9ecc822 100644 --- a/src/pyDataverse/docs/source/user/advanced-usage.rst +++ b/src/pyDataverse/docs/source/user/advanced-usage.rst @@ -101,10 +101,10 @@ converts boolean values, and loads JSON cells properly. Once we have the data in Python, we can easily import the data into pyDataverse. -For this, loop over each Dataset :class:`dict`, to: +For this, loop over each Dataset :class:`dict`, to: #. Instantiate an empty :class:`Dataset ` -#. add the data with :meth:`set() ` and +#. add the data with :meth:`set() ` and #. append the instance to a :class:`list`. :: diff --git a/src/pyDataverse/docs/source/user/basic-usage.rst b/src/pyDataverse/docs/source/user/basic-usage.rst index b1d1807..919acf0 100644 --- a/src/pyDataverse/docs/source/user/basic-usage.rst +++ b/src/pyDataverse/docs/source/user/basic-usage.rst @@ -72,8 +72,8 @@ can then be used (e. g. :meth:`json() `). Create Dataverse Collection ----------------------------- -The top-level data-type in the Dataverse software is called a Dataverse collection, so we will start with that. -Take a look at the figure below to better understand the relationship between a Dataverse collection, a dataset, and a datafile. +The top-level data-type in the Dataverse software is called a Dataverse collection, so we will start with that. +Take a look at the figure below to better understand the relationship between a Dataverse collection, a dataset, and a datafile. .. figure:: ../_images/collection_dataset.png :align: center @@ -81,9 +81,9 @@ Take a look at the figure below to better understand the relationship between a A dataverse collection (also known as a :class:`Dataverse `) acts as a container for your :class:`Datasets`. It can also store other collections (:class:`Dataverses `). -You could create your own Dataverse collections, but it is not a requirement. -A Dataset is a container for :class:`Datafiles`, such as data, documentation, code, metadata, etc. -You need to create a Dataset to deposit your files. All Datasets are uniquely identified with a DOI at Dataverse. +You could create your own Dataverse collections, but it is not a requirement. +A Dataset is a container for :class:`Datafiles`, such as data, documentation, code, metadata, etc. +You need to create a Dataset to deposit your files. All Datasets are uniquely identified with a DOI at Dataverse. For more detailed explanations, check out `the Dataverse User Guide `_. Going back to the example, first, instantiate a :class:`Dataverse ` diff --git a/src/pyDataverse/models.py b/src/pyDataverse/models.py index a938869..2ae9a85 100644 --- a/src/pyDataverse/models.py +++ b/src/pyDataverse/models.py @@ -1,6 +1,4 @@ """Dataverse data-types data model.""" -from __future__ import absolute_import - import json import os diff --git a/tests/api/test_api.py b/tests/api/test_api.py index 7ae57d3..f645f4c 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -1,16 +1,20 @@ -from datetime import datetime import json import os +from datetime import datetime +from time import sleep + import pytest from requests import Response -from time import sleep + +from ..conftest import import_dataset_min_dict +from ..conftest import import_dataverse_min_dict +from ..conftest import test_config from pyDataverse.api import NativeApi from pyDataverse.exceptions import ApiAuthorizationError from pyDataverse.exceptions import ApiResponseError from pyDataverse.exceptions import ApiUrlError from pyDataverse.models import Dataset from pyDataverse.utils import read_file -from ..conftest import test_config, import_dataverse_min_dict, import_dataset_min_dict BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))) diff --git a/tests/conftest.py b/tests/conftest.py index 1c4eb11..83df1d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ """Find out more at https://github.com/GDCC/pyDataverse.""" import os + import pytest + from pyDataverse.api import NativeApi from pyDataverse.utils import read_json diff --git a/tests/models/test_datafile.py b/tests/models/test_datafile.py index 55793da..907bdb3 100644 --- a/tests/models/test_datafile.py +++ b/tests/models/test_datafile.py @@ -1,13 +1,15 @@ """Datafile data model tests.""" import json -import jsonschema import os import platform + +import jsonschema import pytest -from pyDataverse.models import Datafile -from pyDataverse.utils import read_file, write_json from ..conftest import test_config +from pyDataverse.models import Datafile +from pyDataverse.utils import read_file +from pyDataverse.utils import write_json def data_object(): diff --git a/tests/models/test_dataset.py b/tests/models/test_dataset.py index fb01d0f..1d83012 100644 --- a/tests/models/test_dataset.py +++ b/tests/models/test_dataset.py @@ -2,9 +2,11 @@ import json import os import platform + import pytest -from pyDataverse.models import Dataset + from ..conftest import test_config +from pyDataverse.models import Dataset def read_file(filename, mode="r"): diff --git a/tests/models/test_dataverse.py b/tests/models/test_dataverse.py index a4c362e..5245d64 100644 --- a/tests/models/test_dataverse.py +++ b/tests/models/test_dataverse.py @@ -1,11 +1,13 @@ """Dataverse data model tests.""" import json -import jsonschema import os import platform + +import jsonschema import pytest -from pyDataverse.models import Dataverse + from ..conftest import test_config +from pyDataverse.models import Dataverse def read_file(filename, mode="r"): diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index baf5d9d..b9c7233 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -1,5 +1,7 @@ -from pyDataverse.utils import read_json, save_tree_data, dataverse_tree_walker from ..conftest import test_config +from pyDataverse.utils import dataverse_tree_walker +from pyDataverse.utils import read_json +from pyDataverse.utils import save_tree_data class TestUtilsSaveTreeData: diff --git a/tox.ini b/tox.ini index ed5c4a3..f80c1d1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37,py38,coverage,coveralls,docs,packaging,dist_install +envlist = py36,py37,py38,coverage,docs,packaging,dist_install skip_missing_interpreters = True ignore_basepython_conflict = True @@ -37,12 +37,12 @@ deps = commands = pytest tests/ --cov=pyDataverse --cov-report=term-missing --cov-report=xml --cov-report=html -[testenv:coveralls] -description = create reports for coveralls +[testenv:codecov] +description = create report for codecov deps = -r{toxinidir}/requirements/tests.txt commands = - pytest tests/ --doctest-modules -v --cov=pyDataverse + codecov [testenv:pylint] description = pylint for linting @@ -56,47 +56,21 @@ commands = deps = -r{toxinidir}/requirements/lint.txt commands = - mypy src/pyDataverse/ setup.py tests/ - -[flake8] -max-line-length = 80 -ignore = E129 -exclude = - src/pyDataverse/docs/source/conf.py - .tox - .egg + mypy src/pyDataverse/ setup.py [testenv:flake8] description = flake8 for style guide and docstring testing deps = -r{toxinidir}/requirements/lint.txt commands = - pip uninstall -y flake8-pytest-style - pip uninstall -y flake8-rst-docstrings - flake8 src/pyDataverse/ + flake8 src/pyDataverse -[testenv:flake8_docs] +[testenv:flake8_docstrings] description = flake8 for style guide and docstring testing deps = -r{toxinidir}/requirements/lint.txt commands = - pip uninstall -y flake8-blind-except - pip uninstall -y flake8-builtins - pip uninstall -y flake8-bandit - pip uninstall -y flake8-breakpoint - pip uninstall -y flake8-bugbear - pip uninstall -y flake8-comprehensions - pip uninstall -y flake8-requirements - pip uninstall -y flake8-return flake8 --docstring-convention numpy src/pyDataverse/ - flake8 --docstring-convention numpy tests/ - -[testenv:flake8_tests] -description = flake8 for style guide and docstring testing -deps = - -r{toxinidir}/requirements/lint.txt -commands = - flake8 tests/ [testenv:black] description = black for auto-formatting @@ -133,9 +107,6 @@ recreate = True deps = -r{toxinidir}/requirements/packaging.txt commands = - # rm -R {toxinidir}/src/pyDataverse.egg-info - # rm -R {toxinidir}/dist - # rm -R {toxinidir}/build python setup.py sdist bdist_wheel twine check dist/* @@ -151,7 +122,7 @@ description = install from test pypi skip_install = True recreate = True commands = - pip install --index-url https://test.pypi.org/simple/ pyDataverse + pip install -i https://test.pypi.org/simple/ pyDataverse --extra-index-url https://pypi.org/simple pyDataverse pip freeze python -c "import pyDataverse; print(pyDataverse.__name__); print(pyDataverse.__version__)"