Add tests for pip's vcs module from pip

WARNING: tests will fail on this commit.
Next commit will fix import paths and remove unnecessary stuff for the
tests to work.

File structure has been modified from pip, below is the mapping of
fetchcode to pip. Files were copied without any modifications in this
commit. Paths from respective repositories roots:
{
"tests/conftest.py":"tests/conftest.py",
"tests/test_vcs_pip_bazaar.py":"tests/functional/test_vcs_bazaar.py",
"tests/test_vcs_pip_git.py":"tests/functional/test_vcs_git.py",
"tests/test_vcs_pip_mercurial.py":"tests/functional/test_vcs_mercurial.py",
"tests/test_vcs_pip.py_subversion":"tests/functional/test_vcs_subversion.py",
"tests/test_vcs_pip.py":"tests/unit/test_vcs.py",
"tests/test_vcs_pip_mercurial_unit.py":"tests/unit/test_vcs.py",
"tests/lib/__init__.py":"tests/lib/__init_.py",
"tests/lib/git_submodule_helpers.py":"tests/lib/git_submodule_helpers.py",
"tests/lib/local_repos.py":"tests/lib/local_repos.py",
"tests/lib/path.py":"tests/lib/path.py",
}

It has been agreed with @pombredanne & @TG1999 that history from pip
will be rebased on fetchcode by @pombredanne (thanks!). It will be done
only for the files that are of concern to fetchcode.

I'm leaving this commit without SoB intentionally, as this is not my
work, but that of the many pip's authors:
https://github.com/pypa/pip/blob/21.2.4/AUTHORS.txt
License of pip: MIT (https://pypi.org/project/pip/)

add conftest

Signed-off-by: Alexander Mazuruk <a.mazuruk@samsung.com>

Author: aalexanderr

tests/conftest.py

@@ -0,0 +1,21 @@
+import contextlib
+from typing import Iterator
+
+
+@contextlib.contextmanager
+def nullcontext():
+    # type: () -> Iterator[None]
+    """
+    Context manager that does no additional processing.
+
+    Used as a stand-in for a normal context manager, when a particular block of
+    code is only sometimes used with a normal context manager:
+
+        cm = optional_cm if condition else nullcontext()
+        with cm:
+            # Perform operation, using optional_cm if condition is True
+
+    TODO: Replace with contextlib.nullcontext after dropping Python 3.6
+    support.
+    """
+    yield

tests/lib/__init__.py

@@ -0,0 +1,74 @@
+import textwrap
+
+from tests.lib import _create_main_file, _git_commit
+
+
+def _create_test_package_submodule(env):
+    env.scratch_path.joinpath("version_pkg_submodule").mkdir()
+    submodule_path = env.scratch_path / "version_pkg_submodule"
+    env.run("touch", "testfile", cwd=submodule_path)
+    env.run("git", "init", cwd=submodule_path)
+    env.run("git", "add", ".", cwd=submodule_path)
+    _git_commit(env, submodule_path, message="initial version / submodule")
+
+    return submodule_path
+
+
+def _change_test_package_submodule(env, submodule_path):
+    submodule_path.joinpath("testfile").write_text("this is a changed file")
+    submodule_path.joinpath("testfile2").write_text("this is an added file")
+    env.run("git", "add", ".", cwd=submodule_path)
+    _git_commit(env, submodule_path, message="submodule change")
+
+
+def _pull_in_submodule_changes_to_module(env, module_path, rel_path):
+    """
+    Args:
+      rel_path: the location of the submodule relative to the superproject.
+    """
+    submodule_path = module_path / rel_path
+    env.run("git", "pull", "-q", "origin", "master", cwd=submodule_path)
+    # Pass -a to stage the submodule changes that were just pulled in.
+    _git_commit(env, module_path, message="submodule change", stage_modified=True)
+
+
+def _create_test_package_with_submodule(env, rel_path):
+    """
+    Args:
+      rel_path: the location of the submodule relative to the superproject.
+    """
+    env.scratch_path.joinpath("version_pkg").mkdir()
+    version_pkg_path = env.scratch_path / "version_pkg"
+    version_pkg_path.joinpath("testpkg").mkdir()
+    pkg_path = version_pkg_path / "testpkg"
+
+    pkg_path.joinpath("__init__.py").write_text("# hello there")
+    _create_main_file(pkg_path, name="version_pkg", output="0.1")
+    version_pkg_path.joinpath("setup.py").write_text(
+        textwrap.dedent(
+            """\
+                        from setuptools import setup, find_packages
+                        setup(name='version_pkg',
+                              version='0.1',
+                              packages=find_packages(),
+                             )
+                        """
+        )
+    )
+    env.run("git", "init", cwd=version_pkg_path)
+    env.run("git", "add", ".", cwd=version_pkg_path)
+    _git_commit(env, version_pkg_path, message="initial version")
+
+    submodule_path = _create_test_package_submodule(env)
+
+    env.run(
+        "git",
+        "submodule",
+        "add",
+        submodule_path,
+        rel_path,
+        cwd=version_pkg_path,
+    )
+    _git_commit(env, version_pkg_path, message="initial version w submodule")
+
+    return version_pkg_path, submodule_path

tests/lib/compat.py

@@ -0,0 +1,64 @@
+import os
+import subprocess
+import urllib.request
+
+from pip._internal.utils.misc import hide_url
+from pip._internal.vcs import vcs
+from tests.lib import path_to_url
+from tests.lib.path import Path
+
+
+def _create_svn_initools_repo(initools_dir):
+    """
+    Create the SVN INITools repo.
+    """
+    directory = os.path.dirname(initools_dir)
+    subprocess.check_call("svnadmin create INITools".split(), cwd=directory)
+
+    filename, _ = urllib.request.urlretrieve(
+        "http://bitbucket.org/hltbra/pip-initools-dump/raw/8b55c908a320/"
+        "INITools_modified.dump"
+    )
+    with open(filename) as dump:
+        subprocess.check_call(
+            ["svnadmin", "load", initools_dir],
+            stdin=dump,
+            stdout=subprocess.DEVNULL,
+        )
+    os.remove(filename)
+
+
+def local_checkout(
+    remote_repo,  # type: str
+    temp_path,  # type: Path
+):
+    # type: (...) -> str
+    """
+    :param temp_path: the return value of the tmpdir fixture, which is a
+        temp directory Path object unique to each test function invocation,
+        created as a sub directory of the base temp directory.
+    """
+    assert "+" in remote_repo
+    vcs_name = remote_repo.split("+", 1)[0]
+    repository_name = os.path.basename(remote_repo)
+
+    directory = temp_path.joinpath("cache")
+    repo_url_path = os.path.join(directory, repository_name)
+    assert not os.path.exists(repo_url_path)
+
+    if not os.path.exists(directory):
+        os.mkdir(directory)
+
+    if vcs_name == "svn":
+        assert repository_name == "INITools"
+        _create_svn_initools_repo(repo_url_path)
+        repo_url_path = os.path.join(repo_url_path, "trunk")
+    else:
+        vcs_backend = vcs.get_backend(vcs_name)
+        vcs_backend.obtain(repo_url_path, url=hide_url(remote_repo))
+
+    return "{}+{}".format(vcs_name, path_to_url(repo_url_path))
+
+
+def local_repo(remote_repo, temp_path):
+    return local_checkout(remote_repo, temp_path).split("+", 1)[1]

tests/lib/git_submodule_helpers.py

@@ -0,0 +1,192 @@
+# flake8: noqa
+# Author: Aziz Köksal
+import glob
+import os
+
+try:
+    from os import supports_fd
+except ImportError:
+    supports_fd = set()
+
+
+class Path(str):
+    """
+    Models a path in an object oriented way.
+    """
+
+    # File system path separator: '/' or '\'.
+    sep = os.sep
+
+    # Separator in the PATH environment variable.
+    pathsep = os.pathsep
+
+    def __new__(cls, *paths):
+        if len(paths):
+            return super().__new__(cls, os.path.join(*paths))
+        return super().__new__(cls)
+
+    def __div__(self, path):
+        """
+        Joins this path with another path.
+
+        >>> path_obj / 'bc.d'
+        >>> path_obj / path_obj2
+        """
+        return Path(self, path)
+
+    __truediv__ = __div__
+
+    def __rdiv__(self, path):
+        """
+        Joins this path with another path.
+
+        >>> "/home/a" / path_obj
+        """
+        return Path(path, self)
+
+    __rtruediv__ = __rdiv__
+
+    def __idiv__(self, path):
+        """
+        Like __div__ but also assigns to the variable.
+
+        >>> path_obj /= 'bc.d'
+        """
+        return Path(self, path)
+
+    __itruediv__ = __idiv__
+
+    def __add__(self, path):
+        """
+        >>> Path('/home/a') + 'bc.d'
+        '/home/abc.d'
+        """
+        return Path(str(self) + path)
+
+    def __radd__(self, path):
+        """
+        >>> '/home/a' + Path('bc.d')
+        '/home/abc.d'
+        """
+        return Path(path + str(self))
+
+    def __repr__(self):
+        return "Path({inner})".format(inner=str.__repr__(self))
+
+    @property
+    def name(self):
+        """
+        '/home/a/bc.d' -> 'bc.d'
+        """
+        return os.path.basename(self)
+
+    @property
+    def stem(self):
+        """
+        '/home/a/bc.d' -> 'bc'
+        """
+        return Path(os.path.splitext(self)[0]).name
+
+    @property
+    def suffix(self):
+        """
+        '/home/a/bc.d' -> '.d'
+        """
+        return Path(os.path.splitext(self)[1])
+
+    def resolve(self):
+        """
+        Resolves symbolic links.
+        """
+        return Path(os.path.realpath(self))
+
+    @property
+    def parent(self):
+        """
+        Returns the parent directory of this path.
+
+        '/home/a/bc.d' -> '/home/a'
+        '/home/a/' -> '/home/a'
+        '/home/a' -> '/home'
+        """
+        return Path(os.path.dirname(self))
+
+    def exists(self):
+        """
+        Returns True if the path exists.
+        """
+        return os.path.exists(self)
+
+    def mkdir(self, mode=0x1FF, exist_ok=False, parents=False):  # 0o777
+        """
+        Creates a directory, if it doesn't exist already.
+
+        :param parents: Whether to create parent directories.
+        """
+
+        maker_func = os.makedirs if parents else os.mkdir
+        try:
+            maker_func(self, mode)
+        except OSError:
+            if not exist_ok or not os.path.isdir(self):
+                raise
+
+    def unlink(self):
+        """
+        Removes a file.
+        """
+        return os.remove(self)
+
+    def rmdir(self):
+        """
+        Removes a directory.
+        """
+        return os.rmdir(self)
+
+    def rename(self, to):
+        """
+        Renames a file or directory. May throw an OSError.
+        """
+        return os.rename(self, to)
+
+    def glob(self, pattern):
+        return (Path(i) for i in glob.iglob(self.joinpath(pattern)))
+
+    def joinpath(self, *parts):
+        return Path(self, *parts)
+
+    # TODO: Remove after removing inheritance from str.
+    def join(self, *parts):
+        raise RuntimeError("Path.join is invalid, use joinpath instead.")
+
+    def read_bytes(self):
+        # type: () -> bytes
+        with open(self, "rb") as fp:
+            return fp.read()
+
+    def write_bytes(self, content):
+        # type: (bytes) -> None
+        with open(self, "wb") as f:
+            f.write(content)
+
+    def read_text(self):
+        with open(self, "r") as fp:
+            return fp.read()
+
+    def write_text(self, content):
+        with open(self, "w") as fp:
+            fp.write(content)
+
+    def touch(self):
+        with open(self, "a") as fp:
+            path = fp.fileno() if os.utime in supports_fd else self
+            os.utime(path, None)  # times is not optional on Python 2.7
+
+    def symlink_to(self, target):
+        os.symlink(target, self)
+
+    def stat(self):
+        return os.stat(self)
+
+
+curdir = Path(os.path.curdir)