When testing Grakn, randomize its port and poll to detect its startup (typedb#180)

## What is the goal of this PR?

Previously when running tests, Grakn would always start on port 1729. This rendered it impossible for us to parallelise our tests. Now, we pick a random port number between 40000 and 60000.

Also, previously we waited N milliseconds and then ran the tests. This was wasteful, because often Grakn would start way in advance; and unreliable, because sometimes Grakn would not start in time and CI would fail. Now, we add a polling loop that checks if the server port is in use, and signals that the tests may begin once the port is in use.

## What are the changes implemented in this PR?

- Randomize port when running Grakn during BDD tests
- During BDD tests, poll the ports to detect when Grakn has actually started
- Enable test parallelisation in CI

Author: Alex Walker

.bazelrc

@@ -20,13 +20,3 @@
 build --incompatible_strict_action_env
 run --incompatible_strict_action_env
 test --incompatible_strict_action_env --test_env=PATH
-
-# what is defined in this section will be applied when bazel is invoked like this: bazel ... --config=rbe ...
-build:rbe --project_id=grakn-dev
-build:rbe --remote_cache=cloud.buildbuddy.io
-build:rbe --bes_backend=cloud.buildbuddy.io
-build:rbe --bes_results_url=https://app.buildbuddy.io/invocation/
-build:rbe --tls_client_certificate=/opt/credentials/buildbuddy-cert.pem
-build:rbe --tls_client_key=/opt/credentials/buildbuddy-key.pem
-build:rbe --remote_timeout=3600
-build:rbe --remote_download_minimal

.grabl/automation.yml

@@ -73,7 +73,7 @@ build:
         export ARTIFACT_USERNAME=$REPO_GRAKN_USERNAME
         export ARTIFACT_PASSWORD=$REPO_GRAKN_PASSWORD
         bazel run @graknlabs_dependencies//distribution/artifact:create-netrc
-        bazel test //tests/behaviour/concept/... --test_output=errors --jobs=1
+        bazel test //tests/behaviour/concept/... --test_output=errors
     test-behaviour-match:
       image: graknlabs-ubuntu-20.04
       type: foreground
@@ -86,8 +86,8 @@ build:
         export ARTIFACT_USERNAME=$REPO_GRAKN_USERNAME
         export ARTIFACT_PASSWORD=$REPO_GRAKN_PASSWORD
         bazel run @graknlabs_dependencies//distribution/artifact:create-netrc
-        bazel test //tests/behaviour/graql/language/match/... --test_output=errors --jobs=1
-        bazel test //tests/behaviour/graql/language/get/... --test_output=errors --jobs=1
+        bazel test //tests/behaviour/graql/language/match/... --test_output=errors
+        bazel test //tests/behaviour/graql/language/get/... --test_output=errors
     test-behaviour-writable:
       image: graknlabs-ubuntu-20.04
       type: foreground
@@ -100,10 +100,10 @@ build:
         export ARTIFACT_USERNAME=$REPO_GRAKN_USERNAME
         export ARTIFACT_PASSWORD=$REPO_GRAKN_PASSWORD
         bazel run @graknlabs_dependencies//distribution/artifact:create-netrc
-        bazel test //tests/behaviour/graql/language/insert/... --test_output=errors --jobs=1
-        bazel test //tests/behaviour/graql/language/delete/... --test_output=errors --jobs=1
-        bazel test //tests/behaviour/graql/language/update:checkstyle --test_output=errors --jobs=1
-        bazel test //tests/behaviour/graql/language/update:test-core --test_output=errors --jobs=1
+        bazel test //tests/behaviour/graql/language/insert/... --test_output=errors
+        bazel test //tests/behaviour/graql/language/delete/... --test_output=errors
+        bazel test //tests/behaviour/graql/language/update:checkstyle --test_output=errors
+        bazel test //tests/behaviour/graql/language/update:test-core --test_output=errors
     test-behaviour-definable:
       image: graknlabs-ubuntu-20.04
       type: foreground
@@ -116,10 +116,10 @@ build:
         export ARTIFACT_USERNAME=$REPO_GRAKN_USERNAME
         export ARTIFACT_PASSWORD=$REPO_GRAKN_PASSWORD
         bazel run @graknlabs_dependencies//distribution/artifact:create-netrc
-        bazel test //tests/behaviour/graql/language/define:checkstyle --test_output=errors --jobs=1
-        bazel test //tests/behaviour/graql/language/define:test-core --test_output=errors --jobs=1
-        bazel test //tests/behaviour/graql/language/undefine:checkstyle --test_output=errors --jobs=1
-        bazel test //tests/behaviour/graql/language/undefine:test-core --test_output=errors --jobs=1
+        bazel test //tests/behaviour/graql/language/define:checkstyle --test_output=errors
+        bazel test //tests/behaviour/graql/language/define:test-core --test_output=errors
+        bazel test //tests/behaviour/graql/language/undefine:checkstyle --test_output=errors
+        bazel test //tests/behaviour/graql/language/undefine:test-core --test_output=errors
     test-cluster-failover:
       image: graknlabs-ubuntu-20.04
       type: foreground

BUILD

@@ -32,7 +32,6 @@ load("@graknlabs_dependencies//distribution:deployment.bzl", "deployment")
 load(":deployment.bzl", github_deployment = "deployment")
 
 
-
 py_library(
     name = "client_python",
     srcs = glob(["grakn/**/*.py"]),
@@ -114,10 +113,8 @@ release_validate_python_deps(
 filegroup(
     name = "ci",
     data = [
-        "@graknlabs_dependencies//tool/bazelrun:rbe",
+        "@graknlabs_dependencies//tool/checkstyle:test-coverage",
         "@graknlabs_dependencies//distribution/artifact:create-netrc",
-        "@graknlabs_dependencies//tool/sync:dependencies",
-        "@graknlabs_dependencies//tool/release:approval",
         "@graknlabs_dependencies//tool/release:create-notes",
     ],
 )

dependencies/graknlabs/artifacts.bzl

@@ -37,5 +37,5 @@ def graknlabs_grakn_cluster_artifacts():
         artifact_name = "grakn-cluster-server-{platform}-{version}.{ext}",
         tag_source = deployment_private["artifact.release"],
         commit_source = deployment_private["artifact.snapshot"],
-        commit = "892073639b024904a413154011fb42d9dae090f3",
+        commit = "93cbc149d7b9ce588e52d8132c8a0b2265658a3c",
     )

tests/behaviour/background/cluster/environment.py

@@ -27,7 +27,7 @@
 
 def before_all(context: Context):
     environment_base.before_all(context)
-    context.client = GraknClient.cluster([GraknClient.DEFAULT_ADDRESS])
+    context.client = GraknClient.cluster(addresses=["localhost:" + context.config.userdata["port"]])
 
 
 def before_scenario(context: Context, scenario):

tests/behaviour/background/core/environment.py

@@ -27,7 +27,7 @@
 
 def before_all(context: Context):
     environment_base.before_all(context)
-    context.client = GraknClient.core()
+    context.client = GraknClient.core(address="localhost:%d" % int(context.config.userdata["port"]))
 
 
 def before_scenario(context: Context, scenario):

tests/behaviour/background/environment_base.py

@@ -28,7 +28,6 @@
 
 def before_all(context: Context):
     context.THREAD_POOL_SIZE = 32
-    context.client = GraknClient.core()
 
 
 def before_scenario(context: Context, scenario):

tests/behaviour/context.py

@@ -50,6 +50,16 @@
 ConceptSubtype: Concept = Union[ThingSubtype, TypeSubtype]
 
 
+class Config:
+    """
+    Type definitions for Config.
+
+    This class should not be instantiated. The initialisation of the actual Config object occurs in environment.py.
+    """
+    def __init__(self):
+        self.userdata = {}
+
+
 class Context(behave.runner.Context):
     """
     Type definitions for Context.
@@ -69,6 +79,7 @@ def __init__(self):
         self.numeric_answer: Optional[Numeric] = None
         self.answer_groups: Optional[List[ConceptMapGroup]] = None
         self.numeric_answer_groups: Optional[List[NumericGroup]] = None
+        self.config = Config()
 
     def tx(self) -> Transaction:
         return self.sessions_to_transactions[self.sessions[0]][0]

tools/behave_rule.bzl

@@ -60,19 +60,57 @@ def _rule_implementation(ctx):
              fi
            fi
            DIRECTORY=$(ls ./grakn_distribution)
-           echo Successfully unarchived Grakn distribution.
-           echo Starting Grakn Server
+
+           if [[ $GRAKN_DISTRO == *"cluster"* ]]; then
+             PRODUCT=Cluster
+           else
+             PRODUCT=Core
+           fi
+
+           echo Successfully unarchived Grakn $PRODUCT distribution.
+
+           RND=20001
+           while [ $RND -gt 20000 ]  # Guarantee fair distribution of random ports
+           do
+             RND=$RANDOM
+           done
+           PORT=$((40000 + $RND))
+
+           echo Starting Grakn $PRODUCT Server.
            mkdir ./grakn_distribution/"$DIRECTORY"/grakn_test
-           ./grakn_distribution/"$DIRECTORY"/grakn server --data grakn_test &
-           sleep 9
+           if [[ $PRODUCT == "Core" ]]; then
+             ./grakn_distribution/"$DIRECTORY"/grakn server --port $PORT --data grakn_test &
+           else
+             ./grakn_distribution/"$DIRECTORY"/grakn server --address "127.0.0.1:$PORT:$(($PORT+1))" --data grakn_test &
+           fi
+
+           POLL_INTERVAL_SECS=0.5
+           MAX_RETRIES=60
+           RETRY_NUM=0
+           while [[ $RETRY_NUM -lt $MAX_RETRIES ]]; do
+             RETRY_NUM=$(($RETRY_NUM + 1))
+             if [[ $(($RETRY_NUM % 4)) -eq 0 ]]; then
+               echo Waiting for Grakn $PRODUCT server to start \($(($RETRY_NUM / 2))s\)...
+             fi
+             lsof -i :$PORT && STARTED=1 || STARTED=0
+             if [[ $STARTED -eq 1 ]]; then
+               break
+             fi
+             sleep $POLL_INTERVAL_SECS
+           done
+           if [[ $STARTED -eq 0 ]]; then
+             echo Failed to start Grakn $PRODUCT server
+             exit 1
+           fi
+           echo Grakn $PRODUCT database server started
 
            """
     # TODO: If two step files have the same name, we should rename the second one to prevent conflict
     cmd += "cp %s %s" % (ctx.files.background[0].path, feats_dir)
     cmd += " && rm -rf " + steps_out_dir
     cmd += " && mkdir " + steps_out_dir + " && "
     cmd += " && ".join(["cp %s %s" % (step_file.path, steps_out_dir) for step_file in ctx.files.steps])
-    cmd += " && behave %s && export RESULT=0 || export RESULT=1" % feats_dir
+    cmd += " && behave %s -D port=$PORT && export RESULT=0 || export RESULT=1" % feats_dir
     cmd += """
            echo Tests concluded with exit value $RESULT
            echo Stopping server.