Allow throttling testing based on PR labels

master/master.cfg

@@ -942,6 +942,69 @@ def add_halide_cmake_package_steps(factory, builder_type):
         groupfn=pkg_version_and_target))
 
 
+def any_categories_enabled(categories):
+    @renderer
+    @defer.inlineCallbacks
+    def render(props):
+        halide_label_names = props.getProperty('github.halide_label_names')
+        # print('github.halide_label_names is ', halide_label_names)
+        # print('categories is ', categories)
+
+        # When this label is present we ignore the PR immediately,
+        # so even if the label is present, we shouldn't get here
+        assert 'buildbot_test_nothing' not in halide_label_names
+
+        if 'buildbot_test_everything' in halide_label_names:
+            result = True
+        elif 'cpu' in categories:
+            # CPU is always enabled
+            result = True
+        elif 'gpu' in categories and 'buildbot_test_gpu' in halide_label_names:
+            result = True
+        elif 'hvx' in categories and 'buildbot_test_hvx' in halide_label_names:
+            result = True
+        elif 'wasm' in categories and 'buildbot_test_wasm' in halide_label_names:
+            result = True
+        else:
+            result = False
+
+        result = yield props.render(result)
+        return result
+
+    return render
+
+
+_GPU_FEATURES = [
+    "cuda",
+    "d3d12compute",
+    "metal",
+    "opencl",
+    "openglcompute",
+]
+
+
+_HVX_FEATURES = [
+    "hvx"
+]
+
+
+_WASM_FEATURES = [
+    "wasm"
+]
+
+
+def get_categories(halide_target, test_labels):
+    # test_labels is unused for now, but might be someday, so keep it here
+    cats = []
+    if any([f in halide_target for f in _GPU_FEATURES]):
+        cats.append('gpu')
+    if any([f in halide_target for f in _HVX_FEATURES]):
+        cats.append('hvx')
+    if any([f in halide_target for f in _WASM_FEATURES]):
+        cats.append('wasm')
+    return cats
+
+
 # Return a dict with halide-targets as the keys, and a list of test-labels for each value.
 def get_test_labels(builder_type):
     targets = defaultdict(list)
@@ -1026,9 +1089,22 @@ def add_halide_cmake_test_steps(factory, builder_type):
         if halide_target.startswith("wasm-"):
             env = merge_renderable(env, Property('emsdk_env', default={}))
 
+        test_labels = labels[halide_target]
+
+        do_apps = 'apps' in test_labels
+        if do_apps:
+            test_labels.remove('apps')
+
+        if not builder_type.handles_python():
+            if 'python' in test_labels:
+                test_labels.remove('python')
+            # TODO : some of the apps require python, so we must skip them for now also
+            do_apps = False
+
         factory.addStep(
             CMake(name='Reconfigure for Halide_TARGET=%s' % halide_target,
                   description='Reconfigure for Halide_TARGET=%s' % halide_target,
+                  doStepIf=any_categories_enabled(get_categories(halide_target, test_labels)),
                   locks=[performance_lock.access('counting')],
                   haltOnFailure=True,
                   env=env,
@@ -1042,25 +1118,13 @@ def add_halide_cmake_test_steps(factory, builder_type):
         factory.addStep(
             ShellCommand(name='Rebuild for Halide_TARGET=%s' % halide_target,
                          description='Rebuild Halide for Halide_TARGET=%s' % halide_target,
+                         doStepIf=any_categories_enabled(get_categories(halide_target, test_labels)),
                          locks=[performance_lock.access('counting')],
                          haltOnFailure=True,
                          workdir=build_dir,
                          env=env,
                          command=get_cmake_build_command(builder_type, build_dir, targets=['all', 'install'])))
 
-        test_labels = labels[halide_target]
-
-        do_apps = 'apps' in test_labels
-        if do_apps:
-            test_labels.remove('apps')
-
-        if not builder_type.handles_python():
-            if 'python' in test_labels:
-                test_labels.remove('python')
-
-            # TODO : some of the apps require python, so we must skip them for now also
-            do_apps = False
-
         parallel_test_labels = [
             test for test in test_labels if not is_time_critical_test(test)]
         exclusive_test_labels = [test for test in test_labels if is_time_critical_test(test)]
@@ -1091,6 +1155,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
             factory.addStep(
                 CTest(name='Test %s Halide_TARGET=%s' % (test_set, halide_target),
                       description='Test %s Halide_TARGET=%s' % (test_set, halide_target),
+                      doStepIf=any_categories_enabled(get_categories(halide_target, parallel_test_labels)),
                       locks=[performance_lock.access('counting')],
                       workdir=build_dir,
                       env=env,
@@ -1105,6 +1170,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
             factory.addStep(
                 CTest(name='Test %s Halide_TARGET=%s' % (test_set, halide_target),
                       description='Test %s Halide_TARGET=%s' % (test_set, halide_target),
+                      doStepIf=any_categories_enabled(get_categories(halide_target, exclusive_test_labels)),
                       locks=[performance_lock.access('exclusive')],
                       workdir=build_dir,
                       env=env,
@@ -1125,6 +1191,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
             factory.addStep(
                 CMake(name='Configure apps for Halide_TARGET=%s' % halide_target,
                       description='Configure apps for Halide_TARGET=%s' % halide_target,
+                      doStepIf=any_categories_enabled(get_categories(halide_target, 'apps')),
                       locks=[performance_lock.access('counting')],
                       haltOnFailure=True,
                       env=env,
@@ -1137,6 +1204,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
             factory.addStep(
                 ShellCommand(name='Build apps for Halide_TARGET=%s' % halide_target,
                              description='Build apps for Halide_TARGET=%s' % halide_target,
+                             doStepIf=any_categories_enabled(get_categories(halide_target, 'apps')),
                              locks=[performance_lock.access('counting')],
                              haltOnFailure=True,
                              workdir=apps_build_dir,
@@ -1156,6 +1224,7 @@ def add_halide_cmake_test_steps(factory, builder_type):
             factory.addStep(
                 CTest(name='Test apps for Halide_TARGET=%s' % halide_target,
                       description='Test apps for Halide_TARGET=%s' % halide_target,
+                      doStepIf=any_categories_enabled(get_categories(halide_target, 'apps')),
                       locks=[performance_lock.access('exclusive')],
                       workdir=apps_build_dir,
                       env=env,
@@ -1220,19 +1289,21 @@ def create_halide_make_factory(builder_type):
             p = make_threads
             lock_mode = 'counting'
 
-        if label != 'build_tests':
-            label = 'test_%s' % label
+        full_label = label
+        if full_label != 'build_tests':
+            full_label = 'test_%s' % label
 
-        factory.addStep(ShellCommand(name='make ' + label,
-                                     description=label + ' ' + halide_target,
+        factory.addStep(ShellCommand(name='make ' + full_label,
+                                     description=full_label + ' ' + halide_target,
+                                     doStepIf=any_categories_enabled(get_categories(halide_target, [label])),
                                      locks=[performance_lock.access(lock_mode)],
                                      workdir=build_dir,
                                      env=env,
                                      haltOnFailure=False,
                                      command=['make',
                                               '-f', get_halide_source_path('Makefile'),
                                               '-j', p,
-                                              label],
+                                              full_label],
                                      timeout=3600))
     return factory
 
@@ -1494,6 +1565,15 @@ c['prioritizeBuilders'] = prioritize_builders
 
 # GitHub pull request filter
 
+# We vary testing depending on how a PR is labeled:
+# - if 'buildbot_test_nothing' is present, no testing is done and the rest are ignored.
+#   ('skip_buildbots' is a synonym for this.)
+# - if 'buildbot_test_everything' is present, we do all possible testing.
+# - Otherwise, we do (at least) baseline CPU testing on all targets.
+#   - if 'buildbot_test_gpu' is present, we also test all GPU targets (cuda, opencl, metal, d3d12, etc)
+#   - if 'buildbot_test_hvx' is present, we also test Hexagon
+#   - if 'buildbot_test_wasm' is present, we also test WebAssembly
+
 class SafeGitHubEventHandler(GitHubEventHandler):
     def handle_push(self, payload, event):
         ref = payload['ref']
@@ -1506,14 +1586,17 @@ class SafeGitHubEventHandler(GitHubEventHandler):
     def handle_pull_request(self, payload, event):
         pr = payload['pull_request']
         try:
-            # Skip anything with the 'skip_buildbots' label
-            if any(label['name'] == 'skip_buildbots' for label in pr['labels']):
+            # Extract just the label names for ease of examination later
+            pr['halide_label_names'] = [label['name'] for label in pr['labels']]
+
+            if 'buildbot_test_nothing' in pr['halide_label_names'] or \
+               'skip_buildbots' in pr['halide_label_names']:
                 # print("PR %s was skipped due to skip_buildbots" % str(pr['html_url']))
                 return self.skip()
 
             # Test anything (even external) that has 'halidebuildbots' as a reviewer.
             if any(r['login'] == 'halidebuildbots' for r in pr['requested_reviewers']):
-                # print("PR %s was handled due halidebuildbots" % str(pr['html_url']))
+                # print("PR %s was handled due to halidebuildbots" % str(pr['html_url']))
                 if payload['action'] == 'review_requested':
                     # Pretend it's a synchronize event instead since private buildbot code
                     # rejects review_requested for no apparent reason.
@@ -1525,6 +1608,16 @@ class SafeGitHubEventHandler(GitHubEventHandler):
                 # print("PR %s was skipped due to being external:" % str(pr['head']['repo']['full_name']))
                 return self.skip()
 
+            # Changing one of the 'magic' labels should be treated like a synchronize, too.
+            if payload['action'] in ('labeled', 'unlabeled'):
+                label_changed = payload['label']['name']
+                # print("PR %s label_changed=", label_changed)
+                if label_changed.startswith('buildbot_test_'):
+                    # print("PR %s was handled due to action=%s label=%s" % \
+                    # (str(pr['html_url']), str(payload['action']), str(label_changed)))
+                    payload['action'] = 'synchronize'
+                    return super().handle_pull_request(payload, event)
+
             # print("PR %s is being handled normally" % str(pr['html_url']))
             return super().handle_pull_request(payload, event)
 
@@ -1562,7 +1655,7 @@ c['www'] = dict(
             'codebase': 'halide',
             'skips': [],
             'class': SafeGitHubEventHandler,
-            # 'github_property_whitelist': ['github.base.ref'],
+            'github_property_whitelist': ['github.halide_label_names'],
         },
     },
 )