Tests against packaged SDK added to Nightly Test Report

Author: sunmou99

.github/workflows/cpp-packaging.yml

@@ -720,9 +720,13 @@ jobs:
         if [[ -z ${USE_EXPANDED_MATRIX} ]]; then
           USE_EXPANDED_MATRIX=0
         fi
+        if [[ "${{ github.event_name }}" == "schedule" ]]; then
+          # reuse flag --test_pull_request=sdk to generate report
+          generate_report=(-p test_pull_request sdk)
+        fi
         verbose_flag=
         if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then
           verbose_flag=-v
         fi
         set -e
-        python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} -w integration_tests.yml -p test_packaged_sdk ${{ github.run_id }} -p use_expanded_matrix ${USE_EXPANDED_MATRIX} -s 10 -A ${verbose_flag}
+        python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} -w integration_tests.yml -p test_packaged_sdk ${{ github.run_id }} -p use_expanded_matrix ${USE_EXPANDED_MATRIX} ${generate_report[*]} -s 10 -A ${verbose_flag}

.github/workflows/integration_tests.yml

@@ -58,7 +58,7 @@ jobs:
     ### subsequent steps to update the labels as well.
     runs-on: ubuntu-latest
     if: |
-      github.event_name == 'pull_request' || github.event.inputs.test_pull_request != ''
+      github.event_name == 'pull_request' || (github.event.inputs.test_pull_request != '' && github.event.inputs.test_pull_request != 'sdk')
     outputs:
       should_update_pr: ${{ steps.set_outputs.outputs.should_update_pr }}
       requested_tests: ${{ steps.set_outputs.outputs.requested_tests }}
@@ -166,7 +166,7 @@ jobs:
         if: needs.check_trigger.outputs.requested_tests == 'auto'
         run: |
           echo "Autodetecting which tests to run."
-          if [[ -n "${{github.event.inputs.test_pull_request}}" ]]; then
+          if [[ -n "${{github.event.inputs.test_pull_request}}" || "${{github.event.inputs.test_pull_request}}" == "sdk" ]]; then
             # If running this manually, diff against our common ancestor with main.
             MERGE_BASE=$(git merge-base HEAD origin/main)
             echo "::warning ::Auto-diff HEAD..${MERGE_BASE}"
@@ -203,7 +203,7 @@ jobs:
           echo "::set-output name=android_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k android_device -t ${mobile_test_on} )"
           echo "::set-output name=ios_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k ios_device -t ${mobile_test_on} )"
           echo "::set-output name=tvos_device::$( python scripts/gha/print_matrix_configuration.py -w integration_tests ${EXPANDED_MATRIX_PARAM} -k tvos_device -t ${mobile_test_on} )"
-          if [[ -z "${{github.event.inputs.test_pull_request}}" ]]; then
+          if [[ -z "${{github.event.inputs.test_pull_request}}" || "${{github.event.inputs.test_pull_request}}" == "sdk" ]]; then
             echo "::warning ::Running on https://github.com/${{github.repository}}/commits/$GITHUB_SHA"
             echo "::set-output name=github_ref::$GITHUB_SHA"
           elif [[ "${{github.event.inputs.test_pull_request}}" == *:* ]]; then
@@ -1062,12 +1062,19 @@ jobs:
             --run_id ${{github.run_id}} \
             --new_token ${{steps.generate-token.outputs.token}}
       - name: Update Daily Report
-        if: ${{ github.event_name == 'schedule' }}
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.test_pull_request == 'sdk'}}
+        shell: bash
         run: |
+          if [[ "${{ github.event.inputs.test_pull_request }}" == "sdk" ]]; then
+            additional_flags=(--build_against sdk)
+          else
+            additional_flags=(--build_against repo)
+          fi
           python scripts/gha/it_workflow.py --stage report \
             --token ${{github.token}} \
             --actor ${{github.actor}} \
             --commit ${{needs.prepare_matrix.outputs.github_ref}} \
-            --run_id ${{github.run_id}}
+            --run_id ${{github.run_id}} \
+            ${additional_flags[*]}
       - name: Summarize results into GitHub log
         run: python scripts/gha/summarize_test_results.py --dir test_results --github_log

scripts/gha/github.py

@@ -52,16 +52,25 @@ def requests_retry_session(retries=RETRIES,
     session.mount('https://', adapter)
     return session
 
-def create_issue(token, title, label):
+def create_issue(token, title, label, body):
   """Create an issue: https://docs.github.com/en/rest/reference/issues#create-an-issue"""
   url = f'{FIREBASE_URL}/issues'
   headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
-  data = {'title': title, 'labels': [label]}
+  data = {'title': title, 'labels': [label], 'body': body}
   with requests.post(url, headers=headers, data=json.dumps(data), timeout=TIMEOUT) as response:
     logging.info("create_issue: %s response: %s", url, response)
     return response.json()
 
 
+def get_issue_body(token, issue_number):
+  """https://docs.github.com/en/rest/reference/issues#get-an-issue-comment"""
+  url = f'{FIREBASE_URL}/issues/{issue_number}'
+  headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
+  with requests_retry_session().get(url, headers=headers, timeout=TIMEOUT) as response:
+    logging.info("get_issue_body: %s response: %s", url, response)
+    return response.json()["body"]
+
+
 def update_issue(token, issue_number, data):
   """Update an issue: https://docs.github.com/en/rest/reference/issues#update-an-issue"""
   url = f'{FIREBASE_URL}/issues/{issue_number}'
@@ -192,3 +201,13 @@ def get_reviews(token, pull_number):
       # If exactly per_page results were retrieved, read the next page.
       keep_going = (len(response.json()) == per_page)
   return results
+
+
+def workflow_dispatch(token, workflow_id, ref, inputs):
+  """https://docs.github.com/en/rest/reference/actions#create-a-workflow-dispatch-event"""
+  url = f'{FIREBASE_URL}/actions/workflows/{workflow_id}/dispatches'
+  headers = {'Accept': 'application/vnd.github.v3+json', 'Authorization': f'token {token}'}
+  data = {'ref': ref, 'inputs': inputs}
+  with requests.post(url, headers=headers, data=json.dumps(data),
+                    stream=True, timeout=TIMEOUT) as response:
+    logging.info("workflow_dispatch: %s response: %s", url, response)

scripts/gha/it_workflow.py

@@ -61,11 +61,15 @@
 _COMMENT_TITLE_PROGESS_FAIL = "### ❌  Integration test FAILED (but still ⏳  in progress)\n" 
 _COMMENT_TITLE_FAIL = "### ❌  Integration test FAILED\n"
 _COMMENT_TITLE_SUCCEED = "### ✅  Integration test succeeded!\n"
+_COMMENT_TITLE_FAIL_SDK = "\n***\n### ❌  Integration test FAILED (build against SDK)\n"
+_COMMENT_TITLE_SUCCEED_SDK = "\n***\n### ✅  Integration test succeeded! (build against SDK)\n"
+_COMMENT_TITLE_FAIL_REPO = "### ❌  Integration test FAILED (build against repo)\n"
+_COMMENT_TITLE_SUCCEED_REPO = "### ✅  Integration test succeeded! (build against repo)\n"
 
 _COMMENT_FLAKY_TRACKER = "\n\nAdd flaky tests to **[go/fpl-cpp-flake-tracker](http://go/fpl-cpp-flake-tracker)**\n"
 
 _COMMENT_IDENTIFIER = "integration-test-status-comment"
-_COMMENT_SUFFIX = f'\n\n\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>'
+_COMMENT_SUFFIX = f'\n<hidden value="{_COMMENT_IDENTIFIER}"></hidden>'
 
 _LOG_ARTIFACT_NAME = "log-artifact"
 _LOG_OUTPUT_DIR = "test_results"
@@ -76,6 +80,9 @@
 _BUILD_STAGES_REPORT = "report"
 _BUILD_STAGES = [_BUILD_STAGES_START, _BUILD_STAGES_PROGRESS, _BUILD_STAGES_END, _BUILD_STAGES_REPORT]
 
+_BUILD_AGAINST_SDK = "sdk"
+_BUILD_AGAINST_REPO = "repo"
+
 FLAGS = flags.FLAGS
 
 flags.DEFINE_string(
@@ -105,7 +112,12 @@
     "new_token", None,
     "Only used with --stage end"
     "Use a different token to remove the \"in-progress\" label,"
-    "to allow the removal to trigger the \"Check Labels\" workflow.")                    
+    "to allow the removal to trigger the \"Check Labels\" workflow.")   
+
+flags.DEFINE_string(
+    "build_against", None,
+    "Integration testapps could either build against packaged SDK or repo")
+
 
 def test_start(token, issue_number, actor, commit, run_id):
   """In PR, when start testing, add comment and label \"tests: in-progress\""""
@@ -158,33 +170,33 @@ def test_end(token, issue_number, actor, commit, run_id, new_token):
   github.delete_label(new_token, issue_number, _LABEL_PROGRESS)
 
 
-def test_report(token, actor, commit, run_id):
+def test_report(token, actor, commit, run_id, build_against):
   """Update (create if not exist) a Daily Report in Issue. 
-  If test failed, add label \"tests: failed\" and open the Issue, 
-  If test succeed, add label \"tests: succeeded\" and close the Issue.
   The Issue with title _REPORT_TITLE and label _REPORT_LABEL:
-  https://github.com/firebase/firebase-cpp-sdk/issues?q=is%3Aissue+is%3Aclosed+label%3Anightly-testing
+  https://github.com/firebase/firebase-cpp-sdk/issues?q=is%3Aissue+label%3Anightly-testing
   """
   issue_number = _get_issue_number(token, _REPORT_TITLE, _REPORT_LABEL)
+  previous_comment = github.get_issue_body(token, issue_number)
+  [previous_comment_repo, previous_comment_sdk] = previous_comment.split(_COMMENT_SUFFIX)
   log_summary = _get_summary_talbe(token, run_id)
   if log_summary == 0:
-#     github.delete_label(token, issue_number, _LABEL_FAILED)
-#     github.add_label(token, issue_number, _LABEL_SUCCEED)
+    title = _COMMENT_TITLE_SUCCEED_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_SUCCEED_SDK
+    comment = title + _get_description(actor, commit, run_id)
+  else:
+    title = _COMMENT_TITLE_FAIL_REPO if build_against==_BUILD_AGAINST_REPO else _COMMENT_TITLE_FAIL_SDK
+    comment = title + _get_description(actor, commit, run_id) + log_summary + _COMMENT_FLAKY_TRACKER
+  
+  if build_against==_BUILD_AGAINST_REPO:
+    comment = comment + _COMMENT_SUFFIX + previous_comment_sdk
+  else:
+    comment = previous_comment_repo + _COMMENT_SUFFIX + comment
+
+  if (_COMMENT_TITLE_SUCCEED_REPO in comment) and (_COMMENT_TITLE_SUCCEED_SDK in comment):
     github.close_issue(token, issue_number)
-    comment = (_COMMENT_TITLE_SUCCEED +
-               _get_description(actor, commit, run_id) +
-               _COMMENT_SUFFIX)
-    github.update_issue_comment(token, issue_number, comment)
   else:
-#     github.delete_label(token, issue_number, _LABEL_SUCCEED)
-#     github.add_label(token, issue_number, _LABEL_FAILED)
     github.open_issue(token, issue_number)
-    comment = (_COMMENT_TITLE_FAIL +
-               _get_description(actor, commit, run_id) +
-               log_summary +
-               _COMMENT_FLAKY_TRACKER +
-               _COMMENT_SUFFIX)
-    github.update_issue_comment(token, issue_number, comment)
+    
+  github.update_issue_comment(token, issue_number, comment)
 
 
 def _get_issue_number(token, title, label):
@@ -193,7 +205,7 @@ def _get_issue_number(token, title, label):
     if issue["title"] == title:
       return issue["number"]
 
-  return github.create_issue(token, title, label)["number"]
+  return github.create_issue(token, title, label, _COMMENT_SUFFIX)["number"]
 
 
 def _update_comment(token, issue_number, comment):
@@ -255,7 +267,7 @@ def main(argv):
   elif FLAGS.stage == _BUILD_STAGES_END:
     test_end(FLAGS.token, FLAGS.issue_number, FLAGS.actor, FLAGS.commit, FLAGS.run_id, FLAGS.new_token)
   elif FLAGS.stage == _BUILD_STAGES_REPORT:
-    test_report(FLAGS.token, FLAGS.actor, FLAGS.commit, FLAGS.run_id)
+    test_report(FLAGS.token, FLAGS.actor, FLAGS.commit, FLAGS.run_id, FLAGS.build_against)
   else:
     print("Invalid stage value. Valid value: " + ",".join(_BUILD_STAGES))