Move ubuntu2004-aarch64 jobs into production

- move ubuntu2004-aarch64 merge/verify jobs from
  beta into production on master as required for
  https://gerrit.fd.io/r/c/vpp/+/30734

- remove beta job definitions

- refactor os job add/remove from separate
  project definitions to exclude list in
  vpp-<arch> project definitions

- handle errors when parsing yaml files in
  dbld_csit_find_ansible_packages.py

Change-Id: Ife651a6b85c6c3872255f15bd87ede499ee78b22
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

diff --git a/docker/scripts/dbld_csit_find_ansible_packages.py b/docker/scripts/dbld_csit_find_ansible_packages.py
index 2e6c6cf..7f34fb9 100755 (executable)
--- a/docker/scripts/dbld_csit_find_ansible_packages.py
+++ b/docker/scripts/dbld_csit_find_ansible_packages.py
@@ -1,6 +1,6 @@
 #! /usr/bin/env python3
 
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -39,8 +39,13 @@ class CsitAnsibleYamlStruct:
 def packages_in_csit_ansible_yaml_file(yamlfile: str, distro, arch) -> list:
     with open(yamlfile) as yf:
         csit_ansible_yaml = yaml.safe_load(yf)
+        if csit_ansible_yaml is None:
+            return ""
         cays = CsitAnsibleYamlStruct(**csit_ansible_yaml)
-        packages = [pkg for pkg in cays.packages_base if type(pkg) is str]
+        try:
+            packages = [pkg for pkg in cays.packages_base if type(pkg) is str]
+        except AttributeError:
+            return ""
         if arch in [*cays.packages_by_arch]:
             packages += [pkg for pkg in cays.packages_by_arch[arch]
                          if type(pkg) is str]

diff --git a/jjb/vpp/vpp.yaml b/jjb/vpp/vpp.yaml
index c321f04..b5ce7c3 100644 (file)
--- a/jjb/vpp/vpp.yaml
+++ b/jjb/vpp/vpp.yaml
@@ -54,6 +54,14 @@
 # [end] VPP-CHECKSTYLE-X86_64 PROJECT
 
 # VPP-X86_64 PROJECT
+#
+# NOTE: The list of supported branches for each OS are maintained
+#       in .../ci-management/docker/scripts/lib_vpp.sh
+#       The job configuration must be derived from that list, thus
+#       modifications to both should be made at the same time.
+#       Since x86_64 is the primary development architecture, this
+#       project definition should be identical to the docker image
+#       build scripts.  Other architectures may be a subset.
 - project:
     name: vpp-x86_64
     jobs:
@@ -75,80 +83,33 @@
           branch: 'stable/2101'
           branch-refspec: ''
           repo-stream-part: 'stable.2101'
-    os:
-      - ubuntu1804:
-          repo-os-part: 'ubuntu.bionic.main'
-    executor-arch: 'x86_64'
-# [end] VPP-X86_64 PROJECT
-
-# VPP-OS-EOL-X86_64 PROJECT
-- project:
-    name: vpp-os-eol-x86_64
-    jobs:
-      - 'vpp-verify-{stream}-{os}-{executor-arch}'
-      - 'vpp-merge-{stream}-{os}-{executor-arch}'
-    project: 'vpp'
-    make-parallel-jobs: '4'
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/*'
-    stream:
-      - '2009':
-          branch: 'stable/2009'
-          branch-refspec: ''
-          repo-stream-part: 'stable.2009'
     os:
       - centos7:
           repo-os-part: 'centos7'
-      - debian9:
-          repo-os-part: 'stretch'
-    executor-arch: 'x86_64'
-# [end] VPP-OS-EOL-X86_64 PROJECT
-
-# VPP-OS-IN-2009-X86_64 PROJECT
-#
-# This project is used to separate streams which are only
-# supported on the OS in the release in which it was added
-# or later releases.
-#
-# For each 'os-in-XXXX', the stream for the release is
-# added here in addition to the 'vpp-x86_64' project.
-# Each subsequent release is also added to both.
-#
-# When all of the streams older than the 'os-in-XXXX" have
-# been removed from the 'vpp-x86_64' project, then the OSes
-# in this project get moved to the 'vpp-x86_64' project and
-# this project gets deleted or commented out if it is only
-# 'os-in-XXXX' project.
-#
-- project:
-    name: vpp-os-in-2009-x86_64
-    jobs:
-      - 'vpp-verify-{stream}-{os}-{executor-arch}'
-      - 'vpp-merge-{stream}-{os}-{executor-arch}'
-    project: 'vpp'
-    make-parallel-jobs: '4'
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/*'
-    stream:
-      - master:
-          branch: 'master'
-          branch-refspec: ''
-          repo-stream-part: 'master'
-      - '2009':
-          branch: 'stable/2009'
-          branch-refspec: ''
-          repo-stream-part: 'stable.2009'
-      - '2101':
-          branch: 'stable/2101'
-          branch-refspec: ''
-          repo-stream-part: 'stable.2101'
-    os:
       - centos8:
           repo-os-part: 'centos8'
+      - debian9:
+          repo-os-part: 'stretch'
       - debian10:
           repo-os-part: 'buster'
+      - ubuntu1804:
+          repo-os-part: 'ubuntu.bionic.main'
       - ubuntu2004:
           repo-os-part: 'ubuntu.focal.main'
     executor-arch: 'x86_64'
-# [end] VPP-OS-IN-2009-X86_64 PROJECT
+    exclude:
+      # OS deprecated in master
+      - stream: 'master'
+        os: 'centos7'
+      - stream: 'master'
+        os: 'debian9'
+      # 2009 LTS supports all OS variants
+      # OS deprecated in 2101
+      - stream: '2101'
+        os: 'centos7'
+      - stream: '2101'
+        os: 'debian9'
+# [end] VPP-X86_64 PROJECT
 
 # VPP-DEBUG-X86_64 PROJECT
 #
@@ -183,39 +144,6 @@
     executor-arch: 'x86_64'
 # [end] VPP-DEBUG-X86_64 PROJECT
 
-# VPP-BETA-X86_64 PROJECT
-#
-# Beta jobs are experimental jobs for testing new OS versions on master.
-#
-# They are only manually triggered.  Either using the Jenkins WebUI
-# 'Build with Parameters' to run a single job or adding 'beta-verify'
-# or 'beta-merge' as a comment to run all beta verify or merge jobs.
-#
-# Once testing  is complete, the os is moved to the current
-# 'vpp-os-in-XXXX-x86_64' project. If no OSes are being
-# tested then this project should be commented out.
-#
-#- project:
-#    name: vpp-beta-x86_64
-#    jobs:
-#      - 'vpp-beta-verify-{stream}-{os}-{executor-arch}':
-#          comment-trigger-value: 'beta-verify'
-#      - 'vpp-beta-merge-{stream}-{os}-{executor-arch}':
-#          comment-trigger-value: 'beta-merge'
-#    project: 'vpp'
-#    make-parallel-jobs: '4'
-#    archive-artifacts: '/tmp/vpp-failed-unittests/*/*'
-#    stream:
-#      - master:
-#          branch: 'master'
-#          branch-refspec: ''
-#          repo-stream-part: 'master'
-#    os:
-#      - debian10:
-#          repo-os-part: 'buster'
-#    executor-arch: x86_64
-# [end] VPP-BETA-X86_64 PROJECT
-
 # VPP-GCC-X86_64 PROJECT
 - project:
     name: vpp-gcc-x86_64
@@ -242,6 +170,14 @@
 # [end] VPP-GCC-X86_64 PROJECT
 
 # VPP-AARCH64 PROJECT
+#
+# NOTE: The list of supported branches for each OS are maintained
+#       in .../ci-management/docker/scripts/lib_vpp.sh
+#       The job configuration must be derived from that list, thus
+#       modifications to both should be made at the same time.
+#       The AARCH64 architecture is in the process of catching up
+#       with X86_64 in terms of OS coverage.  Thus the jobs are
+#       a subset of those defined in the docker image build script.
 - project:
     name: vpp-aarch64
     jobs:
@@ -268,89 +204,22 @@
           repo-os-part: 'centos8'
       - ubuntu1804:
           repo-os-part: 'ubuntu-arm.bionic.main'
+      - ubuntu2004:
+          repo-os-part: 'ubuntu.focal.main'
     executor-arch: 'aarch64'
     exclude:
       # OS introduced after 2009
       - stream: '2009'
         os: 'centos8'
+      - stream: '2009'
+        os: 'ubuntu2004'
       # OS introduced after 2101
       - stream: '2101'
         os: 'centos8'
+      - stream: '2101'
+        os: 'ubuntu2004'
 # [end] VPP-AARCH64 PROJECT
 
-# VPP-OS-IN-2009-AARCH64 PROJECT
-#
-# This project is used to separate streams which are only
-# supported on the OS in the release in which it was added
-# or later releases.
-#
-# For each 'os-in-XXXX', the stream for the release is
-# added here in addition to the 'vpp-aarch64' project.
-# Each subsequent release is also added to both.
-#
-# When all of the streams older than the 'os-in-XXXX" have
-# been removed from the 'vpp-aarch64' project, then the OSes
-# in this project get moved to the 'vpp-aarch64' project and
-# this project gets deleted or commented out if it is only
-# 'os-in-XXXX' project.
-#
-#- project:
-#    name: vpp-os-in-2009-aarch64
-#    jobs:
-#      - 'vpp-verify-{stream}-{os}-{executor-arch}'
-#      - 'vpp-merge-{stream}-{os}-{executor-arch}'
-#
-#    project: 'vpp'
-#    make-parallel-jobs: '16'
-#    archive-artifacts: '/tmp/vpp-failed-unittests/*/*'
-#    stream:
-#      - master:
-#          branch: 'master'
-#          branch-refspec: ''
-#          repo-stream-part: 'master'
-#      - '2009':
-#          branch: 'stable/2009'
-#          branch-refspec: ''
-#          repo-stream-part: 'stable.2009'
-#    os:
-#      - ubuntu2004:
-#          repo-os-part: 'ubuntu.focal.main'
-#    executor-arch: 'aarch64'
-# [end] VPP-OS-IN-2009-AARCH64 PROJECT
-
-# VPP-BETA-AARCH64 PROJECT
-#
-# Beta jobs are experimental jobs for testing new OS versions on master.
-#
-# They are only manually triggered.  Either using the Jenkins WebUI
-# 'Build with Parameters' to run a single job or adding 'beta-verify'
-# or 'beta-merge' as a comment to run all beta verify or merge jobs.
-#
-# Once testing  is complete, the os is moved to the current
-# 'vpp-os-in-XXXX-aarch64' project. If no OSes are being
-# tested then this project should be commented out.
-#
-- project:
-    name: vpp-beta-aarch64
-    jobs:
-      - 'vpp-beta-verify-{stream}-{os}-{executor-arch}':
-          comment-trigger-value: 'beta-verify'
-      - 'vpp-beta-merge-{stream}-{os}-{executor-arch}':
-          comment-trigger-value: 'beta-merge'
-    project: 'vpp'
-    make-parallel-jobs: '16'
-    archive-artifacts: '/tmp/vpp-failed-unittests/*/*'
-    stream:
-      - master:
-          branch: 'master'
-          branch-refspec: ''
-          repo-stream-part: 'master'
-    os:
-      - ubuntu2004:
-          repo-os-part: 'ubuntu.focal.main'
-    executor-arch: aarch64
-# [end] VPP-BETA-AARCH64 PROJECT
-
 # VPP-CSIT-VERIFY PROJECT
 - project:
     name: vpp-csit-verify