Test commands for release repo cleanup.

Signed-off-by: Mauro Sardara <msardara@cisco.com>
Change-Id: I71bee2717bbcc3021fd85020f2059839ea5a930c
Signed-off-by: Mauro Sardara <msardara@cisco.com>

diff --git a/jjb/ci-management/package-list.sh b/jjb/ci-management/package-list.sh
new file mode 100644 (file)
index 0000000..6172b7b
--- /dev/null
+++ b/jjb/ci-management/package-list.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+export PACKAGE_LIST_COMMON="libhicnctrl \
+libhicnctrl-memif \
+hicn-collectd-plugins \
+hicn-apps \
+hicn-light \
+hicn-apps-memif \
+libhicntransport-memif \
+libhicn \
+libhicntransport \
+hicn-plugin \
+facemgr \
+hicn-utils-memif \
+hicn-utils \
+hicn-sysrepo-plugin \
+hicn-extra-plugin \
+libparc \
+libparc-doc \
+longbow \
+longbow-doc"
+
+export PACKAGE_LIST_UBUNTU="libhicnctrl-dev \
+libhicn-ctrl-dev \
+libhicnctrl-memif-dev \
+libhicntransport-memif-dev \
+libhicn-dev \
+libhicntransport-dev \
+hicn-plugin-dev \
+libdash \
+libdash-dev \
+libdash-doc \
+libparc-dev \
+libmemif-dev \
+longbow-dev"
+
+export PACKAGE_LIST_CENTOS="libhicnctrl-devel \
+libhicnctrl-memif-devel \
+libhicntransport-memif-devel \
+libhicn-devel \
+libhicntransport-devel \
+hicn-plugin-devel \
+libparc-devel \
+libmemif-devel \
+longbow-devel"
+
+export VERSION_WHITELIST="19.01-227 \
+19.08-289 \
+20.01-73 \
+20.01-114"

diff --git a/jjb/ci-management/packagecloud-repo-cleanup.sh b/jjb/ci-management/packagecloud-repo-cleanup.sh
new file mode 100644 (file)
index 0000000..8fdfe11
--- /dev/null
+++ b/jjb/ci-management/packagecloud-repo-cleanup.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+set -euo pipefail
+
+# Number of packages to keep.
+N_PACKAGES=5
+
+PACKAGECLOUD_REPO_DEB="https://packagecloud.io/install/repositories/fdio/${STREAM}/script.deb.sh"
+PACKAGECLOUD_REPO_RPM="https://packagecloud.io/install/repositories/fdio/${STREAM}/script.rpm.sh"
+
+FACTER_OS=$(/usr/bin/facter operatingsystem)
+PACKAGE_LIST=""
+VERSION_REGEX="[0-9]+.[0-9]+[-_][0-9]+[-_]release(-1)?|[0-9]+.[0-9]+[-_][0-9]+~g[[:graph:]]+"
+
+declare -A FUNCTIONS
+
+echo_err () {
+    >&2 echo ${@}
+    exit 1
+}
+
+contains() {
+    [[ ${1} =~ (^|[[:space:]])${2}($|[[:space:]]) ]] && return 1 || return 0
+}
+
+check_version_whitelist() {
+    if [[ ${1} =~ ([0-9]+).([0-9]+)[-_]([0-9]+).+ ]]; then
+        MAJOR=${BASH_REMATCH[1]}
+        MINOR=${BASH_REMATCH[2]}
+        REVISION=${BASH_REMATCH[3]}
+        VER="${MAJOR}.${MINOR}-${REVISION}"
+
+        if contains "${VERSION_WHITELIST}" ${VER}; then
+            return 1;
+        fi
+    fi
+
+    return 0
+}
+
+# Params
+# $1: Package list
+build_package_blacklist_ubuntu () {
+    PACKAGE_LIST=${@}
+    OUTPUT_LIST=""
+    ARCH=$(dpkg --print-architecture)
+
+    for package in ${PACKAGE_LIST}; do
+        OUTPUT=$(apt-cache policy ${package} 2> /dev/null)
+
+        if [[ ${?} -ne 0 || -z "${OUTPUT}" ]]; then
+            continue
+        fi
+
+        # N_PACKAGES + 2 is justified by the fact thathe output of apt-cache policy is in the form:
+        # apt-cache policy libhicn
+        # libhicn:
+        #   Installed: (none)
+        #   Candidate: 20.05-11-release
+        #   Version table:
+        #      20.05-11-release 500
+        #
+        # The first version candidate is printed twice.
+        # So we need to exclude the "Candidate: 20.05-11-release" (+1).
+        # Also `tail -n +M` starts to print from the Mth line, being 1 the first line.
+        # So to exclude the Mth line an additional +1 is needed.
+        VERSIONS="$(echo ${OUTPUT} | grep -E -o ${VERSION_REGEX} | tail -n +$((N_PACKAGES + 2)))"
+
+        for version in ${VERSIONS}; do
+            if ! check_version_whitelist ${version}; then
+                OUTPUT_LIST+="${package}_${version}_${ARCH}.deb "
+            fi
+        done
+    done
+
+    echo ${OUTPUT_LIST}
+}
+
+# Params
+# $1: Package list
+build_package_blacklist_centos () {
+    PACKAGE_LIST=${@}
+    OUTPUT_LIST=""
+    VERSIONS=""
+    ARCH=$(uname -m)
+
+    for package in ${PACKAGE_LIST}; do
+        OUTPUT=$(yum --showduplicates list ${package} 2> /dev/null)
+        if [[ ${?} -ne 0 || -z "${OUTPUT}" ]]; then
+            continue
+        fi
+
+        VERSIONS="$(echo ${OUTPUT} | grep -Eo "${VERSION_REGEX}" | head -n -${N_PACKAGES})"
+
+        for version in ${VERSIONS}; do
+            if ! check_version_whitelist ${version}; then
+                OUTPUT_LIST+="${package}-${version}.${ARCH}.rpm "
+            fi
+        done
+    done
+
+    echo ${OUTPUT_LIST}
+}
+
+promote_attic_repo_centos () {
+    FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease)
+    FACTER_ARCH=$(/usr/bin/facter architecture)
+
+    for package in ${@}; do
+        echo package_cloud promote \
+            ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ \
+            ${package} ${PCIO_CO}/attic/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/
+        package_cloud promote \
+            ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ \
+            ${package} ${PCIO_CO}/attic/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/
+    done
+}
+
+promote_attic_repo_ubuntu () {
+    FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename)
+
+    for package in ${@}; do
+        echo package_cloud promote ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ \
+            ${package} ${PCIO_CO}/attic/ubuntu/${FACTER_LSBNAME}/main/
+        package_cloud promote ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ \
+            ${package} ${PCIO_CO}/attic/ubuntu/${FACTER_LSBNAME}/main/
+    done
+}
+
+promote_to_attic_repo () {
+    ${FUNCTIONS["promote_attic_repo"]} ${@}
+}
+
+setup_fdio_repo () {
+    case "${FACTER_OS}" in
+      Ubuntu)
+        curl -s ${PACKAGECLOUD_REPO_DEB} | sudo bash
+        FUNCTIONS["package_blacklist"]="build_package_blacklist_ubuntu"
+        FUNCTIONS["promote_attic_repo"]="promote_attic_repo_ubuntu"
+        PACKAGE_LIST="${PACKAGE_LIST_COMMON} ${PACKAGE_LIST_UBUNTU}"
+      ;;
+      CentOS)
+        curl -s ${PACKAGECLOUD_REPO_RPM} | sudo bash
+        FUNCTIONS["package_blacklist"]="build_package_blacklist_centos"
+        FUNCTIONS["promote_attic_repo"]="promote_attic_repo_centos"
+        PACKAGE_LIST="${PACKAGE_LIST_COMMON} ${PACKAGE_LIST_CENTOS}"
+      ;;
+      *)
+        echo_err "Distribution ${FACTER_OS} is not supported."
+      ;;
+    esac
+}
+
+# Params
+# $1: Package list
+build_package_blacklist () {
+    ${FUNCTIONS["package_blacklist"]} ${@}
+}
+
+setup_fdio_repo
+PACKAGES_TO_PROMOTE=$(build_package_blacklist ${PACKAGE_LIST})
+promote_to_attic_repo ${PACKAGES_TO_PROMOTE}
\ No newline at end of file

diff --git a/jjb/ci-management/packagecloud-repo-cleanup.yaml b/jjb/ci-management/packagecloud-repo-cleanup.yaml
new file mode 100644 (file)
index 0000000..88966c5
--- /dev/null
+++ b/jjb/ci-management/packagecloud-repo-cleanup.yaml
@@ -0,0 +1,140 @@
+- project:
+    name: packagecloud-repo-cleanup
+    jobs:
+      - 'packagecloud-repo-cleanup-release-{os}'
+      - 'packagecloud-repo-cleanup-hicn-{os}'
+    project: 'ci-management'
+    os:
+      - ubuntu1804:
+          repo-os-part: 'ubuntu.bionic.main'
+      - ubuntu1804arm:
+          repo-os-part: 'ubuntu-arm.bionic.main'
+      - ubuntu1604:
+          repo-os-part: 'ubuntu.xenial.main'
+      - centos7:
+          repo-os-part: 'centos7'
+
+- job-template:
+    name: 'packagecloud-repo-cleanup-release-{os}'
+
+    project-type: freestyle
+    node: '{os}-us'
+    concurrent: true
+
+    build-discarder:
+      daysToKeep: '{build-days-to-keep}'
+      numToKeep: 100
+      artifactDaysToKeep: '{build-artifact-days-to-keep}'
+      artifactNumToKeep: '{build-artifact-num-to-keep}'
+
+    parameters:
+      - project-parameter:
+          project: '{project}'
+      - gerrit-parameter:
+          branch: '{branch}'
+      - os-parameter:
+          os: '{os}'
+      - stream-parameter:
+          stream: 'release'
+      - string:
+          name: ARCHIVE_ARTIFACTS
+          default: '{archive-artifacts}'
+          description: Artifacts to archive to the logs server.
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: 120
+
+    triggers:
+      - gerrit:
+          server-name: 'Primary'
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: 'release-cleanup'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: '.packagecloud'
+              target: '/root/.packagecloud'
+      - config-file-provider:
+          files:
+            - file-id: 'packagecloud_api'
+              target: '/root/packagecloud_api'
+      - shell:
+          !include-raw-escape:
+          - package-list.sh
+          - packagecloud-repo-cleanup.sh
+
+    publishers:
+      - fdio-infra-shiplogs:
+          maven-version: 'mvn36'
+
+- job-template:
+    name: 'packagecloud-repo-cleanup-hicn-{os}'
+
+    project-type: freestyle
+    node: '{os}-us'
+    concurrent: true
+
+    build-discarder:
+      daysToKeep: '{build-days-to-keep}'
+      numToKeep: 100
+      artifactDaysToKeep: '{build-artifact-days-to-keep}'
+      artifactNumToKeep: '{build-artifact-num-to-keep}'
+
+    parameters:
+      - project-parameter:
+          project: '{project}'
+      - gerrit-parameter:
+          branch: '{branch}'
+      - os-parameter:
+          os: '{os}'
+      - stream-parameter:
+          stream: 'hicn'
+      - string:
+          name: ARCHIVE_ARTIFACTS
+          default: '{archive-artifacts}'
+          description: Artifacts to archive to the logs server.
+
+    wrappers:
+      - fdio-infra-wrappers:
+          build-timeout: 120
+
+    triggers:
+      - timed: '@weekly'
+      - gerrit:
+          server-name: 'Primary'
+          trigger-on:
+            - comment-added-contains-event:
+                comment-contains-value: 'release-cleanup'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: '.packagecloud'
+              target: '/root/.packagecloud'
+      - config-file-provider:
+          files:
+            - file-id: 'packagecloud_api'
+              target: '/root/packagecloud_api'
+      - shell:
+          !include-raw-escape:
+          - package-list.sh
+          - packagecloud-repo-cleanup.sh
+
+    publishers:
+      - fdio-infra-shiplogs:
+          maven-version: 'mvn36'