project: '{project}'
- gerrit-parameter:
branch: '{branch}'
- - maven-exec:
- maven-version: 'mvn33-new'
scm:
- gerrit-trigger-scm:
notbuilt: true
builders:
- - maven-target:
- maven-version: 'mvn33-new'
- goals: '--version'
- settings: 'csit-settings'
- settings-type: cfp
- global-settings: 'global-settings'
- global-settings-type: cfp
- - provide-maven-settings:
- settings-file: 'csit-settings'
- global-settings-file: 'global-settings'
- shell:
!include-raw-escape:
- ../scripts/setup_executor_env.sh
- ../scripts/csit/docs.sh
publishers:
+ - fdio-infra-publish-docs
- fdio-infra-publish
- job-template:
project-type: freestyle
node: 'builder-{os}-prod-{executor-arch}'
concurrent: false
- allow-empty: 'true'
- fingerprint: false
- only-if-success: true
- default-excludes: false
+ latest-only: false
build-discarder:
daysToKeep: '{build-days-to-keep}'
project: '{project}'
- gerrit-parameter:
branch: '{branch}'
- - maven-exec:
- maven-version: 'mvn33-new'
+ - gerrit-refspec-parameter
scm:
- gerrit-trigger-scm:
credentials-id: 'jenkins-gerrit-credentials'
refspec: '$GERRIT_REFSPEC'
- choosing-strategy: 'default'
+ choosing-strategy: 'gerrit'
wrappers:
- fdio-infra-wrappers-non-activity-timeout:
notbuilt: true
builders:
- - maven-target:
- maven-version: 'mvn33-new'
- goals: '--version'
- settings: 'csit-settings'
- settings-type: cfp
- global-settings: 'global-settings'
- global-settings-type: cfp
- - provide-maven-settings:
- settings-file: 'csit-settings'
- global-settings-file: 'global-settings'
- shell:
!include-raw-escape:
- ../scripts/setup_executor_env.sh
- ../scripts/csit/report.sh
publishers:
+ - fdio-infra-publish-docs
- fdio-infra-publish
- job-template:
project: '{project}'
- gerrit-parameter:
branch: '{branch}'
- - maven-exec:
- maven-version: 'mvn33-new'
scm:
- gerrit-trigger-scm:
- timed: '0 2,14 * * *'
builders:
- - maven-target:
- maven-version: 'mvn33-new'
- goals: '--version'
- settings: 'csit-settings'
- settings-type: cfp
- global-settings: 'global-settings'
- global-settings-type: cfp
- - provide-maven-settings:
- settings-file: 'csit-settings'
- global-settings-file: 'global-settings'
- shell:
!include-raw-escape:
- ../scripts/setup_executor_env.sh
- shell:
!include-raw-escape:
- - ../scripts/csit/cpta.sh
+ - ../scripts/csit/trending.sh
publishers:
- email-ext:
always: true
failure: false
+ - fdio-infra-publish-docs
- fdio-infra-publish
- job-template:
project: '{project}'
- gerrit-parameter:
branch: 'rls2009_lts'
- - maven-exec:
- maven-version: 'mvn33-new'
scm:
- gerrit-trigger-scm:
- timed: '0 0 * * 1'
builders:
- - maven-target:
- maven-version: 'mvn33-new'
- goals: '--version'
- settings: 'csit-settings'
- settings-type: cfp
- global-settings: 'global-settings'
- global-settings-type: cfp
- - provide-maven-settings:
- settings-file: 'csit-settings'
- global-settings-file: 'global-settings'
- shell:
!include-raw-escape:
- ../scripts/setup_executor_env.sh
- shell:
!include-raw-escape:
- - ../scripts/csit/cpta.sh
+ - ../scripts/csit/trending.sh
publishers:
- email-ext:
always: true
failure: false
+ - fdio-infra-publish-docs
- fdio-infra-publish
+
- project:
name: csit-info
project-name: csit
- developers
- recipients
+- publisher:
+ name: fdio-infra-publish-docs
+ # macro to finish up a build.
+ #
+ # Handles the following:
+ # - Shipping docs S3 logs repository
+ # - Cleanup workspace
+ publishers:
+ - postbuildscript:
+ builders:
+ - role: BOTH
+ build-on:
+ - ABORTED
+ - FAILURE
+ - NOT_BUILT
+ - SUCCESS
+ - UNSTABLE
+ build-steps:
+ - fdio-infra-ship-docs
+ mark-unstable-if-failed: true
+ - workspace-cleanup:
+ exclude:
+ # Do not clean up *.jenkins-trigger files for jobs that use a
+ # properties file as input for triggering another build.
+ - "**/*.jenkins-trigger"
+ fail-build: false
+
- publisher:
name: fdio-infra-publish
# macro to finish up a build.
days-to-keep: '{build-days-to-keep}'
num-to-keep: '{build-num-to-keep}'
+- builder:
+ name: fdio-infra-ship-docs
+ builders:
+ - config-file-provider:
+ files:
+ - file-id: "jenkins-log-archives-settings"
+ variable: "SETTINGS_FILE"
+ - config-file-provider:
+ files:
+ - file-id: "jenkins-s3-docs-ship"
+ target: $HOME/.aws/credentials
+ - shell: !include-raw:
+ - scripts/publish_library_py.sh
+ - shell: !include-raw:
+ - scripts/publish_docs.sh
+ - shell: !include-raw:
+ - ../global-jjb/shell/logs-clear-credentials.sh
+
- builder:
name: fdio-infra-ship-backup-logs
builders:
- file-id: "jenkins-s3-log-ship"
variable: $HOME/.aws/credentials
- shell: !include-raw:
- - scripts/logs_publish.sh
+ - scripts/publish_library_py.sh
+ - shell: !include-raw:
+ - scripts/publish_logs.sh
- shell: !include-raw:
- ../global-jjb/shell/logs-clear-credentials.sh
#!/bin/bash
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
echo "---> jjb/scripts/csit/docs.sh"
-set -xe -o pipefail
+set -exuo pipefail
-[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site"
-[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit
-[ "$DOC_DIR" ] || DOC_DIR=resources/tools/doc_gen
-[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build
-[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site
-[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/doc
-[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn"
-
-cd ${DOC_DIR}
-chmod +x ./run_doc.sh
-./run_doc.sh ${GERRIT_BRANCH}
-
-retval=$?
-if [ ${retval} -ne "0" ]; then
- echo "Documentation generation failed!"
-exit ${retval}
-fi
-
-if [[ ${JOB_NAME} == *merge* ]]; then
-
- cd ${WORKSPACE}
-
- mkdir -p ${RESOURCES_DIR}
- mv -f ${BUILD_DIR}/* ${RESOURCES_DIR}
- cd ${SITE_DIR}
-
- cat > pom.xml << EOF
- <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>io.fd.csit</groupId>
- <artifactId>docs</artifactId>
- <version>1.0.0</version>
- <packaging>pom</packaging>
- <properties>
- <generateReports>false</generateReports>
- </properties>
- <build>
- <extensions>
- <extension>
- <groupId>org.apache.maven.wagon</groupId>
- <artifactId>wagon-webdav-jackrabbit</artifactId>
- <version>2.10</version>
- </extension>
- </extensions>
- </build>
- <distributionManagement>
- <site>
- <id>fdio-site</id>
- <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url>
- </site>
- </distributionManagement>
- </project>
-EOF
-
- ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C
-
- cd -
-
-fi
+csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry"
+source "${csit_entry_dir}/bootstrap_docs.sh"
#!/bin/bash
-# Copyright (c) 2020 Cisco and/or its affiliates.
+# Copyright (c) 2021 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
echo "---> jjb/scripts/csit/report.sh"
-set -xe -o pipefail
+set -exuo pipefail
-[ "$DOCS_REPO_URL" ] || DOCS_REPO_URL="https://nexus.fd.io/content/sites/site"
-[ "$PROJECT_PATH" ] || PROJECT_PATH=io/fd/csit
-[ "$DOC_DIR" ] || DOC_DIR=resources/tools/presentation
-[ "$BUILD_DIR" ] || BUILD_DIR=${DOC_DIR}/_build
-[ "$SITE_DIR" ] || SITE_DIR=build-root/docs/deploy-site
-[ "$RESOURCES_DIR" ] || RESOURCES_DIR=${SITE_DIR}/src/site/resources/report
-[ "$MVN" ] || MVN="/opt/apache/maven/bin/mvn"
-
-cd ${DOC_DIR}
-chmod +x ./run_report.sh
-./run_report.sh ${GERRIT_BRANCH}
-
-retval=$?
-if [ ${retval} -ne "0" ]; then
- echo "Report generation failed!"
-exit ${retval}
-fi
-
-if [[ ${JOB_NAME} == *merge* ]]; then
-
- cd ${WORKSPACE}
-
- mkdir -p ${RESOURCES_DIR}
- mv -f ${BUILD_DIR}/* ${RESOURCES_DIR}
- cd ${SITE_DIR}
-
- cat > pom.xml << EOF
- <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>io.fd.csit</groupId>
- <artifactId>docs</artifactId>
- <version>1.0.0</version>
- <packaging>pom</packaging>
- <properties>
- <generateReports>false</generateReports>
- </properties>
- <build>
- <extensions>
- <extension>
- <groupId>org.apache.maven.wagon</groupId>
- <artifactId>wagon-webdav-jackrabbit</artifactId>
- <version>2.10</version>
- </extension>
- </extensions>
- </build>
- <distributionManagement>
- <site>
- <id>fdio-site</id>
- <url>dav:${DOCS_REPO_URL}/${PROJECT_PATH}/${GERRIT_BRANCH}</url>
- </site>
- </distributionManagement>
- </project>
-EOF
-
- ${MVN} site:site site:deploy -gs "${GLOBAL_SETTINGS_FILE}" -s "${SETTINGS_FILE}" -T 4C
-
- cd -
-
-fi
+csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry"
+source "${csit_entry_dir}/bootstrap_report.sh"
--- /dev/null
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> jjb/scripts/csit/trending.sh"
+
+set -exuo pipefail
+
+csit_entry_dir="${WORKSPACE}/resources/libraries/bash/entry"
+source "${csit_entry_dir}/bootstrap_trending.sh"
--- /dev/null
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> publish_docs.sh"
+
+set -exuo pipefail
+
+S3_BUCKET="fdio-docs-s3-cloudfront-index"
+CDN_URL="s3-docs.fd.io"
+PYTHON_SCRIPT="/w/workspace/publish_library.py"
+
+if [[ ${JOB_NAME} == *merge* ]]; then
+ case "${JOB_NAME}" in
+ *"csit-trending"*)
+ SITE_DIR="${WORKSPACE}/resources/tools/trending/_build"
+ s3_path="csit/${GERRIT_BRANCH}/trending"
+ ;;
+ *"csit-report"*)
+ SITE_DIR="${WORKSPACE}/resources/tools/presentation/_build"
+ s3_path="csit/${GERRIT_BRANCH}/report"
+ ;;
+ *"csit-docs"*)
+ SITE_DIR="${WORKSPACE}/resources/tools/doc_gen/_build"
+ s3_path="csit/${GERRIT_BRANCH}/docs"
+ ;;
+ *)
+ die "Unknown job: ${JOB_NAME}"
+ esac
+
+ echo "INFO: S3 path $s3_path"
+
+ echo "INFO: archiving docs to S3"
+ python3 $PYTHON_SCRIPT deploy_docs "$S3_BUCKET" "$s3_path" "$SITE_DIR"
+
+ echo "S3 docs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>"
+fi
# See the License for the specific language governing permissions and
# limitations under the License.
-echo "---> logs_publish.sh"
+echo "---> publish_library_py.sh"
-CDN_URL="logs.nginx.service.consul"
-export AWS_ENDPOINT_URL="http://storage.service.consul:9000"
+set -exuo pipefail
-# FIXME: s3 config (until migrated to config provider, then pwd will be reset)
-mkdir -p ${HOME}/.aws
-echo "[default]
-aws_access_key_id = storage
-aws_secret_access_key = Storage1234" >> "$HOME/.aws/credentials"
-
-PYTHON_SCRIPT="/w/workspace/test-logs/logs_publish.py"
-
-# This script uploads the artifacts to a backup upload location
-if [ -f "$PYTHON_SCRIPT" ]; then
- echo "WARNING: $PYTHON_SCRIPT already exists - assume backup archive upload already done"
- exit 0
-fi
+PYTHON_SCRIPT="/w/workspace/publish_library.py"
pip3 install boto3
mkdir -p $(dirname "$PYTHON_SCRIPT")
cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT'
#!/usr/bin/python3
-"""Storage utilities library."""
+"""S3 publish library."""
import gzip
import logging
if not mime:
mime = u"application/octet-stream"
- if s3_bucket not in u"docs.fd.io":
+ if u"logs" in s3_bucket:
if mime in COMPRESS_MIME and encoding != u"gzip":
compress(src_fpath)
src_fpath = src_fpath + u".gz"
)
+def deploy_docs(s3_bucket, s3_path, docs_dir):
+ """Ship docs dir content to S3 bucket. Requires the s3 bucket to exist.
+
+ :param s3_bucket: Name of S3 bucket. Eg: lf-project-date
+ :param s3_path: Path on S3 bucket to place the docs. Eg:
+ csit/${GERRIT_BRANCH}/report
+ :param docs_dir: Directory in which to recursively upload content.
+ :type s3_bucket: Object
+ :type s3_path: str
+ :type docs_dir: str
+ """
+ s3_resource = boto3.resource(u"s3")
+
+ upload_recursive(
+ s3_resource=s3_resource,
+ s3_bucket=s3_bucket,
+ src_fpath=docs_dir,
+ s3_path=s3_path
+ )
+
+
def deploy_s3(s3_bucket, s3_path, build_url, workspace):
"""Add logs and archives to temp directory to be shipped to S3 bucket.
Fetches logs and system information and pushes them and archives to S3
globals()[sys.argv[1]](*sys.argv[2:])
END_OF_PYTHON_SCRIPT
-
-# The 'deploy_s3' command below expects the archives
-# directory to exist. Normally lf-infra-sysstat or similar would
-# create it and add content, but to make sure this script is
-# self-contained, we ensure it exists here.
-mkdir -p "$WORKSPACE/archives"
-
-s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/"
-echo "INFO: S3 path $s3_path"
-
-echo "INFO: archiving backup logs to S3"
-# shellcheck disable=SC2086
-python3 $PYTHON_SCRIPT deploy_s3 "logs.fd.io" "$s3_path" \
- "$BUILD_URL" "$WORKSPACE"
-
-echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>"
--- /dev/null
+#!/bin/bash
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "---> publish_logs.sh"
+
+S3_BUCKET="logs.fd.io"
+CDN_URL="logs.nginx.service.consul"
+export AWS_ENDPOINT_URL="http://storage.service.consul:9000"
+PYTHON_SCRIPT="/w/workspace/publish_library.py"
+
+# FIXME: s3 config (until migrated to config provider, then pwd will be reset)
+mkdir -p ${HOME}/.aws
+echo "[default]
+aws_access_key_id = storage
+aws_secret_access_key = Storage1234" > "$HOME/.aws/credentials"
+
+mkdir -p "$WORKSPACE/archives"
+
+s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/"
+
+echo "INFO: S3 path $s3_path"
+
+echo "INFO: archiving backup logs to S3"
+python3 $PYTHON_SCRIPT deploy_s3 "$S3_BUCKET" "$s3_path" \
+ "$BUILD_URL" "$WORKSPACE"
+
+echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>"
Code Review / ci-management.git / commitdiff