jjb/scripts/logs_publish.sh

   1 #!/bin/bash
   2
   3 # Copyright (c) 2021 Cisco and/or its affiliates.
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 echo "---> logs_publish.sh"
  17
  18 CDN_URL="logs.nginx.service.consul"
  19 export AWS_ENDPOINT_URL="http://storage.service.consul:9000"
  20
  21 # FIXME: s3 config (until migrated to config provider, then pwd will be reset)
  22 mkdir -p ${HOME}/.aws
  23 echo "[default]
  24 aws_access_key_id = storage
  25 aws_secret_access_key = Storage1234" >> "$HOME/.aws/credentials"
  26
  27 PYTHON_SCRIPT="/w/workspace/test-logs/logs_publish.py"
  28
  29 # This script uploads the artifacts to a backup upload location
  30 if [ -f "$PYTHON_SCRIPT" ]; then
  31     echo "WARNING: $PYTHON_SCRIPT already exists - assume backup archive upload already done"
  32     exit 0
  33 fi
  34
  35 pip3 install boto3
  36 mkdir -p $(dirname "$PYTHON_SCRIPT")
  37
  38 cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT'
  39 #!/usr/bin/python3
  40
  41 """Storage utilities library."""
  42
  43 import gzip
  44 import logging
  45 import os
  46 import shutil
  47 import subprocess
  48 import sys
  49 import tempfile
  50 from mimetypes import MimeTypes
  51
  52 import boto3
  53 from botocore.exceptions import ClientError
  54 import requests
  55 import six
  56
  57
  58 logging.basicConfig(
  59     format=u"%(levelname)s: %(message)s",
  60     stream=sys.stdout,
  61     level=logging.INFO
  62 )
  63 logging.getLogger(u"botocore").setLevel(logging.INFO)
  64
  65 COMPRESS_MIME = (
  66     u"text/html",
  67     u"text/xml",
  68     u"text/plain",
  69     u"application/octet-stream"
  70 )
  71
  72
  73 def compress(src_fpath):
  74     """Compress a single file.
  75
  76     :param src_fpath: Input file path.
  77     :type src_fpath: str
  78     """
  79     with open(src_fpath, u"rb") as orig_file:
  80         with gzip.open(src_fpath + ".gz", u"wb") as zipped_file:
  81             zipped_file.writelines(orig_file)
  82
  83
  84 def copy_archives(workspace):
  85     """Copy files or directories in a $WORKSPACE/archives to the current
  86     directory.
  87
  88     :params workspace: Workspace directery with archives directory.
  89     :type workspace: str
  90     """
  91     archives_dir = os.path.join(workspace, u"archives")
  92     dest_dir = os.getcwd()
  93
  94     logging.debug(u"Copying files from " + archives_dir + u" to " + dest_dir)
  95
  96     if os.path.exists(archives_dir):
  97         if os.path.isfile(archives_dir):
  98             logging.error(u"Target is a file, not a directory.")
  99             raise RuntimeError(u"Not a directory.")
 100         else:
 101             logging.debug("Archives dir {} does exist.".format(archives_dir))
 102             for file_or_dir in os.listdir(archives_dir):
 103                 f = os.path.join(archives_dir, file_or_dir)
 104                 try:
 105                     logging.debug(u"Copying " + f)
 106                     shutil.copy(f, dest_dir)
 107                 except shutil.Error as e:
 108                     logging.error(e)
 109                     raise RuntimeError(u"Could not copy " + f)
 110     else:
 111         logging.error(u"Archives dir does not exist.")
 112         raise RuntimeError(u"Missing directory " + archives_dir)
 113
 114
 115 def upload(s3_resource, s3_bucket, src_fpath, s3_path):
 116     """Upload single file to destination bucket.
 117
 118     :param s3_resource: S3 storage resource.
 119     :param s3_bucket: S3 bucket name.
 120     :param src_fpath: Input file path.
 121     :param s3_path: Destination file path on remote storage.
 122     :type s3_resource: Object
 123     :type s3_bucket: str
 124     :type src_fpath: str
 125     :type s3_path: str
 126     """
 127     mime_guess = MimeTypes().guess_type(src_fpath)
 128     mime = mime_guess[0]
 129     encoding = mime_guess[1]
 130     if not mime:
 131         mime = u"application/octet-stream"
 132
 133     if s3_bucket not in u"docs.fd.io":
 134         if mime in COMPRESS_MIME and encoding != u"gzip":
 135             compress(src_fpath)
 136             src_fpath = src_fpath + u".gz"
 137             s3_path = s3_path + u".gz"
 138
 139     extra_args = {u"ContentType": mime}
 140
 141     try:
 142         logging.info(u"Attempting to upload file " + src_fpath)
 143         s3_resource.Bucket(s3_bucket).upload_file(
 144             src_fpath, s3_path, ExtraArgs=extra_args
 145         )
 146         logging.info(u"Successfully uploaded to " + s3_path)
 147     except ClientError as e:
 148         logging.error(e)
 149
 150
 151 def upload_recursive(s3_resource, s3_bucket, src_fpath, s3_path):
 152     """Recursively uploads input folder to destination.
 153
 154     Example:
 155       - s3_bucket: logs.fd.io
 156       - src_fpath: /workspace/archives.
 157       - s3_path: /hostname/job/id/
 158
 159     :param s3_resource: S3 storage resource.
 160     :param s3_bucket: S3 bucket name.
 161     :param src_fpath: Input folder path.
 162     :param s3_path: S3 destination path.
 163     :type s3_resource: Object
 164     :type s3_bucket: str
 165     :type src_fpath: str
 166     :type s3_path: str
 167     """
 168     for path, _, files in os.walk(src_fpath):
 169         for file in files:
 170             _path = path.replace(src_fpath, u"")
 171             _src_fpath = path + u"/" + file
 172             _s3_path = os.path.normpath(s3_path + u"/" + _path + u"/" + file)
 173             upload(
 174                 s3_resource=s3_resource,
 175                 s3_bucket=s3_bucket,
 176                 src_fpath=_src_fpath,
 177                 s3_path=_s3_path
 178             )
 179
 180
 181 def deploy_s3(s3_bucket, s3_path, build_url, workspace):
 182     """Add logs and archives to temp directory to be shipped to S3 bucket.
 183     Fetches logs and system information and pushes them and archives to S3
 184     for log archiving.
 185     Requires the s3 bucket to exist.
 186
 187     :param s3_bucket: Name of S3 bucket. Eg: lf-project-date
 188     :param s3_path: Path on S3 bucket place the logs and archives. Eg:
 189         $JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
 190     :param build_url: URL of the Jenkins build. Jenkins typically provides this
 191         via the $BUILD_URL environment variable.
 192     :param workspace: Directory in which to search, typically in Jenkins this is
 193         $WORKSPACE
 194     :type s3_bucket: Object
 195     :type s3_path: str
 196     :type build_url: str
 197     :type workspace: str
 198     """
 199     s3_resource = boto3.resource(
 200         u"s3",
 201         endpoint_url=os.environ[u"AWS_ENDPOINT_URL"]
 202     )
 203
 204     previous_dir = os.getcwd()
 205     work_dir = tempfile.mkdtemp(prefix="backup-s3.")
 206     os.chdir(work_dir)
 207
 208     # Copy archive files to tmp dir.
 209     copy_archives(workspace)
 210
 211     # Create additional build logs.
 212     with open(u"_build-details.log", u"w+") as f:
 213         f.write(u"build-url: " + build_url)
 214
 215     with open(u"_sys-info.log", u"w+") as f:
 216         sys_cmds = []
 217
 218         logging.debug(u"Platform: " + sys.platform)
 219         if sys.platform == u"linux" or sys.platform == u"linux2":
 220             sys_cmds = [
 221                 [u"uname", u"-a"],
 222                 [u"lscpu"],
 223                 [u"nproc"],
 224                 [u"df", u"-h"],
 225                 [u"free", u"-m"],
 226                 [u"ip", u"addr"],
 227                 [u"sar", u"-b", u"-r", u"-n", u"DEV"],
 228                 [u"sar", u"-P", u"ALL"],
 229             ]
 230
 231         for c in sys_cmds:
 232             try:
 233                 output = subprocess.check_output(c).decode(u"utf-8")
 234             except FileNotFoundError:
 235                 logging.debug(u"Command not found: " + c)
 236                 continue
 237
 238             cmd = u" ".join(c)
 239             output = u"---> " + cmd + "\n" + output + "\n"
 240             f.write(output)
 241             logging.info(output)
 242
 243     # Magic string used to trim console logs at the appropriate level during
 244     # wget.
 245     MAGIC_STRING = u"-----END_OF_BUILD-----"
 246     logging.info(MAGIC_STRING)
 247
 248     resp = requests.get(build_url + u"/consoleText")
 249     with open(u"console.log", u"w+", encoding=u"utf-8") as f:
 250         f.write(
 251             six.text_type(resp.content.decode(u"utf-8").split(MAGIC_STRING)[0])
 252         )
 253
 254     query = u"time=HH:mm:ss&appendLog"
 255     resp = requests.get(build_url + u"/timestamps?" + query)
 256     with open(u"console-timestamp.log", u"w+", encoding=u"utf-8") as f:
 257         f.write(
 258             six.text_type(resp.content.decode(u"utf-8").split(MAGIC_STRING)[0])
 259         )
 260
 261     upload_recursive(
 262         s3_resource=s3_resource,
 263         s3_bucket=s3_bucket,
 264         src_fpath=work_dir,
 265         s3_path=s3_path
 266     )
 267
 268     os.chdir(previous_dir)
 269     shutil.rmtree(work_dir)
 270
 271
 272 if __name__ == u"__main__":
 273     globals()[sys.argv[1]](*sys.argv[2:])
 274
 275 END_OF_PYTHON_SCRIPT
 276
 277 # The 'deploy_s3' command below expects the archives
 278 # directory to exist.  Normally lf-infra-sysstat or similar would
 279 # create it and add content, but to make sure this script is
 280 # self-contained, we ensure it exists here.
 281 mkdir -p "$WORKSPACE/archives"
 282
 283 s3_path="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER/"
 284 echo "INFO: S3 path $s3_path"
 285
 286 echo "INFO: archiving backup logs to S3"
 287 # shellcheck disable=SC2086
 288 python3 $PYTHON_SCRIPT deploy_s3 "logs.fd.io" "$s3_path" \
 289     "$BUILD_URL" "$WORKSPACE"
 290
 291 echo "S3 build backup logs: <a href=\"https://$CDN_URL/$s3_path\">https://$CDN_URL/$s3_path</a>"