1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
20 from os import rename, remove
21 from os.path import join, getsize
22 from zipfile import ZipFile, is_zipfile, BadZipfile
23 from httplib import responses
24 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
25 HTTPError, ConnectionError
27 from errors import PresentationError
30 # Chunk size used for file download
33 # Separator used in file names
37 def download_data_files(config):
38 """Download all data specified in the configuration file in the section
39 type: input --> builds.
41 :param config: Configuration.
42 :type config: Configuration
43 :raises: PresentationError if there is no url defined for the job.
46 for job, builds in config.builds.items():
48 if job.startswith("csit-"):
49 url = config.environment["urls"]["URL[JENKINS,CSIT]"]
50 elif job.startswith("hc2vpp-"):
51 url = config.environment["urls"]["URL[JENKINS,HC]"]
53 raise PresentationError("No url defined for the job '{}'.".
55 file_name = config.input["file-name"]
56 full_name = config.input["download-path"].\
57 format(job=job, build=build["build"], filename=file_name)
58 url = "{0}/{1}".format(url, full_name)
60 config.environment["paths"]["DIR[WORKING,DATA]"],
61 "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR,
65 logging.info("Downloading the file '{0}' to '{1}'.".
66 format(url, new_name))
70 response = get(url, stream=True)
71 code = response.status_code
72 if code != codes["OK"]:
73 logging.error("{0}: {1}".format(code, responses[code]))
74 config.set_input_state(job, build["build"], "not found")
77 file_handle = open(new_name, "wb")
78 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
80 file_handle.write(chunk)
83 expected_length = int(response.headers["Content-Length"])
84 logging.debug(" Expected file size: {0}B".
85 format(expected_length))
86 real_length = getsize(new_name)
87 logging.debug(" Downloaded size: {0}B".format(real_length))
89 if real_length == expected_length:
91 logging.info("{0}: {1}".format(code, responses[code]))
93 logging.error("The file size differs from the expected "
95 except ConnectionError as err:
96 logging.error("Not possible to connect to '{0}'.".format(url))
98 except HTTPError as err:
99 logging.error("Invalid HTTP response from '{0}'.".format(url))
101 except TooManyRedirects as err:
102 logging.error("Request exceeded the configured number "
103 "of maximum re-directions.")
105 except Timeout as err:
106 logging.error("Request timed out.")
108 except RequestException as err:
109 logging.error("Unexpected HTTP request exception.")
111 except (IOError, ValueError, KeyError) as err:
112 logging.error("Download failed.")
113 logging.debug("Reason: {0}".format(err))
115 config.set_input_state(job, build["build"], status)
116 config.set_input_file_name(job, build["build"], new_name)
118 if status == "failed":
119 logging.info("Removing the file '{0}'".format(new_name))
122 except OSError as err:
123 logging.warning(str(err))
124 config.set_input_file_name(job, build["build"], None)
129 def unzip_files(config):
130 """Unzip downloaded zip files
132 :param config: Configuration.
133 :type config: Configuration
134 :raises: PresentationError if the zip file does not exist or it is not a
139 data_file = config.debug["extract"]
141 data_file = config.input["extract"]
143 for job, builds in config.builds.items():
147 file_name = build["file-name"]
148 directory = config.environment["paths"]["DIR[WORKING,DATA]"]
149 if build["status"] == "downloaded" and is_zipfile(file_name):
150 logging.info("Unziping: '{0}' from '{1}'.".
151 format(data_file, file_name))
152 new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
153 SEPARATOR, data_file)
155 with ZipFile(file_name, 'r') as zip_file:
156 zip_file.extract(data_file, directory)
157 logging.info("Renaming the file '{0}' to '{1}'".
158 format(data_file, new_name))
159 rename(join(directory, data_file), new_name)
161 config.set_input_state(job, build["build"], status)
162 config.set_input_file_name(job, build["build"],
164 except (BadZipfile, RuntimeError) as err:
165 logging.error("Failed to unzip the file '{0}': {1}.".
166 format(file_name, str(err)))
167 except OSError as err:
168 logging.error("Failed to rename the file '{0}': {1}.".
169 format(data_file, str(err)))
171 logging.info("Removing the file '{0}'".
176 except OSError as err:
177 logging.warning(str(err))
178 if status == "failed":
179 config.set_input_file_name(job, build["build"],
182 raise PresentationError("The file '{0}' does not exist or "
183 "it is not a zip file".
186 config.set_input_state(job, build["build"], status)