PAL Trending: download input files
[csit.git] / resources / tools / presentation / input_data_files.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Inputs
15 Download all data.
16 """
17
18 import re
19 import gzip
20 import logging
21
22 from os import rename, remove
23 from os.path import join, getsize
24 from shutil import move
25 from zipfile import ZipFile, is_zipfile, BadZipfile
26
27 from httplib import responses
28 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
29     HTTPError, ConnectionError
30
31 from errors import PresentationError
32
33
34 # Chunk size used for file download
35 CHUNK_SIZE = 512
36
37 # Separator used in file names
38 SEPARATOR = "__"
39
40 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
41
42
43 def download_data_files(spec):
44     """Download all data specified in the specification file in the section
45     type: input --> builds.
46
47     :param spec: Specification.
48     :type spec: Specification
49     :raises: PresentationError if there is no url defined for the job.
50     """
51
52     for job, builds in spec.builds.items():
53         for build in builds:
54             if job.startswith("csit-"):
55                 if spec.input["file-name"].endswith(".zip"):
56                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
57                 elif spec.input["file-name"].endswith(".gz"):
58                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
59                 else:
60                     logging.error("Not supported file format.")
61                     continue
62             elif job.startswith("hc2vpp-"):
63                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
64             else:
65                 raise PresentationError("No url defined for the job '{}'.".
66                                         format(job))
67             file_name = spec.input["file-name"]
68             full_name = spec.input["download-path"].\
69                 format(job=job, build=build["build"], filename=file_name)
70             url = "{0}/{1}".format(url, full_name)
71             new_name = join(
72                 spec.environment["paths"]["DIR[WORKING,DATA]"],
73                 "{job}{sep}{build}{sep}{name}".format(job=job,
74                                                       sep=SEPARATOR,
75                                                       build=build["build"],
76                                                       name=file_name))
77             logging.info(
78                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
79
80             status = "failed"
81             try:
82                 response = get(url, stream=True)
83                 code = response.status_code
84                 # temporary workaround, remove when output.log.xml is not needed
85                 if code != codes["OK"] and \
86                         spec.input["file-name"].endswith(".gz"):
87                     url = '.'.join(url.split('.')[:-1]) + ".log.gz"
88                     response = get(url, stream=True)
89                     code = response.status_code
90
91                 if code != codes["OK"]:
92                     logging.warning(
93                         "Jenkins: {0}: {1}.".format(code, responses[code]))
94                     logging.info("Trying to download from Nexus:")
95                     spec.set_input_state(job, build["build"], "not found")
96                     if code == codes["not_found"]:
97                         release = re.search(REGEX_RELEASE, job).group(2)
98                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
99                             format(job=job, sep=SEPARATOR, build=build["build"],
100                                    name=file_name)
101                         url = "{url}/rls{release}/{dir}/{file}".\
102                             format(url=spec.environment["urls"]["URL[NEXUS]"],
103                                    release=release,
104                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
105                                    file=nexus_file_name)
106                         logging.info("Downloading the file '{0}' to '{1}' ...".
107                                      format(url, new_name))
108                         response = get(url, stream=True)
109                         code = response.status_code
110                         if code != codes["OK"]:
111                             logging.error(
112                                 "Nexus: {0}: {1}".format(code, responses[code]))
113                             spec.set_input_state(
114                                 job, build["build"], "not found")
115                             continue
116
117                 file_handle = open(new_name, "wb")
118                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
119                     if chunk:
120                         file_handle.write(chunk)
121                 file_handle.close()
122
123                 if spec.input["file-name"].endswith(".zip"):
124                     expected_length = None
125                     try:
126                         expected_length = int(response.
127                                               headers["Content-Length"])
128                         logging.debug("  Expected file size: {0}B".
129                                       format(expected_length))
130                     except KeyError:
131                         logging.debug("  No information about expected size.")
132
133                     real_length = getsize(new_name)
134                     logging.debug("  Downloaded size: {0}B".format(real_length))
135
136                     if expected_length:
137                         if real_length == expected_length:
138                             status = "downloaded"
139                             logging.info("{0}: {1}".format(code,
140                                                            responses[code]))
141                         else:
142                             logging.error("The file size differs from the "
143                                           "expected size.")
144                     else:
145                         status = "downloaded"
146                         logging.info("{0}: {1}".format(code, responses[code]))
147
148                 elif spec.input["file-name"].endswith(".gz"):
149                     rename(new_name, new_name[:-3])
150                     with open(new_name[:-3], 'r') as xml_file:
151                         with gzip.open(new_name, 'wb') as gz_file:
152                             gz_file.write(xml_file.read())
153                     new_name = new_name[:-3]
154                     status = "downloaded"
155                     logging.info("{0}: {1}".format(code, responses[code]))
156
157             except ConnectionError as err:
158                 logging.error("Not possible to connect to '{0}'.".format(url))
159                 logging.debug(err)
160             except HTTPError as err:
161                 logging.error("Invalid HTTP response from '{0}'.".format(url))
162                 logging.debug(err)
163             except TooManyRedirects as err:
164                 logging.error("Request exceeded the configured number "
165                               "of maximum re-directions.")
166                 logging.debug(err)
167             except Timeout as err:
168                 logging.error("Request timed out.")
169                 logging.debug(err)
170             except RequestException as err:
171                 logging.error("Unexpected HTTP request exception.")
172                 logging.debug(err)
173             except (IOError, ValueError, KeyError) as err:
174                 logging.error("Download failed.")
175                 logging.debug("Reason: {0}".format(err))
176
177             spec.set_input_state(job, build["build"], status)
178             spec.set_input_file_name(job, build["build"], new_name)
179
180             if status == "failed":
181                 logging.info("Removing the file '{0}'".format(new_name))
182                 try:
183                     remove(new_name)
184                 except OSError as err:
185                     logging.warning(str(err))
186                 spec.set_input_file_name(job, build["build"], None)
187
188     unzip_files(spec)
189
190
191 def unzip_files(spec):
192     """Unzip downloaded zip files
193
194     :param spec: Specification.
195     :type spec: Specification
196     :raises: PresentationError if the zip file does not exist or it is not a
197     zip file.
198     """
199
200     if spec.is_debug:
201         data_file = spec.debug["extract"]
202     else:
203         data_file = spec.input["extract"]
204
205     for job, builds in spec.builds.items():
206         for build in builds:
207             if build["status"] == "failed" or build["status"] == "not found":
208                 continue
209             try:
210                 status = "failed"
211                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
212                 file_name = join(build["file-name"])
213
214                 if build["status"] == "downloaded":
215                     logging.info("Unziping: '{0}' from '{1}'.".
216                                  format(data_file, file_name))
217                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
218                                                   SEPARATOR,
219                                                   data_file.split("/")[-1])
220                     try:
221                         if is_zipfile(file_name):
222                             with ZipFile(file_name, 'r') as zip_file:
223                                 zip_file.extract(data_file, directory)
224                             logging.info("Moving {0} to {1} ...".
225                                          format(join(directory, data_file),
226                                                 directory))
227                             move(join(directory, data_file), directory)
228                             logging.info("Renaming the file '{0}' to '{1}'".
229                                          format(join(directory,
230                                                      data_file.split("/")[-1]),
231                                                 new_name))
232                             rename(join(directory, data_file.split("/")[-1]),
233                                    new_name)
234                             spec.set_input_file_name(job, build["build"],
235                                                      new_name)
236                         status = "unzipped"
237                         spec.set_input_state(job, build["build"], status)
238                     except (BadZipfile, RuntimeError) as err:
239                         logging.error("Failed to unzip the file '{0}': {1}.".
240                                       format(file_name, str(err)))
241                     except OSError as err:
242                         logging.error("Failed to rename the file '{0}': {1}.".
243                                       format(data_file, str(err)))
244                     finally:
245                         if status == "failed":
246                             spec.set_input_file_name(job, build["build"], None)
247                 else:
248                     raise PresentationError("The file '{0}' does not exist or "
249                                             "it is not a zip file".
250                                             format(file_name))
251
252                 spec.set_input_state(job, build["build"], status)
253
254             except KeyError:
255                 pass

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.