b45d21ac6616f14f70a50a66446e8cfe6666fb16
[csit.git] / resources / tools / presentation / input_data_files.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Inputs
15 Download all data.
16 """
17
18 import re
19 import gzip
20 import logging
21
22 from os import rename, remove
23 from os.path import join, getsize
24 from shutil import move
25 from zipfile import ZipFile, is_zipfile, BadZipfile
26
27 from httplib import responses
28 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
29     HTTPError, ConnectionError
30
31 from errors import PresentationError
32
33
34 # Chunk size used for file download
35 CHUNK_SIZE = 512
36
37 # Separator used in file names
38 SEPARATOR = "__"
39
40 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
41
42
43 def download_data_files(spec):
44     """Download all data specified in the specification file in the section
45     type: input --> builds.
46
47     :param spec: Specification.
48     :type spec: Specification
49     :raises: PresentationError if there is no url defined for the job.
50     """
51
52     for job, builds in spec.builds.items():
53         for build in builds:
54             if job.startswith("csit-"):
55                 if spec.input["file-name"].endswith(".zip"):
56                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
57                 elif spec.input["file-name"].endswith(".gz"):
58                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
59                 else:
60                     logging.error("Not supported file format.")
61                     continue
62             elif job.startswith("hc2vpp-"):
63                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
64             else:
65                 raise PresentationError("No url defined for the job '{}'.".
66                                         format(job))
67             file_name = spec.input["file-name"]
68             full_name = spec.input["download-path"].\
69                 format(job=job, build=build["build"], filename=file_name)
70             url = "{0}/{1}".format(url, full_name)
71             new_name = join(
72                 spec.environment["paths"]["DIR[WORKING,DATA]"],
73                 "{job}{sep}{build}{sep}{name}".format(job=job,
74                                                       sep=SEPARATOR,
75                                                       build=build["build"],
76                                                       name=file_name))
77             logging.info(
78                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
79
80             status = "failed"
81             try:
82                 response = get(url, stream=True)
83                 code = response.status_code
84
85                 # temporary workaround, remove when output.log.xml is not needed
86                 if code != codes["OK"] and \
87                         spec.input["file-name"].endswith(".gz"):
88                     url = '.'.join(url.split('.')[:-1]) + ".log.gz"
89                     response = get(url, stream=True)
90                     code = response.status_code
91
92                 if code != codes["OK"]:
93                     logging.warning(
94                         "Jenkins: {0}: {1}.".format(code, responses[code]))
95                     logging.info("Trying to download from Nexus:")
96                     spec.set_input_state(job, build["build"], "not found")
97                     if code == codes["not_found"]:
98                         release = re.search(REGEX_RELEASE, job).group(2)
99                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
100                             format(job=job, sep=SEPARATOR, build=build["build"],
101                                    name=file_name)
102                         try:
103                             release = "rls".format(int(release))
104                         except ValueError:
105                             pass
106                         url = "{url}/{release}/{dir}/{file}".\
107                             format(url=spec.environment["urls"]["URL[NEXUS]"],
108                                    release=release,
109                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
110                                    file=nexus_file_name)
111                         logging.info("Downloading the file '{0}' to '{1}' ...".
112                                      format(url, new_name))
113                         response = get(url, stream=True)
114                         code = response.status_code
115                         if code != codes["OK"]:
116                             logging.error(
117                                 "Nexus: {0}: {1}".format(code, responses[code]))
118                             spec.set_input_state(
119                                 job, build["build"], "not found")
120                             continue
121
122                 file_handle = open(new_name, "wb")
123                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
124                     if chunk:
125                         file_handle.write(chunk)
126                 file_handle.close()
127
128                 if spec.input["file-name"].endswith(".zip"):
129                     expected_length = None
130                     try:
131                         expected_length = int(response.
132                                               headers["Content-Length"])
133                         logging.debug("  Expected file size: {0}B".
134                                       format(expected_length))
135                     except KeyError:
136                         logging.debug("  No information about expected size.")
137
138                     real_length = getsize(new_name)
139                     logging.debug("  Downloaded size: {0}B".format(real_length))
140
141                     if expected_length:
142                         if real_length == expected_length:
143                             status = "downloaded"
144                             logging.info("{0}: {1}".format(code,
145                                                            responses[code]))
146                         else:
147                             logging.error("The file size differs from the "
148                                           "expected size.")
149                     else:
150                         status = "downloaded"
151                         logging.info("{0}: {1}".format(code, responses[code]))
152
153                 elif spec.input["file-name"].endswith(".gz"):
154                     rename(new_name, new_name[:-3])
155                     with open(new_name[:-3], 'r') as xml_file:
156                         with gzip.open(new_name, 'w') as gz_file:
157                             gz_file.write(xml_file.read())
158                     new_name = new_name[:-3]
159                     status = "downloaded"
160                     logging.info("{0}: {1}".format(code, responses[code]))
161
162             except ConnectionError as err:
163                 logging.error("Not possible to connect to '{0}'.".format(url))
164                 logging.debug(err)
165             except HTTPError as err:
166                 logging.error("Invalid HTTP response from '{0}'.".format(url))
167                 logging.debug(err)
168             except TooManyRedirects as err:
169                 logging.error("Request exceeded the configured number "
170                               "of maximum re-directions.")
171                 logging.debug(err)
172             except Timeout as err:
173                 logging.error("Request timed out.")
174                 logging.debug(err)
175             except RequestException as err:
176                 logging.error("Unexpected HTTP request exception.")
177                 logging.debug(err)
178             except (IOError, ValueError, KeyError) as err:
179                 logging.error("Download failed.")
180                 logging.debug("Reason: {0}".format(err))
181
182             spec.set_input_state(job, build["build"], status)
183             spec.set_input_file_name(job, build["build"], new_name)
184
185             if status == "failed":
186                 logging.info("Removing the file '{0}'".format(new_name))
187                 try:
188                     remove(new_name)
189                 except OSError as err:
190                     logging.warning(str(err))
191                 spec.set_input_file_name(job, build["build"], None)
192
193     unzip_files(spec)
194
195
196 def unzip_files(spec):
197     """Unzip downloaded zip files
198
199     :param spec: Specification.
200     :type spec: Specification
201     :raises: PresentationError if the zip file does not exist or it is not a
202     zip file.
203     """
204
205     if spec.is_debug:
206         data_file = spec.debug["extract"]
207     else:
208         data_file = spec.input["extract"]
209
210     for job, builds in spec.builds.items():
211         for build in builds:
212             if build["status"] == "failed" or build["status"] == "not found":
213                 continue
214             try:
215                 status = "failed"
216                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
217                 file_name = join(build["file-name"])
218
219                 if build["status"] == "downloaded":
220                     logging.info("Unziping: '{0}' from '{1}'.".
221                                  format(data_file, file_name))
222                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
223                                                   SEPARATOR,
224                                                   data_file.split("/")[-1])
225                     try:
226                         if is_zipfile(file_name):
227                             with ZipFile(file_name, 'r') as zip_file:
228                                 zip_file.extract(data_file, directory)
229                             logging.info("Moving {0} to {1} ...".
230                                          format(join(directory, data_file),
231                                                 directory))
232                             move(join(directory, data_file), directory)
233                             logging.info("Renaming the file '{0}' to '{1}'".
234                                          format(join(directory,
235                                                      data_file.split("/")[-1]),
236                                                 new_name))
237                             rename(join(directory, data_file.split("/")[-1]),
238                                    new_name)
239                             spec.set_input_file_name(job, build["build"],
240                                                      new_name)
241                         status = "unzipped"
242                         spec.set_input_state(job, build["build"], status)
243                     except (BadZipfile, RuntimeError) as err:
244                         logging.error("Failed to unzip the file '{0}': {1}.".
245                                       format(file_name, str(err)))
246                     except OSError as err:
247                         logging.error("Failed to rename the file '{0}': {1}.".
248                                       format(data_file, str(err)))
249                     finally:
250                         if status == "failed":
251                             spec.set_input_file_name(job, build["build"], None)
252                 else:
253                     raise PresentationError("The file '{0}' does not exist or "
254                                             "it is not a zip file".
255                                             format(file_name))
256
257                 spec.set_input_state(job, build["build"], status)
258
259             except KeyError:
260                 pass

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.