CSIT Trending: Input files
[csit.git] / resources / tools / presentation / input_data_files.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Inputs
15 Download all data.
16 """
17
18 import re
19 import gzip
20 import logging
21
22 from os import rename, remove
23 from os.path import join, getsize
24 from shutil import move
25 from zipfile import ZipFile, is_zipfile, BadZipfile
26 from httplib import responses
27 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
28     HTTPError, ConnectionError
29
30 from errors import PresentationError
31 from utils import execute_command
32
33 # Chunk size used for file download
34 CHUNK_SIZE = 512
35
36 # Separator used in file names
37 SEPARATOR = "__"
38
39 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
40
41
42 def download_data_files(spec):
43     """Download all data specified in the specification file in the section
44     type: input --> builds.
45
46     :param spec: Specification.
47     :type spec: Specification
48     :raises: PresentationError if there is no url defined for the job.
49     """
50
51     for job, builds in spec.builds.items():
52         for build in builds:
53             if job.startswith("csit-"):
54                 if spec.input["file-name"].endswith(".zip"):
55                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
56                 elif spec.input["file-name"].endswith(".gz"):
57                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
58                 else:
59                     logging.error("Not supported file format.")
60                     continue
61             elif job.startswith("hc2vpp-"):
62                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
63             else:
64                 raise PresentationError("No url defined for the job '{}'.".
65                                         format(job))
66             file_name = spec.input["file-name"]
67             full_name = spec.input["download-path"].\
68                 format(job=job, build=build["build"], filename=file_name)
69             url = "{0}/{1}".format(url, full_name)
70             new_name = join(
71                 spec.environment["paths"]["DIR[WORKING,DATA]"],
72                 "{job}{sep}{build}{sep}{name}".format(job=job,
73                                                       sep=SEPARATOR,
74                                                       build=build["build"],
75                                                       name=file_name))
76             logging.info(
77                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
78
79             status = "failed"
80             try:
81                 response = get(url, stream=True)
82                 code = response.status_code
83
84                 # temporary workaround, remove when output.log.xml is not needed
85                 if code != codes["OK"] and \
86                         spec.input["file-name"].endswith(".gz"):
87                     url = '.'.join(url.split('.')[:-1]) + ".log.gz"
88                     response = get(url, stream=True)
89                     code = response.status_code
90
91                 if code != codes["OK"]:
92                     logging.warning(
93                         "Jenkins: {0}: {1}.".format(code, responses[code]))
94                     logging.info("Trying to download from Nexus:")
95                     spec.set_input_state(job, build["build"], "not found")
96                     if code == codes["not_found"]:
97                         release = re.search(REGEX_RELEASE, job).group(2)
98                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
99                             format(job=job, sep=SEPARATOR, build=build["build"],
100                                    name=file_name)
101                         try:
102                             release = "rls".format(int(release))
103                         except ValueError:
104                             pass
105                         url = "{url}/{release}/{dir}/{file}".\
106                             format(url=spec.environment["urls"]["URL[NEXUS]"],
107                                    release=release,
108                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
109                                    file=nexus_file_name)
110                         logging.info("Downloading the file '{0}' to '{1}' ...".
111                                      format(url, new_name))
112                         response = get(url, stream=True)
113                         code = response.status_code
114                         if code != codes["OK"]:
115                             logging.error(
116                                 "Nexus: {0}: {1}".format(code, responses[code]))
117                             spec.set_input_state(
118                                 job, build["build"], "not found")
119                             continue
120
121                 file_handle = open(new_name, "wb")
122                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
123                     if chunk:
124                         file_handle.write(chunk)
125                 file_handle.close()
126
127                 if spec.input["file-name"].endswith(".zip"):
128                     expected_length = None
129                     try:
130                         expected_length = int(response.
131                                               headers["Content-Length"])
132                         logging.debug("  Expected file size: {0}B".
133                                       format(expected_length))
134                     except KeyError:
135                         logging.debug("  No information about expected size.")
136
137                     real_length = getsize(new_name)
138                     logging.debug("  Downloaded size: {0}B".format(real_length))
139
140                     if expected_length:
141                         if real_length == expected_length:
142                             status = "downloaded"
143                             logging.info("{0}: {1}".format(code,
144                                                            responses[code]))
145                         else:
146                             logging.error("The file size differs from the "
147                                           "expected size.")
148                     else:
149                         status = "downloaded"
150                         logging.info("{0}: {1}".format(code, responses[code]))
151
152                 elif spec.input["file-name"].endswith(".gz"):
153                     rename(new_name, new_name[:-3])
154                     execute_command("gzip --keep {0}".format(new_name[:-3]))
155                     new_name = new_name[:-3]
156                     status = "downloaded"
157                     logging.info("{0}: {1}".format(code, responses[code]))
158
159             except ConnectionError as err:
160                 logging.error("Not possible to connect to '{0}'.".format(url))
161                 logging.debug(err)
162             except HTTPError as err:
163                 logging.error("Invalid HTTP response from '{0}'.".format(url))
164                 logging.debug(err)
165             except TooManyRedirects as err:
166                 logging.error("Request exceeded the configured number "
167                               "of maximum re-directions.")
168                 logging.debug(err)
169             except Timeout as err:
170                 logging.error("Request timed out.")
171                 logging.debug(err)
172             except RequestException as err:
173                 logging.error("Unexpected HTTP request exception.")
174                 logging.debug(err)
175             except (IOError, ValueError, KeyError) as err:
176                 logging.error("Download failed.")
177                 logging.debug("Reason: {0}".format(err))
178
179             spec.set_input_state(job, build["build"], status)
180             spec.set_input_file_name(job, build["build"], new_name)
181
182             if status == "failed":
183                 logging.info("Removing the file '{0}'".format(new_name))
184                 try:
185                     remove(new_name)
186                 except OSError as err:
187                     logging.warning(str(err))
188                 spec.set_input_file_name(job, build["build"], None)
189
190     unzip_files(spec)
191
192
193 def unzip_files(spec):
194     """Unzip downloaded zip files
195
196     :param spec: Specification.
197     :type spec: Specification
198     :raises: PresentationError if the zip file does not exist or it is not a
199     zip file.
200     """
201
202     if spec.is_debug:
203         data_file = spec.debug["extract"]
204     else:
205         data_file = spec.input["extract"]
206
207     for job, builds in spec.builds.items():
208         for build in builds:
209             if build["status"] == "failed" or build["status"] == "not found":
210                 continue
211             try:
212                 status = "failed"
213                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
214                 file_name = join(build["file-name"])
215
216                 if build["status"] == "downloaded":
217                     logging.info("Unziping: '{0}' from '{1}'.".
218                                  format(data_file, file_name))
219                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
220                                                   SEPARATOR,
221                                                   data_file.split("/")[-1])
222                     try:
223                         if is_zipfile(file_name):
224                             with ZipFile(file_name, 'r') as zip_file:
225                                 zip_file.extract(data_file, directory)
226                             logging.info("Moving {0} to {1} ...".
227                                          format(join(directory, data_file),
228                                                 directory))
229                             move(join(directory, data_file), directory)
230                             logging.info("Renaming the file '{0}' to '{1}'".
231                                          format(join(directory,
232                                                      data_file.split("/")[-1]),
233                                                 new_name))
234                             rename(join(directory, data_file.split("/")[-1]),
235                                    new_name)
236                             spec.set_input_file_name(job, build["build"],
237                                                      new_name)
238                         status = "unzipped"
239                         spec.set_input_state(job, build["build"], status)
240                     except (BadZipfile, RuntimeError) as err:
241                         logging.error("Failed to unzip the file '{0}': {1}.".
242                                       format(file_name, str(err)))
243                     except OSError as err:
244                         logging.error("Failed to rename the file '{0}': {1}.".
245                                       format(data_file, str(err)))
246                     finally:
247                         if status == "failed":
248                             spec.set_input_file_name(job, build["build"], None)
249                 else:
250                     raise PresentationError("The file '{0}' does not exist or "
251                                             "it is not a zip file".
252                                             format(file_name))
253
254                 spec.set_input_state(job, build["build"], status)
255
256             except KeyError:
257                 pass

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.