CSIT Trending: input files
[csit.git] / resources / tools / presentation / input_data_files.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Inputs
15 Download all data.
16 """
17
18 import re
19 import gzip
20 import logging
21
22 from os import rename, remove
23 from os.path import join, getsize
24 from shutil import move
25 from zipfile import ZipFile, is_zipfile, BadZipfile
26 from httplib import responses
27 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
28     HTTPError, ConnectionError
29
30 from errors import PresentationError
31 from utils import execute_command
32
33 # Chunk size used for file download
34 CHUNK_SIZE = 512
35
36 # Separator used in file names
37 SEPARATOR = "__"
38
39 REGEX_RELEASE = re.compile(r'(\D*)(\d{4}|master)(\D*)')
40
41
42 def download_data_files(spec):
43     """Download all data specified in the specification file in the section
44     type: input --> builds.
45
46     :param spec: Specification.
47     :type spec: Specification
48     :raises: PresentationError if there is no url defined for the job.
49     """
50
51     for job, builds in spec.builds.items():
52         for build in builds:
53             if job.startswith("csit-"):
54                 if spec.input["file-name"].endswith(".zip"):
55                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
56                 elif spec.input["file-name"].endswith(".gz"):
57                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
58                 else:
59                     logging.error("Not supported file format.")
60                     continue
61             elif job.startswith("hc2vpp-"):
62                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
63             else:
64                 raise PresentationError("No url defined for the job '{}'.".
65                                         format(job))
66             file_name = spec.input["file-name"]
67             full_name = spec.input["download-path"].\
68                 format(job=job, build=build["build"], filename=file_name)
69             url = "{0}/{1}".format(url, full_name)
70             new_name = join(
71                 spec.environment["paths"]["DIR[WORKING,DATA]"],
72                 "{job}{sep}{build}{sep}{name}".format(job=job,
73                                                       sep=SEPARATOR,
74                                                       build=build["build"],
75                                                       name=file_name))
76             logging.info(
77                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
78
79             status = "failed"
80             try:
81                 response = get(url, stream=True)
82                 code = response.status_code
83
84                 if code != codes["OK"]:
85                     logging.warning(
86                         "Jenkins: {0}: {1}.".format(code, responses[code]))
87                     logging.info("Trying to download from Nexus:")
88                     spec.set_input_state(job, build["build"], "not found")
89                     if code == codes["not_found"]:
90                         release = re.search(REGEX_RELEASE, job).group(2)
91                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
92                             format(job=job, sep=SEPARATOR, build=build["build"],
93                                    name=file_name)
94                         try:
95                             release = "rls".format(int(release))
96                         except ValueError:
97                             pass
98                         url = "{url}/{release}/{dir}/{file}".\
99                             format(url=spec.environment["urls"]["URL[NEXUS]"],
100                                    release=release,
101                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
102                                    file=nexus_file_name)
103                         logging.info("Downloading the file '{0}' to '{1}' ...".
104                                      format(url, new_name))
105                         response = get(url, stream=True)
106                         code = response.status_code
107                         if code != codes["OK"]:
108                             logging.error(
109                                 "Nexus: {0}: {1}".format(code, responses[code]))
110                             spec.set_input_state(
111                                 job, build["build"], "not found")
112                             continue
113
114                 file_handle = open(new_name, "wb")
115                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
116                     if chunk:
117                         file_handle.write(chunk)
118                 file_handle.close()
119
120                 if spec.input["file-name"].endswith(".zip"):
121                     expected_length = None
122                     try:
123                         expected_length = int(response.
124                                               headers["Content-Length"])
125                         logging.debug("  Expected file size: {0}B".
126                                       format(expected_length))
127                     except KeyError:
128                         logging.debug("  No information about expected size.")
129
130                     real_length = getsize(new_name)
131                     logging.debug("  Downloaded size: {0}B".format(real_length))
132
133                     if expected_length:
134                         if real_length == expected_length:
135                             status = "downloaded"
136                             logging.info("{0}: {1}".format(code,
137                                                            responses[code]))
138                         else:
139                             logging.error("The file size differs from the "
140                                           "expected size.")
141                     else:
142                         status = "downloaded"
143                         logging.info("{0}: {1}".format(code, responses[code]))
144
145                 elif spec.input["file-name"].endswith(".gz"):
146                     if "docs.fd.io" in url:
147                         execute_command("gzip --decompress --keep --force {0}".
148                                         format(new_name))
149                     else:
150                         rename(new_name, new_name[:-3])
151                         execute_command("gzip --keep {0}".format(new_name[:-3]))
152                     new_name = new_name[:-3]
153                     status = "downloaded"
154                     logging.info("{0}: {1}".format(code, responses[code]))
155
156             except ConnectionError as err:
157                 logging.error("Not possible to connect to '{0}'.".format(url))
158                 logging.debug(err)
159             except HTTPError as err:
160                 logging.error("Invalid HTTP response from '{0}'.".format(url))
161                 logging.debug(err)
162             except TooManyRedirects as err:
163                 logging.error("Request exceeded the configured number "
164                               "of maximum re-directions.")
165                 logging.debug(err)
166             except Timeout as err:
167                 logging.error("Request timed out.")
168                 logging.debug(err)
169             except RequestException as err:
170                 logging.error("Unexpected HTTP request exception.")
171                 logging.debug(err)
172             except (IOError, ValueError, KeyError) as err:
173                 logging.error("Download failed.")
174                 logging.debug("Reason: {0}".format(err))
175
176             spec.set_input_state(job, build["build"], status)
177             spec.set_input_file_name(job, build["build"], new_name)
178
179             if status == "failed":
180                 logging.info("Removing the file '{0}'".format(new_name))
181                 try:
182                     remove(new_name)
183                 except OSError as err:
184                     logging.warning(str(err))
185                 spec.set_input_file_name(job, build["build"], None)
186
187     unzip_files(spec)
188
189
190 def unzip_files(spec):
191     """Unzip downloaded zip files
192
193     :param spec: Specification.
194     :type spec: Specification
195     :raises: PresentationError if the zip file does not exist or it is not a
196     zip file.
197     """
198
199     if spec.is_debug:
200         data_file = spec.debug["extract"]
201     else:
202         data_file = spec.input["extract"]
203
204     for job, builds in spec.builds.items():
205         for build in builds:
206             if build["status"] == "failed" or build["status"] == "not found":
207                 continue
208             try:
209                 status = "failed"
210                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
211                 file_name = join(build["file-name"])
212
213                 if build["status"] == "downloaded":
214                     logging.info("Unziping: '{0}' from '{1}'.".
215                                  format(data_file, file_name))
216                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
217                                                   SEPARATOR,
218                                                   data_file.split("/")[-1])
219                     try:
220                         if is_zipfile(file_name):
221                             with ZipFile(file_name, 'r') as zip_file:
222                                 zip_file.extract(data_file, directory)
223                             logging.info("Moving {0} to {1} ...".
224                                          format(join(directory, data_file),
225                                                 directory))
226                             move(join(directory, data_file), directory)
227                             logging.info("Renaming the file '{0}' to '{1}'".
228                                          format(join(directory,
229                                                      data_file.split("/")[-1]),
230                                                 new_name))
231                             rename(join(directory, data_file.split("/")[-1]),
232                                    new_name)
233                             spec.set_input_file_name(job, build["build"],
234                                                      new_name)
235                         status = "unzipped"
236                         spec.set_input_state(job, build["build"], status)
237                     except (BadZipfile, RuntimeError) as err:
238                         logging.error("Failed to unzip the file '{0}': {1}.".
239                                       format(file_name, str(err)))
240                     except OSError as err:
241                         logging.error("Failed to rename the file '{0}': {1}.".
242                                       format(data_file, str(err)))
243                     finally:
244                         if status == "failed":
245                             spec.set_input_file_name(job, build["build"], None)
246                 else:
247                     raise PresentationError("The file '{0}' does not exist or "
248                                             "it is not a zip file".
249                                             format(file_name))
250
251                 spec.set_input_state(job, build["build"], status)
252
253             except KeyError:
254                 pass

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.