CSIT-913: Continuous Trending, Analysis and Change Detection
[csit.git] / resources / tools / presentation / input_data_files.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Inputs
15 Download all data.
16 """
17
18 import re
19 import gzip
20 import logging
21
22 from os import rename, remove
23 from os.path import join, getsize
24 from shutil import move
25 from zipfile import ZipFile, is_zipfile, BadZipfile
26
27 from httplib import responses
28 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
29     HTTPError, ConnectionError
30
31 from errors import PresentationError
32
33
34 # Chunk size used for file download
35 CHUNK_SIZE = 512
36
37 # Separator used in file names
38 SEPARATOR = "__"
39
40 REGEX_RELEASE = re.compile(r'(\D*)(\d{4})(\D*)')
41
42
43 def download_data_files(spec):
44     """Download all data specified in the specification file in the section
45     type: input --> builds.
46
47     :param spec: Specification.
48     :type spec: Specification
49     :raises: PresentationError if there is no url defined for the job.
50     """
51
52     for job, builds in spec.builds.items():
53         for build in builds:
54             if job.startswith("csit-"):
55                 if spec.input["file-name"].endswith(".zip"):
56                     url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
57                 elif spec.input["file-name"].endswith(".gz"):
58                     url = spec.environment["urls"]["URL[NEXUS,LOG]"]
59                 else:
60                     logging.error("Not supported file format.")
61                     continue
62             elif job.startswith("hc2vpp-"):
63                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
64             else:
65                 raise PresentationError("No url defined for the job '{}'.".
66                                         format(job))
67             file_name = spec.input["file-name"]
68             full_name = spec.input["download-path"].\
69                 format(job=job, build=build["build"], filename=file_name)
70             url = "{0}/{1}".format(url, full_name)
71             new_name = join(
72                 spec.environment["paths"]["DIR[WORKING,DATA]"],
73                 "{job}{sep}{build}{sep}{name}".format(job=job,
74                                                       sep=SEPARATOR,
75                                                       build=build["build"],
76                                                       name=file_name))
77             logging.info(
78                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
79
80             status = "failed"
81             try:
82                 response = get(url, stream=True)
83                 code = response.status_code
84                 if code != codes["OK"]:
85                     logging.warning(
86                         "Jenkins: {0}: {1}.".format(code, responses[code]))
87                     logging.info("Trying to download from Nexus:")
88                     spec.set_input_state(job, build["build"], "not found")
89                     if code == codes["not_found"]:
90                         release = re.search(REGEX_RELEASE, job).group(2)
91                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
92                             format(job=job, sep=SEPARATOR, build=build["build"],
93                                    name=file_name)
94                         url = "{url}/rls{release}/{dir}/{file}".\
95                             format(url=spec.environment["urls"]["URL[NEXUS]"],
96                                    release=release,
97                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
98                                    file=nexus_file_name)
99                         logging.info("Downloading the file '{0}' to '{1}' ...".
100                                      format(url, new_name))
101                         response = get(url, stream=True)
102                         code = response.status_code
103                         if code != codes["OK"]:
104                             logging.error(
105                                 "Nexus: {0}: {1}".format(code, responses[code]))
106                             spec.set_input_state(
107                                 job, build["build"], "not found")
108                             continue
109
110                 file_handle = open(new_name, "wb")
111                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
112                     if chunk:
113                         file_handle.write(chunk)
114                 file_handle.close()
115
116                 if spec.input["file-name"].endswith(".zip"):
117                     expected_length = None
118                     try:
119                         expected_length = int(response.
120                                               headers["Content-Length"])
121                         logging.debug("  Expected file size: {0}B".
122                                       format(expected_length))
123                     except KeyError:
124                         logging.debug("  No information about expected size.")
125
126                     real_length = getsize(new_name)
127                     logging.debug("  Downloaded size: {0}B".format(real_length))
128
129                     if expected_length:
130                         if real_length == expected_length:
131                             status = "downloaded"
132                             logging.info("{0}: {1}".format(code,
133                                                            responses[code]))
134                         else:
135                             logging.error("The file size differs from the "
136                                           "expected size.")
137                     else:
138                         status = "downloaded"
139                         logging.info("{0}: {1}".format(code, responses[code]))
140
141                 elif spec.input["file-name"].endswith(".gz"):
142                     rename(new_name, new_name[:-7])
143                     with open(new_name[:-7], 'r') as xml_file:
144                         with gzip.open(new_name, 'wb') as gz_file:
145                             gz_file.write(xml_file.read())
146                     new_name = new_name[:-7]
147                     status = "downloaded"
148                     logging.info("{0}: {1}".format(code, responses[code]))
149
150             except ConnectionError as err:
151                 logging.error("Not possible to connect to '{0}'.".format(url))
152                 logging.debug(err)
153             except HTTPError as err:
154                 logging.error("Invalid HTTP response from '{0}'.".format(url))
155                 logging.debug(err)
156             except TooManyRedirects as err:
157                 logging.error("Request exceeded the configured number "
158                               "of maximum re-directions.")
159                 logging.debug(err)
160             except Timeout as err:
161                 logging.error("Request timed out.")
162                 logging.debug(err)
163             except RequestException as err:
164                 logging.error("Unexpected HTTP request exception.")
165                 logging.debug(err)
166             except (IOError, ValueError, KeyError) as err:
167                 logging.error("Download failed.")
168                 logging.debug("Reason: {0}".format(err))
169
170             spec.set_input_state(job, build["build"], status)
171             spec.set_input_file_name(job, build["build"], new_name)
172
173             if status == "failed":
174                 logging.info("Removing the file '{0}'".format(new_name))
175                 try:
176                     remove(new_name)
177                 except OSError as err:
178                     logging.warning(str(err))
179                 spec.set_input_file_name(job, build["build"], None)
180
181     unzip_files(spec)
182
183
184 def unzip_files(spec):
185     """Unzip downloaded zip files
186
187     :param spec: Specification.
188     :type spec: Specification
189     :raises: PresentationError if the zip file does not exist or it is not a
190     zip file.
191     """
192
193     if spec.is_debug:
194         data_file = spec.debug["extract"]
195     else:
196         data_file = spec.input["extract"]
197
198     for job, builds in spec.builds.items():
199         for build in builds:
200             if build["status"] == "failed" or build["status"] == "not found":
201                 continue
202             try:
203                 status = "failed"
204                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
205                 file_name = join(build["file-name"])
206
207                 if build["status"] == "downloaded":
208                     logging.info("Unziping: '{0}' from '{1}'.".
209                                  format(data_file, file_name))
210                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
211                                                   SEPARATOR,
212                                                   data_file.split("/")[-1])
213                     try:
214                         if is_zipfile(file_name):
215                             with ZipFile(file_name, 'r') as zip_file:
216                                 zip_file.extract(data_file, directory)
217                             logging.info("Moving {0} to {1} ...".
218                                          format(join(directory, data_file),
219                                                 directory))
220                             move(join(directory, data_file), directory)
221                             logging.info("Renaming the file '{0}' to '{1}'".
222                                          format(join(directory,
223                                                      data_file.split("/")[-1]),
224                                                 new_name))
225                             rename(join(directory, data_file.split("/")[-1]),
226                                    new_name)
227                             spec.set_input_file_name(job, build["build"],
228                                                      new_name)
229                         status = "unzipped"
230                         spec.set_input_state(job, build["build"], status)
231                     except (BadZipfile, RuntimeError) as err:
232                         logging.error("Failed to unzip the file '{0}': {1}.".
233                                       format(file_name, str(err)))
234                     except OSError as err:
235                         logging.error("Failed to rename the file '{0}': {1}.".
236                                       format(data_file, str(err)))
237                     finally:
238                         if status == "failed":
239                             spec.set_input_file_name(job, build["build"], None)
240                 else:
241                     raise PresentationError("The file '{0}' does not exist or "
242                                             "it is not a zip file".
243                                             format(file_name))
244
245                 spec.set_input_state(job, build["build"], status)
246
247             except KeyError:
248                 pass

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.