CSIT-891: Add data sources for 1801 report
[csit.git] / resources / tools / presentation / input_data_files.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Inputs
15 Download all data.
16 """
17
18 import re
19
20 import logging
21
22 from os import rename, remove
23 from os.path import join, getsize
24 from shutil import move
25 from zipfile import ZipFile, is_zipfile, BadZipfile
26 from httplib import responses
27 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
28     HTTPError, ConnectionError
29
30 from errors import PresentationError
31
32
33 # Chunk size used for file download
34 CHUNK_SIZE = 512
35
36 # Separator used in file names
37 SEPARATOR = "__"
38
39 REGEX_RELEASE = re.compile(r'(\D*)(\d{4})(\D*)')
40
41
42 def download_data_files(spec):
43     """Download all data specified in the specification file in the section
44     type: input --> builds.
45
46     :param spec: Specification.
47     :type spec: Specification
48     :raises: PresentationError if there is no url defined for the job.
49     """
50
51     for job, builds in spec.builds.items():
52         for build in builds:
53             if job.startswith("csit-"):
54                 url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
55             elif job.startswith("hc2vpp-"):
56                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
57             else:
58                 raise PresentationError("No url defined for the job '{}'.".
59                                         format(job))
60             file_name = spec.input["file-name"]
61             full_name = spec.input["download-path"].\
62                 format(job=job, build=build["build"], filename=file_name)
63             url = "{0}/{1}".format(url, full_name)
64             new_name = join(
65                 spec.environment["paths"]["DIR[WORKING,DATA]"],
66                 "{job}{sep}{build}{sep}{name}".format(job=job,
67                                                       sep=SEPARATOR,
68                                                       build=build["build"],
69                                                       name=file_name))
70             logging.info(
71                 "Downloading the file '{0}' to '{1}' ...".format(url, new_name))
72
73             status = "failed"
74             try:
75                 response = get(url, stream=True)
76                 code = response.status_code
77                 if code != codes["OK"]:
78                     logging.warning(
79                         "Jenkins: {0}: {1}.".format(code, responses[code]))
80                     logging.info("Trying to download from Nexus:")
81                     spec.set_input_state(job, build["build"], "not found")
82                     if code == codes["not_found"]:
83                         release = re.search(REGEX_RELEASE, job).group(2)
84                         nexus_file_name = "{job}{sep}{build}{sep}{name}".\
85                             format(job=job, sep=SEPARATOR, build=build["build"],
86                                    name=file_name)
87                         url = "{url}/rls{release}/{dir}/{file}".\
88                             format(url=spec.environment["urls"]["URL[NEXUS]"],
89                                    release=release,
90                                    dir=spec.environment["urls"]["DIR[NEXUS]"],
91                                    file=nexus_file_name)
92                         logging.info("Downloading the file '{0}' to '{1}' ...".
93                                      format(url, new_name))
94                         response = get(url, stream=True)
95                         code = response.status_code
96                         if code != codes["OK"]:
97                             logging.error(
98                                 "Nexus: {0}: {1}".format(code, responses[code]))
99                             spec.set_input_state(
100                                 job, build["build"], "not found")
101                             continue
102
103                 file_handle = open(new_name, "wb")
104                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
105                     if chunk:
106                         file_handle.write(chunk)
107                 file_handle.close()
108
109                 expected_length = None
110                 try:
111                     expected_length = int(response.headers["Content-Length"])
112                     logging.debug("  Expected file size: {0}B".
113                                   format(expected_length))
114                 except KeyError:
115                     logging.debug("  No information about expected size.")
116
117                 real_length = getsize(new_name)
118                 logging.debug("  Downloaded size: {0}B".format(real_length))
119
120                 if expected_length:
121                     if real_length == expected_length:
122                         status = "downloaded"
123                         logging.info("{0}: {1}".format(code, responses[code]))
124                     else:
125                         logging.error("The file size differs from the expected "
126                                       "size.")
127                 else:
128                     status = "downloaded"
129                     logging.info("{0}: {1}".format(code, responses[code]))
130
131             except ConnectionError as err:
132                 logging.error("Not possible to connect to '{0}'.".format(url))
133                 logging.debug(err)
134             except HTTPError as err:
135                 logging.error("Invalid HTTP response from '{0}'.".format(url))
136                 logging.debug(err)
137             except TooManyRedirects as err:
138                 logging.error("Request exceeded the configured number "
139                               "of maximum re-directions.")
140                 logging.debug(err)
141             except Timeout as err:
142                 logging.error("Request timed out.")
143                 logging.debug(err)
144             except RequestException as err:
145                 logging.error("Unexpected HTTP request exception.")
146                 logging.debug(err)
147             except (IOError, ValueError, KeyError) as err:
148                 logging.error("Download failed.")
149                 logging.debug("Reason: {0}".format(err))
150
151             spec.set_input_state(job, build["build"], status)
152             spec.set_input_file_name(job, build["build"], new_name)
153
154             if status == "failed":
155                 logging.info("Removing the file '{0}'".format(new_name))
156                 try:
157                     remove(new_name)
158                 except OSError as err:
159                     logging.warning(str(err))
160                 spec.set_input_file_name(job, build["build"], None)
161
162     unzip_files(spec)
163
164
165 def unzip_files(spec):
166     """Unzip downloaded zip files
167
168     :param spec: Specification.
169     :type spec: Specification
170     :raises: PresentationError if the zip file does not exist or it is not a
171     zip file.
172     """
173
174     if spec.is_debug:
175         data_file = spec.debug["extract"]
176     else:
177         data_file = spec.input["extract"]
178
179     for job, builds in spec.builds.items():
180         for build in builds:
181             if build["status"] == "failed" or build["status"] == "not found":
182                 continue
183             try:
184                 status = "failed"
185                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
186                 file_name = join(build["file-name"])
187
188                 if build["status"] == "downloaded" and is_zipfile(file_name):
189                     logging.info("Unziping: '{0}' from '{1}'.".
190                                  format(data_file, file_name))
191                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
192                                                   SEPARATOR,
193                                                   data_file.split("/")[-1])
194                     try:
195                         with ZipFile(file_name, 'r') as zip_file:
196                             zip_file.extract(data_file, directory)
197                         logging.info("Moving {0} to {1} ...".
198                                      format(join(directory, data_file),
199                                             directory))
200                         move(join(directory, data_file), directory)
201                         logging.info("Renaming the file '{0}' to '{1}'".
202                                      format(join(directory,
203                                                  data_file.split("/")[-1]),
204                                             new_name))
205                         rename(join(directory, data_file.split("/")[-1]),
206                                new_name)
207                         status = "unzipped"
208                         spec.set_input_state(job, build["build"], status)
209                         spec.set_input_file_name(job, build["build"],
210                                                    new_name)
211                     except (BadZipfile, RuntimeError) as err:
212                         logging.error("Failed to unzip the file '{0}': {1}.".
213                                       format(file_name, str(err)))
214                     except OSError as err:
215                         logging.error("Failed to rename the file '{0}': {1}.".
216                                       format(data_file, str(err)))
217                     finally:
218                         if status == "failed":
219                             spec.set_input_file_name(job, build["build"],
220                                                        None)
221                 else:
222                     raise PresentationError("The file '{0}' does not exist or "
223                                             "it is not a zip file".
224                                             format(file_name))
225
226                 spec.set_input_state(job, build["build"], status)
227
228             except KeyError:
229                 pass

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.