CSIT-755: Presentation and analytics layer
[csit.git] / resources / tools / presentation / input_data_files.py
1 # Copyright (c) 2017 Cisco and/or its affiliates.
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at:
5 #
6 #     http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13
14 """Inputs
15 Download all data.
16 """
17
18 import logging
19
20 from os import rename, remove
21 from os.path import join, getsize
22 from shutil import move
23 from zipfile import ZipFile, is_zipfile, BadZipfile
24 from httplib import responses
25 from requests import get, codes, RequestException, Timeout, TooManyRedirects, \
26     HTTPError, ConnectionError
27
28 from errors import PresentationError
29
30
31 # Chunk size used for file download
32 CHUNK_SIZE = 512
33
34 # Separator used in file names
35 SEPARATOR = "__"
36
37
38 def download_data_files(spec):
39     """Download all data specified in the specification file in the section
40     type: input --> builds.
41
42     :param spec: Specification.
43     :type spec: Specification
44     :raises: PresentationError if there is no url defined for the job.
45     """
46
47     for job, builds in spec.builds.items():
48         for build in builds:
49             if job.startswith("csit-"):
50                 url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
51             elif job.startswith("hc2vpp-"):
52                 url = spec.environment["urls"]["URL[JENKINS,HC]"]
53             else:
54                 raise PresentationError("No url defined for the job '{}'.".
55                                         format(job))
56             file_name = spec.input["file-name"]
57             full_name = spec.input["download-path"].\
58                 format(job=job, build=build["build"], filename=file_name)
59             url = "{0}/{1}".format(url, full_name)
60             new_name = join(
61                 spec.environment["paths"]["DIR[WORKING,DATA]"],
62                 "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR,
63                                                       build=build["build"],
64                                                       name=file_name))
65
66             logging.info("Downloading the file '{0}' to '{1}'.".
67                          format(url, new_name))
68
69             status = "failed"
70             try:
71                 response = get(url, stream=True)
72                 code = response.status_code
73                 if code != codes["OK"]:
74                     logging.error("{0}: {1}".format(code, responses[code]))
75                     spec.set_input_state(job, build["build"], "not found")
76                     continue
77
78                 file_handle = open(new_name, "wb")
79                 for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
80                     if chunk:
81                         file_handle.write(chunk)
82                 file_handle.close()
83
84                 expected_length = None
85                 try:
86                     expected_length = int(response.headers["Content-Length"])
87                     logging.debug("  Expected file size: {0}B".
88                                   format(expected_length))
89                 except KeyError:
90                     logging.debug("  No information about expected size.")
91
92                 real_length = getsize(new_name)
93                 logging.debug("  Downloaded size: {0}B".format(real_length))
94
95                 if expected_length:
96                     if real_length == expected_length:
97                         status = "downloaded"
98                         logging.info("{0}: {1}".format(code, responses[code]))
99                     else:
100                         logging.error("The file size differs from the expected "
101                                       "size.")
102                 else:
103                     status = "downloaded"
104                     logging.info("{0}: {1}".format(code, responses[code]))
105
106             except ConnectionError as err:
107                 logging.error("Not possible to connect to '{0}'.".format(url))
108                 logging.debug(err)
109             except HTTPError as err:
110                 logging.error("Invalid HTTP response from '{0}'.".format(url))
111                 logging.debug(err)
112             except TooManyRedirects as err:
113                 logging.error("Request exceeded the configured number "
114                               "of maximum re-directions.")
115                 logging.debug(err)
116             except Timeout as err:
117                 logging.error("Request timed out.")
118                 logging.debug(err)
119             except RequestException as err:
120                 logging.error("Unexpected HTTP request exception.")
121                 logging.debug(err)
122             except (IOError, ValueError, KeyError) as err:
123                 logging.error("Download failed.")
124                 logging.debug("Reason: {0}".format(err))
125
126             spec.set_input_state(job, build["build"], status)
127             spec.set_input_file_name(job, build["build"], new_name)
128
129             if status == "failed":
130                 logging.info("Removing the file '{0}'".format(new_name))
131                 try:
132                     remove(new_name)
133                 except OSError as err:
134                     logging.warning(str(err))
135                 spec.set_input_file_name(job, build["build"], None)
136
137     unzip_files(spec)
138
139
140 def unzip_files(spec):
141     """Unzip downloaded zip files
142
143     :param spec: Specification.
144     :type spec: Specification
145     :raises: PresentationError if the zip file does not exist or it is not a
146     zip file.
147     """
148
149     if spec.is_debug:
150         data_file = spec.debug["extract"]
151     else:
152         data_file = spec.input["extract"]
153
154     for job, builds in spec.builds.items():
155         for build in builds:
156             if build["status"] == "failed" or build["status"] == "not found":
157                 continue
158             try:
159                 status = "failed"
160                 directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
161                 file_name = join(build["file-name"])
162
163                 if build["status"] == "downloaded" and is_zipfile(file_name):
164                     logging.info("Unziping: '{0}' from '{1}'.".
165                                  format(data_file, file_name))
166                     new_name = "{0}{1}{2}".format(file_name.rsplit('.')[-2],
167                                                   SEPARATOR,
168                                                   data_file.split("/")[-1])
169                     try:
170                         with ZipFile(file_name, 'r') as zip_file:
171                             zip_file.extract(data_file, directory)
172                         logging.info("Moving {0} to {1} ...".
173                                      format(join(directory, data_file),
174                                             directory))
175                         move(join(directory, data_file), directory)
176                         logging.info("Renaming the file '{0}' to '{1}'".
177                                      format(join(directory,
178                                                  data_file.split("/")[-1]),
179                                             new_name))
180                         rename(join(directory, data_file.split("/")[-1]),
181                                new_name)
182                         status = "unzipped"
183                         spec.set_input_state(job, build["build"], status)
184                         spec.set_input_file_name(job, build["build"],
185                                                    new_name)
186                     except (BadZipfile, RuntimeError) as err:
187                         logging.error("Failed to unzip the file '{0}': {1}.".
188                                       format(file_name, str(err)))
189                     except OSError as err:
190                         logging.error("Failed to rename the file '{0}': {1}.".
191                                       format(data_file, str(err)))
192                     finally:
193                         if status == "failed":
194                             spec.set_input_file_name(job, build["build"],
195                                                        None)
196                 else:
197                     raise PresentationError("The file '{0}' does not exist or "
198                                             "it is not a zip file".
199                                             format(file_name))
200
201                 spec.set_input_state(job, build["build"], status)
202
203             except KeyError:
204                 pass

©2016 FD.io a Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation. Linux is a registered trademark of Linus Torvalds.
Please see our privacy policy and terms of use.