From ed0258a440cfad7023d643f717ab78ac568dc59b Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Mon, 4 Nov 2019 09:27:28 +0000 Subject: [PATCH] Python3: PIP requirement - Install via PIP3 - Use virtualenv Python3 interpret Signed-off-by: Peter Mikus Change-Id: Iad22b344b418a1d91774d23d5ea17bfd48d394b3 --- pylint.cfg | 9 +- requirements.txt | 118 ++++++++++++--------- resources/libraries/bash/function/common.sh | 11 +- resources/libraries/python/SetupFramework.py | 5 +- resources/tools/doc_gen/run_doc.sh | 2 - resources/tools/doc_gen/src/requirements.txt | 3 - resources/tools/presentation/run_cpta.sh | 6 +- resources/tools/presentation/run_report.sh | 6 +- .../inventories/lf_inventory/group_vars/all.yaml | 4 +- .../sample_inventory/group_vars/all.yaml | 4 +- .../roles/cobbler/files/etc/cobbler/dhcp.template | 2 +- .../kickstarts/ubuntu-18.04.2-server-x86_64.seed | 2 +- .../ansible/roles/common/tasks/main.yaml | 66 +++++++++--- tox.ini | 4 +- 14 files changed, 150 insertions(+), 92 deletions(-) delete mode 100644 resources/tools/doc_gen/src/requirements.txt diff --git a/pylint.cfg b/pylint.cfg index 69668ed79c..5ae4ed551b 100644 --- a/pylint.cfg +++ b/pylint.cfg @@ -12,7 +12,7 @@ profile=no # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS +ignore=CVS, conf.py # Pickle collected data for later comparisons. persistent=yes @@ -282,3 +282,10 @@ max-public-methods=60 # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=Exception + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=fstr \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3fa2a9d879..24263c7aae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,57 +1,79 @@ # DISCLAIMER: BEFORE EDITING THIS FILE! # -# This file has two closely related consequences. -# The common part is that this file defines -# the content of virtual Python environment -# used when Robot tests are running. -# The difference is in which machine the environment is created on. -# For the Jenkins executor machines, -# editing this file has no surprising consequences. +# This file has two closely related consequences. The common part is that this +# file defines the content of virtual Python environment used when Robot tests +# are running. # -# But the environment is also being created -# on virtual machines created by VIRL, which do have specific restrictions. -# Complete package and setting snapshot (for both VIRL and Physical Testbed), -# with network being limited in case of VIRL to local only. -# This implies pip currently being set to site-packages only -# for both VIRL and Physical Testbed. -# So if the "virl image" applied on the machine -# does not reflect the edits, some installations might fail. -# Even if the failure may not directly impact execution, -# this leads into inconsistency between installed environment -# and the code base which may lead into fatal error or uncaught exceptions. +# But the virtual environment is also being created on containerized +# environment created by either Nomad cluster (Jenkins, vpp_device) or during +# execution on SUT machines, which might have specific restrictions. # -# The "virl image" is built from files such as -# resources/tools/disk-image-builder/ubuntu/lists/ubuntu-16.04.1_2017-10-21_2.0/pip-requirements.txt -# but the build is not automated, -# the new "virl image" has to be built and applied manually -# before your edit can pass. -# This also means, incompatible virl images -# would make existing tests fail until your edit is merged. +# Complete package and setting snapshot (for all container images), +# with network being limited to local only. This implies pip currently being +# set to site-packages only. So if the "container image" applied on the machine +# does not reflect the edits, some installations might fail. +# Even if the failure may not directly impact execution, this leads into +# inconsistency between installed environment and the code base which may lead +# into fatal error or uncaught exceptions. # -# TODO: Automate the virtualenv download and distribution -# on all types of testbeds prior the test execution. -# TODO: Figure out a way to verify edits to this file automatically. - -# TODO: Split into directly needed packages and their dependencies. -aenum==2.1.2 -docopt==0.6.2 # for pykwalify -ecdsa==0.13 -enum34==1.1.2 -ipaddress==1.0.16 -paramiko==1.16.0 -pexpect==4.6.0 +# CSIT Core dependencies +ecdsa==0.13.3 +paramiko==2.6.0 pycrypto==2.6.1 -pykwalify==1.5.0 -pypcap==1.1.5 -python-dateutil==2.4.2 # for pykwalify -PyYAML==3.11 -requests==2.9.1 -robotframework==2.9.2 -scapy==2.3.1 -scp==0.10.2 -six==1.12.0 # for python-dateutil +pypcap==1.2.3 +PyYAML==5.1.1 +requests==2.22.0 +robotframework==3.1.2 +scapy==2.4.3 +scp==0.13.2 -# The following is only there for PLRsearch. +# Bootstraping dependencies +ansible==2.7.8 + +# PLRsearch dependencies dill==0.2.8.2 -numpy==1.14.5 +numpy==1.17.3 scipy==1.1.0 + +# PAL/DOC dependencies +hdrhistogram==0.6.1 +pandas==0.25.3 +plotly==4.1.1 +PTable==0.9.2 +Sphinx==2.2.1 +sphinx-rtd-theme==0.4.0 +sphinxcontrib-programoutput==0.15 +sphinxcontrib-robotdoc==0.11.0 + +# PIP freeze dependencies +alabaster==0.7.12 +Babel==2.7.0 +bcrypt==3.1.7 +certifi==2019.9.11 +cffi==1.13.2 +chardet==3.0.4 +cryptography==2.8 +docutils==0.15.2 +future==0.18.2 +idna==2.8 +imagesize==1.1.0 +Jinja2==2.10.3 +MarkupSafe==1.1.1 +packaging==19.2 +pbr==5.4.3 +pycparser==2.19 +Pygments==2.4.2 +PyNaCl==1.3.0 +pyparsing==2.4.4 +python-dateutil==2.8.1 +pytz==2019.3 +retrying==1.3.3 +six==1.13.0 +snowballstemmer==2.0.0 +sphinxcontrib-applehelp==1.0.1 +sphinxcontrib-devhelp==1.0.1 +sphinxcontrib-htmlhelp==1.0.2 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.2 +sphinxcontrib-serializinghtml==1.1.3 +urllib3==1.25.6 diff --git a/resources/libraries/bash/function/common.sh b/resources/libraries/bash/function/common.sh index 1ef7c93da3..3a7a0fbf7d 100644 --- a/resources/libraries/bash/function/common.sh +++ b/resources/libraries/bash/function/common.sh @@ -113,16 +113,16 @@ function activate_virtualenv () { env_dir="${root_path}/env" req_path=${2-$CSIT_DIR/requirements.txt} rm -rf "${env_dir}" || die "Failed to clean previous virtualenv." - pip install --upgrade virtualenv || { + pip3 install --upgrade virtualenv || { die "Virtualenv package install failed." } - virtualenv "${env_dir}" || { - die "Virtualenv creation failed." + virtualenv -p $(which python3) "${env_dir}" || { + die "Virtualenv creation for $(which python) failed." } set +u source "${env_dir}/bin/activate" || die "Virtualenv activation failed." set -u - pip install --upgrade -r "${req_path}" || { + pip3 install --upgrade -r "${req_path}" || { die "Requirements installation failed." } # Most CSIT Python scripts assume PYTHONPATH is set and exported. @@ -637,7 +637,8 @@ function run_pybot () { pushd "${CSIT_DIR}" || die "Change directory operation failed." set +e - pybot "${all_options[@]}" "${GENERATED_DIR}/tests/" + python -V + robot "${all_options[@]}" "${GENERATED_DIR}/tests/" PYBOT_EXIT_STATUS="$?" set -e diff --git a/resources/libraries/python/SetupFramework.py b/resources/libraries/python/SetupFramework.py index 760d958c54..3f74057832 100644 --- a/resources/libraries/python/SetupFramework.py +++ b/resources/libraries/python/SetupFramework.py @@ -111,8 +111,9 @@ def create_env_directory_at_node(node): .format(host)) exec_cmd_no_error( node, 'cd {0} && rm -rf env' - ' && virtualenv --system-site-packages --never-download env' - ' && source env/bin/activate && pip install -r requirements.txt' + ' && virtualenv -p $(which python3) ' + '--system-site-packages --never-download env' + ' && source env/bin/activate && pip3 install -r requirements.txt' .format(con.REMOTE_FW_DIR), timeout=100, include_reason=True, message="Failed install at node {host}".format(host=host)) logger.console('Virtualenv setup on {0} done.'.format(host)) diff --git a/resources/tools/doc_gen/run_doc.sh b/resources/tools/doc_gen/run_doc.sh index 523b4ae505..36535df031 100755 --- a/resources/tools/doc_gen/run_doc.sh +++ b/resources/tools/doc_gen/run_doc.sh @@ -33,8 +33,6 @@ virtualenv ${WORKING_DIR}/env # Install CSIT requirements: pip install -r ../../../requirements.txt -# Install Sphinx: -pip install -r ${WORKING_DIR}/requirements.txt export PYTHONPATH=`pwd` diff --git a/resources/tools/doc_gen/src/requirements.txt b/resources/tools/doc_gen/src/requirements.txt deleted file mode 100644 index 2cf7cd9687..0000000000 --- a/resources/tools/doc_gen/src/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -Sphinx==1.5.5 -sphinxcontrib_robotdoc -sphinx-rtd-theme \ No newline at end of file diff --git a/resources/tools/presentation/run_cpta.sh b/resources/tools/presentation/run_cpta.sh index 954f17d53c..a60c1cc7a9 100755 --- a/resources/tools/presentation/run_cpta.sh +++ b/resources/tools/presentation/run_cpta.sh @@ -16,11 +16,11 @@ sudo apt-get -y install libxml2 libxml2-dev libxslt-dev build-essential \ mkdir ${DIR[WORKING]} # Create virtual environment -virtualenv ${DIR[WORKING]}/env -. ${DIR[WORKING]}/env/bin/activate +virtualenv -p $(which python3) ${DIR[WORKING]}/env +source ${DIR[WORKING]}/env/bin/activate # Install python dependencies: -pip install -r requirements.txt +pip3 install -r requirements.txt export PYTHONPATH=`pwd` diff --git a/resources/tools/presentation/run_report.sh b/resources/tools/presentation/run_report.sh index 24b6c21a70..56c671a8bb 100755 --- a/resources/tools/presentation/run_report.sh +++ b/resources/tools/presentation/run_report.sh @@ -27,11 +27,11 @@ fi mkdir ${DIR[WORKING]} # Create virtual environment -virtualenv ${DIR[WORKING]}/env -. ${DIR[WORKING]}/env/bin/activate +virtualenv -p $(which python3) ${DIR[WORKING]}/env +source ${DIR[WORKING]}/env/bin/activate # Install python dependencies: -pip install -r requirements.txt +pip3 install -r requirements.txt export PYTHONPATH=`pwd` diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml index f53584a939..f2e76a729d 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/group_vars/all.yaml @@ -1,8 +1,8 @@ --- # file: lf_inventory/group_vars/all.yaml -# General variables -ansible_python_interpreter: '/usr/bin/python2.7' +# Ansible interpreter (for PIP) +ansible_python_interpreter: 'python3' # provision via cobbler provision_enabled: False # name_servers_search is used in /etc/hosts file on target machine. diff --git a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml index 484e6ddb4b..d02746cb4f 100644 --- a/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml +++ b/resources/tools/testbed-setup/ansible/inventories/sample_inventory/group_vars/all.yaml @@ -1,8 +1,8 @@ --- # file: group_vars/all.yaml -# General variables -ansible_python_interpreter: '/usr/bin/python2.7' +# Ansible interpreter (for PIP) +ansible_python_interpreter: 'python3' # provision via cobbler provision_enabled: True # name_servers_search is used in /etc/hosts file on target machine. diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template index cf2fbdfe34..965562ae0a 100644 --- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/etc/cobbler/dhcp.template @@ -21,7 +21,7 @@ option pxe-system-type code 93 = unsigned integer 16; subnet 10.30.51.0 netmask 255.255.255.0 { option routers 10.30.51.1; option domain-name "linuxfoundation.org"; - option domain-name-servers 199.204.44.24, 199.204.47.54; + option domain-name-servers 1.1.1.1, 8.8.8.8; option subnet-mask 255.255.255.0; range dynamic-bootp 10.30.51.2 10.30.51.254; default-lease-time 600; diff --git a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed index 250701232c..3827562c06 100644 --- a/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed +++ b/resources/tools/testbed-setup/ansible/roles/cobbler/files/var/lib/cobbler/kickstarts/ubuntu-18.04.2-server-x86_64.seed @@ -114,7 +114,7 @@ d-i user-setup/encrypt-home boolean false ### Package selection tasksel tasksel/first multiselect ubuntu-server # Individual additional packages to install -d-i pkgsel/include string openssh-server python2.7 +d-i pkgsel/include string openssh-server python2.7 python3.6 # Whether to upgrade packages after debootstrap. # Allowed values: none, safe-upgrade, full-upgrade #d-i pkgsel/upgrade select none diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml index 3e7e5a00fa..84358bf863 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml @@ -70,25 +70,57 @@ - name: Install CSIT PIP requirements pip: name: - - 'docopt==0.6.2' - - 'ecdsa==0.13' - - 'enum34==1.1.2' - - 'ipaddress==1.0.16' - - 'paramiko==1.16.0' - - 'pexpect==4.6.0' + - 'ecdsa==0.13.3' + - 'paramiko==2.6.0' - 'pycrypto==2.6.1' - - 'pykwalify==1.5.0' - - 'pypcap==1.1.5' - - 'python-dateutil==2.4.2' - - 'PyYAML==3.11' - - 'requests==2.9.1' - - 'robotframework==2.9.2' - - 'scapy==2.3.1' - - 'scp==0.10.2' - - 'six==1.12.0' + - 'pypcap==1.2.3' + - 'PyYAML==5.1.1' + - 'requests==2.22.0' + - 'robotframework==3.1.2' + - 'scapy==2.4.3' + - 'scp==0.13.2' + - 'ansible==2.7.8' - 'dill==0.2.8.2' - - 'numpy==1.14.5' - - 'aenum==2.1.2' + - 'numpy==1.17.3' + - 'hdrhistogram==0.6.1' + - 'pandas==0.25.3' + - 'plotly==4.1.1' + - 'PTable==0.9.2' + - 'Sphinx==2.2.1' + - 'sphinx-rtd-theme==0.4.0' + - 'sphinxcontrib-programoutput==0.15' + - 'sphinxcontrib-robotdoc==0.11.0' + - 'alabaster==0.7.12' + - 'Babel==2.7.0' + - 'bcrypt==3.1.7' + - 'certifi==2019.9.11' + - 'cffi==1.13.2' + - 'chardet==3.0.4' + - 'cryptography==2.8' + - 'docutils==0.15.2' + - 'future==0.18.2' + - 'idna==2.8' + - 'imagesize==1.1.0' + - 'Jinja2==2.10.3' + - 'MarkupSafe==1.1.1' + - 'packaging==19.2' + - 'pbr==5.4.3' + - 'pycparser==2.19' + - 'Pygments==2.4.2' + - 'PyNaCl==1.3.0' + - 'pyparsing==2.4.4' + - 'python-dateutil==2.8.1' + - 'pytz==2019.3' + - 'retrying==1.3.3' + - 'six==1.13.0' + - 'snowballstemmer==2.0.0' + - 'sphinxcontrib-applehelp==1.0.1' + - 'sphinxcontrib-devhelp==1.0.1' + - 'sphinxcontrib-htmlhelp==1.0.2' + - 'sphinxcontrib-jsmath==1.0.1' + - 'sphinxcontrib-qthelp==1.0.2' + - 'sphinxcontrib-serializinghtml==1.1.3' + - 'urllib3==1.25.6' tags: install-pip - name: Install CSIT PIP requirements - SciPy workaround diff --git a/tox.ini b/tox.ini index b235bf3ab8..4617e73e54 100644 --- a/tox.ini +++ b/tox.ini @@ -36,9 +36,9 @@ checker_dir = ./resources/libraries/bash/entry/check # TODO: Tox prints various warnings. Figure them out and fix them. [testenv:pylint] -basepython = python2 +basepython = python3 deps = - pylint==1.5.4 + pylint==2.4.3 -r ./requirements.txt whitelist_externals = /bin/bash setenv = PYTHONPATH = {toxinidir} -- 2.16.6