From bf3ce71ec5074eb30a866ea8b6e01aad03d58e64 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Thu, 14 Mar 2019 08:02:56 +0000 Subject: [PATCH] Add PAPI dependencies + Various small structure optmizations Change-Id: I1aac9353f72436de53797fc734bef498905c8189 Signed-off-by: Peter Mikus --- .../csit-sut-dcr/ubuntu_1804_amd64/Dockerfile | 14 ++++-- ...es-arm.list => apt-sources_bionic_aarch64.list} | 0 ...sources.list => apt-sources_bionic_x86_64.list} | 0 .../ansible/roles/common/tasks/main.yaml | 5 +- .../tasks/{ubuntu.yaml => ubuntu_bionic.yaml} | 15 ++---- .../ansible/roles/sut/tasks/main.yaml | 5 +- .../sut/tasks/{ubuntu.yaml => ubuntu_bionic.yaml} | 5 +- .../testbed-setup/ansible/roles/tg/tasks/main.yaml | 5 +- .../tg/tasks/{ubuntu.yaml => ubuntu_bionic.yaml} | 2 +- .../ansible/roles/tg_sut/files/requirements.txt | 56 ---------------------- .../ansible/roles/tg_sut/tasks/main.yaml | 49 ++++++++++++------- .../tasks/{ubuntu.yaml => ubuntu_bionic.yaml} | 3 +- 12 files changed, 57 insertions(+), 102 deletions(-) rename resources/tools/testbed-setup/ansible/roles/common/files/{apt-sources-arm.list => apt-sources_bionic_aarch64.list} (100%) rename resources/tools/testbed-setup/ansible/roles/common/files/{apt-sources.list => apt-sources_bionic_x86_64.list} (100%) rename resources/tools/testbed-setup/ansible/roles/common/tasks/{ubuntu.yaml => ubuntu_bionic.yaml} (52%) rename resources/tools/testbed-setup/ansible/roles/sut/tasks/{ubuntu.yaml => ubuntu_bionic.yaml} (77%) rename resources/tools/testbed-setup/ansible/roles/tg/tasks/{ubuntu.yaml => ubuntu_bionic.yaml} (81%) delete mode 100644 resources/tools/testbed-setup/ansible/roles/tg_sut/files/requirements.txt rename resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/{ubuntu.yaml => ubuntu_bionic.yaml} (97%) diff --git a/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile b/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile index c7aa3363cf..a681c21e4f 100644 --- a/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile +++ b/resources/tools/disk-image-builder/csit-sut-dcr/ubuntu_1804_amd64/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2018 Cisco and/or its affiliates. +# Copyright (c) 2019 Cisco and/or its affiliates. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -14,9 +14,8 @@ FROM ubuntu:bionic-20180821 MAINTAINER csit-dev -LABEL Description="CSIT vpp-device ubuntu 18.04 baseline image" -LABEL Vendor="cisco.com" -LABEL Version="0.4" +LABEL Description="CSIT vpp-device ubuntu 18.04 baseline image (x86)" +LABEL Version="0.6" # Setup the environment ENV DEBIAN_FRONTEND=noninteractive @@ -82,6 +81,7 @@ RUN apt-get -q update \ libnuma1 \ python-cffi \ python-enum34 \ + python3-cffi \ && rm -rf /var/lib/apt/lists/* # Configure locales @@ -97,7 +97,7 @@ RUN mkdir -p /tmp/dumps \ && mkdir -p /var/cache/vpp/python \ && mkdir -p /var/run/sshd -# PIP pre-cache +# CSIT PIP pre-cache RUN pip install \ docopt==0.6.2 \ ecdsa==0.13 \ @@ -119,6 +119,10 @@ RUN pip install \ numpy==1.14.5 \ scipy==1.1.0 +# VPP PIP pre-cache +RUN pip install \ + aenum + # SSH settings RUN echo 'root:Csit1234' | chpasswd \ && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \ diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources-arm.list b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_aarch64.list similarity index 100% rename from resources/tools/testbed-setup/ansible/roles/common/files/apt-sources-arm.list rename to resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_aarch64.list diff --git a/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources.list b/resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_x86_64.list similarity index 100% rename from resources/tools/testbed-setup/ansible/roles/common/files/apt-sources.list rename to resources/tools/testbed-setup/ansible/roles/common/files/apt-sources_bionic_x86_64.list diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml index 6844358eca..d7355883e0 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/main.yaml @@ -9,9 +9,8 @@ with_dict: "{{ proxy_env }}" when: proxy_env is defined -- name: Ubuntu specific - import_tasks: ubuntu.yaml - when: ansible_distribution|lower == 'ubuntu' +- name: Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' - name: Set hostname hostname: diff --git a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml similarity index 52% rename from resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu.yaml rename to resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml index ae606cd494..d80eb5edac 100644 --- a/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu.yaml +++ b/resources/tools/testbed-setup/ansible/roles/common/tasks/ubuntu_bionic.yaml @@ -1,24 +1,17 @@ --- -# file: roles/common/tasks/ubuntu.yaml +# file: roles/common/tasks/ubuntu_bionic.yaml -- name: Copy aarch64 apt sources file +- name: Copy apt sources file template: - src: 'files/apt-sources-arm.list' + src: 'files/apt-sources_{{ ansible_distribution_release }}_{{ ansible_machine }}.list' dest: '/etc/apt/sources.list' tags: copy-apt-sources - when: ansible_machine == 'aarch64' - -- name: Copy x86_64 apt sources file - template: - src: 'files/apt-sources.list' - dest: '/etc/apt/sources.list' - tags: copy-apt-sources - when: ansible_machine == 'x86_64' - name: Install CSIT dependencies apt: name: '{{ item }}' state: 'present' + cache_valid_time: 3600 install_recommends: False with_items: - 'python-apt' diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml index f5a2dd86ad..90990b6c3b 100644 --- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/main.yaml @@ -1,9 +1,8 @@ --- # file: roles/sut/tasks/main.yaml -- name: Ubuntu specific - import_tasks: ubuntu.yaml - when: ansible_distribution|lower == 'ubuntu' +- name: Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' - name: Taishan aarch64 specific import_tasks: taishan_workaround.yaml diff --git a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml similarity index 77% rename from resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu.yaml rename to resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml index 69ca52916e..7ed9daed22 100644 --- a/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu.yaml +++ b/resources/tools/testbed-setup/ansible/roles/sut/tasks/ubuntu_bionic.yaml @@ -1,5 +1,5 @@ --- -# file: roles/sut/tasks/ubuntu.yaml +# file: roles/sut/tasks/ubuntu_bionic.yaml - name: Install CSIT dependencies apt: @@ -20,4 +20,7 @@ - 'openjdk-8-jdk' - 'libpixman-1-dev' - 'python-cffi' + - 'python-cffi-backend' + - 'python3-cffi' + - 'python3-cffi-backend' tags: install-csit-dependencies diff --git a/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml index 9fa19b5b0f..f3f5d85aa1 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg/tasks/main.yaml @@ -1,6 +1,5 @@ --- # file: roles/tg/tasks/main.yaml -- name: Ubuntu specific - import_tasks: ubuntu.yaml - when: ansible_distribution|lower == 'ubuntu' +- name: Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' diff --git a/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml similarity index 81% rename from resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu.yaml rename to resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml index acb4f9802e..f60aaeec64 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg/tasks/ubuntu_bionic.yaml @@ -1,5 +1,5 @@ --- -# file: roles/tg/tasks/ubuntu.yaml +# file: roles/tg/tasks/ubuntu_bionic.yaml - name: Install CSIT dependencies apt: diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/requirements.txt b/resources/tools/testbed-setup/ansible/roles/tg_sut/files/requirements.txt deleted file mode 100644 index 0ce17e2b47..0000000000 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/files/requirements.txt +++ /dev/null @@ -1,56 +0,0 @@ -# DISCLAIMER: BEFORE EDITING THIS FILE! -# -# This file has two closely related consequences. -# The common part is that this file defines -# the content of virtual Python environment -# used when Robot tests are running. -# The difference is in which machine the environment is created on. -# For the Jenkins executor machines, -# editing this file has no surprising consequences. -# -# But the environment is also being created -# on virtual machines created by VIRL, which do have specific restrictions. -# Complete package and setting snapshot (for both VIRL and Physical Testbed), -# with network being limited in case of VIRL to local only. -# This implies pip currently being set to site-packages only -# for both VIRL and Physical Testbed. -# So if the "virl image" applied on the machine -# does not reflect the edits, some installations might fail. -# Even if the failure may not directly impact execution, -# this leads into inconsistency between installed environment -# and the code base which may lead into fatal error or uncaught exceptions. -# -# The "virl image" is built from files such as -# resources/tools/disk-image-builder/ubuntu/lists/ubuntu-16.04.1_2017-10-21_2.0/pip-requirements.txt -# but the build is not automated, -# the new "virl image" has to be built and applied manually -# before your edit can pass. -# This also means, incompatible virl images -# would make existing tests fail until your edit is merged. -# -# TODO: Automate the virtualenv download and distribution -# on all types of testbeds prior the test execution. -# TODO: Figure out a way to verify edits to this file automatically. - -# TODO: Split into directly needed packages and their dependencies. -docopt==0.6.2 # for pykwalify -ecdsa==0.13 -enum34==1.1.2 -ipaddress==1.0.16 -paramiko==1.16.0 -pexpect==4.6.0 -pycrypto==2.6.1 -pykwalify==1.5.0 -pypcap==1.1.5 -python-dateutil==2.4.2 # for pykwalify -PyYAML==3.11 -requests==2.9.1 -robotframework==2.9.2 -scapy==2.3.1 -scp==0.10.2 -six==1.12.0 # for python-dateutil - -# The following is only there for PLRsearch. -dill==0.2.8.2 -numpy==1.14.5 -scipy==1.1.0 diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml index e5da2fc72d..1ad96b26fc 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/main.yaml @@ -1,22 +1,16 @@ --- # file: roles/tg_sut/tasks/main.yaml -- name: Ubuntu specific - import_tasks: ubuntu.yaml - when: ansible_distribution|lower == 'ubuntu' +- name: Install distribution - release - machine prerequisites + include_tasks: '{{ ansible_distribution|lower }}_{{ ansible_distribution_release }}.yaml' -- name: x86 specific - import_tasks: x86_64.yaml - when: ansible_machine == 'x86_64' +- name: Machine specifics + include_tasks: '{{ ansible_machine }}.yaml' - name: skylake specific import_tasks: skylake.yaml when: ("skylake" in groups) and inventory_hostname in groups['skylake'] -- name: aarch specific - import_tasks: aarch64.yaml - when: ansible_machine == 'aarch64' - - name: Copy netplan network config file template: src: 'files/netplan_config' @@ -53,21 +47,40 @@ mode: '0644' tags: copy-cpufrequtils -- name: Copy Python requirements file - copy: - src: 'files/requirements.txt' - dest: '/tmp/requirements.txt' - tags: copy-pip - - name: Set ondemand service to disable service: name: 'ondemand' enabled: 'no' tags: set-ondemand -- name: Install PIP requirements +- name: Install CSIT PIP requirements + pip: + name: '{{ item }}' + with_items: + - 'docopt==0.6.2' + - 'ecdsa==0.13' + - 'enum34==1.1.2' + - 'ipaddress==1.0.16' + - 'paramiko==1.16.0' + - 'pexpect==4.6.0' + - 'pycrypto==2.6.1' + - 'pykwalify==1.5.0' + - 'pypcap==1.1.5' + - 'python-dateutil==2.4.2' + - 'PyYAML==3.11' + - 'requests==2.9.1' + - 'robotframework==2.9.2' + - 'scapy==2.3.1' + - 'scp==0.10.2' + - 'six==1.12.0' + - 'dill==0.2.8.2' + - 'numpy==1.14.5' + - 'scipy==1.1.0' + tags: install-pip + +- name: Install VPP PIP requirements pip: - requirements: '/tmp/requirements.txt' + name: 'aenum' tags: install-pip - name: Load vfio-pci by default diff --git a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu.yaml b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml similarity index 97% rename from resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu.yaml rename to resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml index 2f75ae545d..194609722e 100644 --- a/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu.yaml +++ b/resources/tools/testbed-setup/ansible/roles/tg_sut/tasks/ubuntu_bionic.yaml @@ -1,5 +1,5 @@ --- -# file: roles/tg_sut/tasks/ubuntu.yaml +# file: roles/tg_sut/tasks/ubuntu_bionic.yaml - name: Install CSIT dependencies apt: @@ -38,6 +38,7 @@ apt: name: 'docker-{{ docker_edition }}={{ docker_apt_package_name }}' state: 'present' + force: yes tags: install-docker - name: Creates Docker service directory -- 2.16.6