From: Peter Mikus Date: Tue, 7 Oct 2025 12:14:16 +0000 (+0200) Subject: feat(ansible): K8S tune X-Git-Url: https://gerrit.fd.io/r/gitweb?a=commitdiff_plain;h=7a80766881d74d420b5583ab54a4c78b7faf633a;p=csit.git feat(ansible): K8S tune Signed-off-by: Peter Mikus Change-Id: Ie317a53a14847302ca771951dd4cf3b4433a2651 (cherry picked from commit a202b38009d610c8ca0ea1aa259bebd64a067c38) --- diff --git a/.github/actions/aws_s3_publish_logs/action.yml b/.github/actions/aws_s3_publish_logs/action.yml new file mode 100644 index 0000000000..492b72d29b --- /dev/null +++ b/.github/actions/aws_s3_publish_logs/action.yml @@ -0,0 +1,137 @@ +--- +name: "🛠️ AWS S3 Publish Logs" +description: | + This GitHub Action uploads logs to AWS S3. + +inputs: + AWS_ACCESS_KEY_ID: + description: "Unique, public identifier for an AWS IAM user." + required: true + type: string + AWS_SECRET_ACCESS_KEY: + description: "Long-term security credential for AWS account or an IAM user." + required: true + type: string + AWS_REGION: + description: "Long-term security credential for AWS account or an IAM user." + required: true + type: string + S3_BUCKET: + description: "Name of the Amazon S3 bucket." + required: true + default: "fdio-logs-s3-cloudfront-index" + type: string + S3_PATH: + description: "Path within Amazon AWS S3 bucket." + required: false + default: "vex-yul-rot-jenkins-1/${{ github.job }}/${{ github.run_id }}" + type: string + ARCHIVES_PATH: + description: "Source directory with logs artifact to archive." + required: false + default: "${{ github.workspace }}/archives" + type: string + +runs: + using: "composite" + steps: + - name: Check if AWS CLI is pre-installed + id: aws-binary-check + shell: bash + run: | + if command -v aws >/dev/null 2>&1; then + echo "AWS CLI is already installed. Skipping install." + echo "AWS_CLI_PREINSTALLED=true" >> "$GITHUB_OUTPUT" + aws --version + exit 0 + fi + echo "AWS_CLI_PREINSTALLED=false" >> "$GITHUB_OUTPUT" + + - name: Cache or Restore the zip + if: ${{ steps.aws-binary-check.outputs.AWS_CLI_PREINSTALLED == 'false' }} + uses: actions/cache@v4 + id: aws-cli-cache + with: + path: ${{ runner.temp }}/aws-cli-cache/*.zip + key: ${{ runner.os }}-${{ runner.arch }}-aws-cli-v2-zip + + - name: Install AWS CLI + shell: bash + if: ${{ steps.aws-binary-check.outputs.AWS_CLI_PREINSTALLED == 'false' }} + env: + CACHE_HIT: ${{ steps.aws-cli-cache.outputs.cache-hit == 'true' }} + CACHE_PATH: ${{ runner.temp }}/aws-cli-cache + run: | + AWS_PACKAGE="awscli-exe-linux-$(uname -m).zip" + + mkdir -p "$CACHE_PATH" + cd "$CACHE_PATH" + + if [ "$CACHE_HIT" = "false" ]; then + curl -fsSL \ + --retry 3 \ + --retry-delay 5 \ + --connect-timeout 15 \ + --max-time 60 \ + -o "$AWS_PACKAGE" "https://awscli.amazonaws.com/$AWS_PACKAGE" + fi + + unzip -o -q "$AWS_PACKAGE" + sudo ./aws/install --update + rm -rf ./aws + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v5.0.0 + with: + aws-access-key-id: ${{ inputs.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ inputs.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.AWS_REGION }} + + - name: Upload archives directory content to S3 + shell: bash + run: | + get_content_type() { + local file_ext="${1##*.}" + case "$file_ext" in + xml) echo "application/xml" ;; + html) echo "text/html" ;; + txt) echo "text/plain" ;; + log) echo "text/plain" ;; + css) echo "text/css" ;; + md) echo "text/markdown" ;; + rst) echo "text/x-rst" ;; + csv) echo "text/csv" ;; + svg) echo "image/svg+xml" ;; + jpg|jpeg) echo "image/jpeg" ;; + png) echo "image/png" ;; + gif) echo "image/gif" ;; + js) echo "application/javascript" ;; + pdf) echo "application/pdf" ;; + json) echo "application/json" ;; + otf) echo "application/otf" ;; + ttf) echo "application/ttf" ;; + woff) echo "application/woff" ;; + woff2) echo "application/woff2" ;; + *) echo "application/octet-stream" ;; + esac + } + + export -f get_content_type + + pushd ${{ inputs.ARCHIVES_PATH }} + + # Traverse and upload + find . -type f | while read -r file; do + rel_path="${file#./}" # relative path + tmp_file="$(mktemp).gz" # temp gzip file + gzip -c "${file}" > "${tmp_file}" # compress + + content_type=$(get_content_type "${file}") + + S3_ARN="s3://${{ inputs.S3_BUCKET }}/${{ inputs.S3_PATH }}/${rel_path}.gz" + + echo "Uploading ${rel_path}.gz -> ${S3_ARN} (Content-Type: ${content_type}, gzip)" + #aws s3 cp "${tmp_file}" "${S3_ARN}" --content-type "${content_type}" --content-encoding "gzip" + + rm -f "${tmp_file}" + done diff --git a/.github/workflows/gerrit-csit-perf-mrr-daily.yml b/.github/workflows/gerrit-csit-perf-mrr-daily.yml deleted file mode 100644 index 778b5d375c..0000000000 --- a/.github/workflows/gerrit-csit-perf-mrr-daily.yml +++ /dev/null @@ -1,96 +0,0 @@ ---- -name: csit-vpp-perf-mrr-daily - -# yamllint disable-line rule:truthy -on: - # Runs periodically - #schedule: - # - cron: "30 22 * * 0-4" - workflow_dispatch: - -jobs: - csit-vpp-perf-mrr-daily: - name: csit-vpp-perf-mrr-daily-master-${{ matrix.node }} - runs-on: - - self-hosted - - nomad - - fdio:arch=${{ matrix.executor_arch }} - - fdio:class=builder - - fdio:namespace=prod - - fdio:os=${{ matrix.os }} - - strategy: - matrix: - include: - - node: 2n-grc - os: ubuntu2404 - executor_arch: aarch64 - - node: 2n-icx - os: ubuntu2404 - executor_arch: x86_64 - - node: 2n-spr - os: ubuntu2404 - executor_arch: x86_64 - - node: 2n-zn2 - os: ubuntu2404 - executor_arch: x86_64 - - node: 3n-icx - os: ubuntu2404 - executor_arch: x86_64 - - node: 3n-alt - os: ubuntu2404 - executor_arch: aarch64 - - node: 3n-snr - os: ubuntu2404 - executor_arch: x86_64 - - node: 3n-icxd - os: ubuntu2404 - executor_arch: x86_64 - - node: 3na-spr - os: ubuntu2404 - executor_arch: x86_64 - - node: 3nb-spr - os: ubuntu2404 - executor_arch: x86_64 - - node: 2n-emr - os: ubuntu2404 - executor_arch: x86_64 - - node: 3n-emr - os: ubuntu2404 - executor_arch: x86_64 - - node: 3n-oct - os: ubuntu2404 - executor_arch: aarch64 - - env: - JOB_NAME: "${{ github.job }}-master-${{ matrix.node }}" - - steps: - - name: Gerrit Checkout - # yamllint disable-line rule:line-length - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 - with: - gerrit-refspec: refs/heads/master - gerrit-project: "csit" - gerrit-url: ${{ vars.GERRIT_URL }} - delay: "0s" - ref: refs/heads/master - - - name: "Retrieve GIT commit message" - uses: lfit/releng-reusable-workflows/.github/actions/git-commit-message-action@main - - - name: Setup Environment - uses: fdio/csit/.github/actions/setup_executor_env@master - - - name: Run Performance Bootstrap - run: | - csit_entry_dir="./resources/libraries/bash/entry" - #source ${csit_entry_dir}/with_oper_for_vpp.sh "bootstrap_verify_perf.sh" - - - name: Archive Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ env.JOB_NAME }}-${{ github.run_number }} - path: archives/ - if-no-files-found: "ignore" \ No newline at end of file diff --git a/.github/workflows/gerrit-csit-terraform-aws-eb-version.yml b/.github/workflows/gerrit-csit-terraform-aws-eb-version.yml deleted file mode 100644 index af564c104b..0000000000 --- a/.github/workflows/gerrit-csit-terraform-aws-eb-version.yml +++ /dev/null @@ -1,140 +0,0 @@ ---- -name: csit-terraform-aws-eb-version - -# yamllint disable-line rule:truthy -on: - workflow_dispatch: - inputs: - GERRIT_BRANCH: - description: "Branch that change is against" - required: true - type: string - GERRIT_CHANGE_ID: - description: "The ID for the change" - required: true - type: string - GERRIT_CHANGE_NUMBER: - description: "The Gerrit number" - required: true - type: string - GERRIT_CHANGE_URL: - description: "URL to the change" - required: true - type: string - GERRIT_EVENT_TYPE: - description: "Type of Gerrit event" - required: true - type: string - GERRIT_PATCHSET_NUMBER: - description: "The patch number for the change" - required: true - type: string - GERRIT_PATCHSET_REVISION: - description: "The revision sha" - required: true - type: string - GERRIT_PROJECT: - description: "Project in Gerrit" - required: true - type: string - GERRIT_REFSPEC: - description: "Gerrit refspec of change" - required: true - type: string - -concurrency: - # yamllint disable-line rule:line-length - group: verify-${{ github.workflow }}-${{ github.event.inputs.GERRIT_BRANCH }}-${{ github.event.inputs.GERRIT_CHANGE_ID || github.run_id }} - cancel-in-progress: true - -jobs: - verify: - runs-on: - - self-hosted - - nomad - - fdio:arch=${{ matrix.executor_arch }} - - fdio:class=builder - - fdio:namespace=prod - - fdio:os=${{ matrix.os }} - - strategy: - matrix: - os: [ubuntu2404] - executor_arch: [x86_64] - - env: - JOB_NAME: "csit-terraform-aws-eb-version-verify" - BUILD_ID: "${{ github.run_number }}" - - steps: - - name: Gerrit Checkout - # yamllint disable-line rule:line-length - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 - with: - gerrit-refspec: ${{ github.event.inputs.GERRIT_REFSPEC }} - gerrit-project: ${{ github.event.inputs.GERRIT_PROJECT }} - gerrit-url: ${{ vars.GERRIT_URL }} - delay: "30s" - ref: refs/heads/${{ github.event.inputs.GERRIT_BRANCH }} - - - name: "Retrieve GIT commit message" - # yamllint disable-line rule:line-length - uses: lfit/releng-reusable-workflows/.github/actions/git-commit-message-action@main - - - name: Setup Environment - uses: fdio/csit/.github/actions/setup_executor_env@master - - - name: Run Terraform Bootstrap - run: | - csit_entry_dir="./resources/libraries/bash/entry" - source ${csit_entry_dir}/bootstrap_aws_eb_version_verify.sh - - deploy: - needs: verify - runs-on: - - self-hosted - - nomad - - fdio:arch=${{ matrix.executor_arch }} - - fdio:class=builder - - fdio:namespace=sandbox - - fdio:os=${{ matrix.os }} - - strategy: - matrix: - os: [ubuntu2404] - executor_arch: [x86_64] - - env: - JOB_NAME: "csit-terraform-aws-eb-version-deploy" - BUILD_ID: "${{ github.run_number }}" - - steps: - - name: Gerrit Checkout - # yamllint disable-line rule:line-length - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 - with: - gerrit-refspec: ${{ github.event.inputs.GERRIT_REFSPEC }} - gerrit-project: ${{ github.event.inputs.GERRIT_PROJECT }} - gerrit-url: ${{ vars.GERRIT_URL }} - delay: "30s" - ref: refs/heads/${{ github.event.inputs.GERRIT_BRANCH }} - - - name: "Retrieve GIT commit message" - # yamllint disable-line rule:line-length - uses: lfit/releng-reusable-workflows/.github/actions/git-commit-message-action@main - - - name: Setup Environment - uses: pmikus/csit/.github/actions/setup_executor_env@master - - - name: Run Terraform Bootstrap - run: | - csit_entry_dir="./resources/libraries/bash/entry" - source ${csit_entry_dir}/bootstrap_aws_eb_version_deploy.sh - - - name: Archive Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ env.JOB_NAME }}-${{ github.run_number }} - path: archives/ - if-no-files-found: "ignore" \ No newline at end of file diff --git a/.github/workflows/gerrit-csit-tox-verify.yml b/.github/workflows/gerrit-csit-tox-verify.yml deleted file mode 100644 index 64e9beec09..0000000000 --- a/.github/workflows/gerrit-csit-tox-verify.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -name: csit-verify-tox - -# yamllint disable-line rule:truthy -on: - workflow_dispatch: - inputs: - GERRIT_BRANCH: - description: "Branch that change is against" - required: true - type: string - GERRIT_CHANGE_ID: - description: "The ID for the change" - required: true - type: string - GERRIT_CHANGE_NUMBER: - description: "The Gerrit number" - required: true - type: string - GERRIT_CHANGE_URL: - description: "URL to the change" - required: true - type: string - GERRIT_EVENT_TYPE: - description: "Type of Gerrit event" - required: true - type: string - GERRIT_PATCHSET_NUMBER: - description: "The patch number for the change" - required: true - type: string - GERRIT_PATCHSET_REVISION: - description: "The revision sha" - required: true - type: string - GERRIT_PROJECT: - description: "Project in Gerrit" - required: true - type: string - GERRIT_REFSPEC: - description: "Gerrit refspec of change" - required: true - type: string - -concurrency: - # yamllint disable-line rule:line-length - group: csit-verify-tox-${{ github.workflow }}-${{ github.event.inputs.GERRIT_BRANCH }}-${{ github.event.inputs.GERRIT_CHANGE_ID || github.run_id }} - cancel-in-progress: true - -jobs: - csit-verify-tox: - name: "csit-verify-tox" - runs-on: - - self-hosted - - nomad - - fdio:arch=${{ matrix.executor_arch }} - - fdio:class=builder - - fdio:namespace=prod - - fdio:os=${{ matrix.os }} - - strategy: - matrix: - os: [ubuntu2404] - executor_arch: [x86_64] - - env: - JOB_NAME: "${{ github.job }}" - - steps: - - name: Gerrit Checkout - # yamllint disable-line rule:line-length - uses: lfit/checkout-gerrit-change-action@54d751e8bd167bc91f7d665dabe33fae87aaaa63 # v0.9 - with: - gerrit-refspec: ${{ inputs.GERRIT_REFSPEC }} - gerrit-project: ${{ inputs.GERRIT_PROJECT }} - gerrit-url: ${{ vars.GERRIT_URL }} - delay: "30s" - ref: refs/heads/${{ inputs.GERRIT_BRANCH }} - - - name: "Retrieve GIT commit message" - # yamllint disable-line rule:line-length - uses: lfit/releng-reusable-workflows/.github/actions/git-commit-message-action@main - - - name: Setup Environment - uses: fdio/csit/.github/actions/setup_executor_env@master - - - name: Run Tox - run: | - source ./resources/libraries/bash/entry/tox.sh - - - name: Archive Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ env.JOB_NAME }}-${{ github.run_number }} - path: archives/ - if-no-files-found: "ignore" \ No newline at end of file diff --git a/fdio.infra.ansible/sut.yaml b/fdio.infra.ansible/sut.yaml index 38735d721c..fb6397b05a 100644 --- a/fdio.infra.ansible/sut.yaml +++ b/fdio.infra.ansible/sut.yaml @@ -24,6 +24,8 @@ tags: docker - role: kubernetes tags: kubernetes + #- role: calico + # tags: calico - role: vpp tags: vpp - role: dpdk