feat(cdash): Add alternative S3 storage as an option

Signed-off-by: Peter Mikus <peter.mikus@icloud.com>
Change-Id: Idb23d1522ccee605d5e9908878c8e4fd56b7e6bf

diff --git a/csit.infra.dash/app/cdash/data/data.py b/csit.infra.dash/app/cdash/data/data.py
index 749769c..0d1e313 100644 (file)
--- a/csit.infra.dash/app/cdash/data/data.py
+++ b/csit.infra.dash/app/cdash/data/data.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -115,6 +115,8 @@ class Data:
         if days:
             last_modified_begin = datetime.now(tz=UTC) - timedelta(days=days)
         try:
+            if C.AWS_ENDPOINT_URL:
+                wr.config.s3_endpoint_url = C.AWS_ENDPOINT_URL
             file_list = wr.s3.list_objects(
                 path=path,
                 suffix="parquet",
@@ -202,6 +204,8 @@ class Data:
         if days:
             last_modified_begin = datetime.now(tz=UTC) - timedelta(days=days)
 
+        if C.AWS_ENDPOINT_URL:
+            wr.config.s3_endpoint_url = C.AWS_ENDPOINT_URL
         df = wr.s3.read_parquet(
             path=path,
             path_suffix="parquet",
@@ -290,6 +294,8 @@ class Data:
             wr.config.botocore_config = botocore.config.Config(
                 max_pool_connections=C.MAX_POOL_SIZE
             )
+            if C.AWS_ENDPOINT_URL:
+                wr.config.s3_endpoint_url = C.AWS_ENDPOINT_URL
             df = wr.s3.read_parquet(
                 path=path,
                 path_suffix="parquet",

diff --git a/csit.infra.dash/app/cdash/utils/constants.py b/csit.infra.dash/app/cdash/utils/constants.py
index 823ecc6..f4a680e 100644 (file)
--- a/csit.infra.dash/app/cdash/utils/constants.py
+++ b/csit.infra.dash/app/cdash/utils/constants.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Cisco and/or its affiliates.
+# Copyright (c) 2025 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -93,6 +93,9 @@ class Constants:
     ############################################################################
     # General, application wide constants.
 
+    # AWS specific settings.
+    AWS_ENDPOINT_URL = get_str_from_env("AWS_ENDPOINT_URL", "")
+
     # Select applications to start.
     START_TRENDING = get_bool_from_env("START_TRENDING", True)
     START_REPORT = get_bool_from_env("START_REPORT", True)

diff --git a/csit.infra.dash/do_run.sh b/csit.infra.dash/do_run.sh
index 9ededc7..73f4205 100755 (executable)
--- a/csit.infra.dash/do_run.sh
+++ b/csit.infra.dash/do_run.sh
@@ -7,4 +7,5 @@ command -v docker || exit 1
 export UID=$(id -u)
 export GID=$(id -g)
 
+docker network create csit-data || true
 docker compose up --remove-orphans --detach

diff --git a/csit.infra.dash/docker-compose.yaml b/csit.infra.dash/docker-compose.yaml
index 8ec97fa..bff7f58 100644 (file)
--- a/csit.infra.dash/docker-compose.yaml
+++ b/csit.infra.dash/docker-compose.yaml
@@ -1,9 +1,16 @@
-version: "3"
+networks:
+  csit-data:
+    external: true
+
 services:
   cdash:
     build: "."
     command: "uwsgi --ini app.ini"
     environment:
+      #AWS_ACCESS_KEY_ID: ""
+      #AWS_SECRET_ACCESS_KEY: ""
+      #CSIT_AWS_ENDPOINT_URL: "http://minio:9000" # If configured, set below value to True.
+      AWS_IGNORE_CONFIGURED_ENDPOINT_URLS: "False"
       FLASK_DEBUG: 1
       FLASK_ENV: "development"
       CSIT_START_TRENDING: "True"
@@ -22,6 +29,8 @@ services:
       CSIT_URL_DOC: "https://csit.fd.io/cdocs/"
       CSIT_TIME_PERIOD: 250
     mem_limit: "16g"
+    networks:
+      - "csit-data"
     ports:
       - "5000:5000"
     user: "${UID}:${GID}"