fix: API does not suspend after 1st request

feat: loop over all clusters

.env

@@ -0,0 +1,3 @@
+cluster_inventory_path = config/inventory.yml
+redis_host = '172.16.0.208'
+redis_port = '6379'

.gitignore

@@ -129,7 +129,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
+#.env
 .venv
 env/
 venv/

README.md

@@ -1,3 +1,19 @@
 # generic_api_endpoint
 
 Hackathon API endpoint
+
+## management summary // usecase
+This API acts as a middelware for service portals and frontends (like SNOW), that can retrieve data via REST API. It manages metadata.
+
+## ideas for future
+- store the data in redis on initialization or on first request
+  - also first query redis, and not directly ONTAP
+- documentation -> make it understandable, so that users will use it!
+- add capability to apply filters/conditions on the return
+  - Alexeys
+    - 
+- performance based filtering
+
+- add capability for finding best clusters, volumes
+- get credentials from credential-mgmt-system
+- 

config/inventory.yml

@@ -1,8 +1,6 @@
-- 1:
+- hostname: "172.16.57.2"
-    hostname: '172.16.57.2'
+  username: "admin"
-    username: 'admin'
+  password: "Netapp12"
-    password: 'Netapp12'
+- hostname: "172.16.56.2"
-- 2:
+  username: "admin"
-    hostname: '172.16.56.2'
+  password: "Netapp12"
-    username: 'admin'
-    password: 'Netapp12'

pyproject.toml

@@ -12,5 +12,6 @@ requires-python = ">=3.13"
 dependencies = [
     "fastapi[standard]>=0.116.2",
     "httpx>=0.28.1",
+    "python-dotenv>=1.1.1",
     "redis>=6.4.0",
 ]

requirements.txt

@@ -1,6 +1,7 @@
 fastapi[standard]>=0.116.2
 httpx>=0.28.1
 redis>=6.4.0
+python-dotenv>=1.1.1
 pydantic
 redis[hiredis]
 dotenv

src/.env

@@ -0,0 +1,3 @@
+cluster_inventory_path = ./config/inventory.yml
+redis_host = '172.16.0.208'
+redis_port = '6379'

src/aggregate/__init__.py

@@ -0,0 +1,5 @@
+from src.example.router import router as example_router
+
+from src.aggregate.aggregate_router import router as aggregate_router
+
+__all__ = ["example_router", "aggregate_router"]

src/aggregate/aggregate_router.py

@@ -0,0 +1,23 @@
+# contains the router for the aggregates endpoint
+from fastapi import APIRouter, Query, Request
+from typing import List, Dict
+from .aggregate_schema import AggregateSchema, MetricEnum
+from .aggregate_service import get_aggregates
+
+
+router = APIRouter(tags=["aggregates"])
+
+
+@router.get("/aggregates", response_model=List[AggregateSchema])
+async def aggregates_endpoint(
+    request: Request,
+    metric: MetricEnum = Query(MetricEnum.relative, description="Metric type"),
+):
+    # Extract tag parameters from query string
+    tags: Dict[str, str] = {}
+    for param_name, param_value in request.query_params.items():
+        if param_name.startswith("tag."):
+            tag_key = param_name[4:] 
+            tags[tag_key] = param_value
+    
+    return await get_aggregates(request, metric, tags)

src/aggregate/aggregate_schema.py

@@ -0,0 +1,23 @@
+# contains the schema definitions for aggregates
+from pydantic import BaseModel
+from enum import Enum
+
+
+class AggregateSchema(BaseModel):
+    aggregate: str
+    node: str
+    available: int
+    available_str: str
+
+
+class MetricEnum(str, Enum):
+    relative = "relative"
+    absolute = "absolute"
+
+TAG2REST = {
+'worm_compliance': { 'snaplock_type': 'compliance' },
+'worm_enterprise': { 'snaplock_type': 'enterprise' },
+'flash': { 'block_storage.storage_type': 'ssd' },
+'hdd': { 'block_storage.storage_type': 'hdd' }, 
+'mcc': { 'block_storage.mirror.enabled': 'true' }
+}

src/aggregate/aggregate_service.py

@@ -0,0 +1,58 @@
+# contains the business logic for aggregates
+
+from typing import List, Dict
+from pprint import pprint
+from fastapi import Request
+from src.aggregate.aggregate_schema import AggregateSchema, MetricEnum
+from logging import getLogger
+from src.utils import round_bytes, get_data_from_ontap
+
+logger = getLogger("uvicorn")
+logger.setLevel("DEBUG")
+
+# TAG2REST = {
+# 'worm_compliance': { 'snaplock_type': 'compliance' },
+# 'worm_enterprise': { 'snaplock_type': 'enterprise' },
+# 'flash': { 'block_storage.storage_type': 'ssd' },
+# 'hdd': { 'block_storage.storage_type': 'hdd' }, 
+# 'mcc': { 'block_storage.mirror.enabled': 'true' }
+# }
+
+# {
+#     "flash": "production",
+#     "performance": "gold", 
+#     "worm": "compliance"
+# }
+
+async def get_aggregates(request: Request, metric: str = "relative", tags: Dict[str, str] = None) -> List[AggregateSchema]:
+    # Dummy data for demonstration
+    # You can use the metric parameter to filter or modify results as needed
+    # For now, just return the same data and show metric usage
+    logger.debug(f"Metric used: {metric}")
+    logger.debug(f"Tags used: {tags}")
+
+    # convert tags to ONTAP filter
+    # filter_str = ""
+    # if tags:
+    #     str_filter_parts = [f"tag.{key} eq '{value}'" for key, value in tags.items()]
+    #     param_str = "&".join([f"{TAG2REST[key]}" for key, value in tags.items()])
+
+
+    __aggregates = await get_data_from_ontap(request, logger, "storage/aggregates", "fields=*")
+    pprint(__aggregates)
+    if metric == MetricEnum.relative:
+        __aggregates = sorted(__aggregates, key=lambda r: r["space"]["block_storage"].get("used_percent"), reverse=True)
+    elif metric == MetricEnum.absolute:
+        __aggregates = sorted(__aggregates, key=lambda r: r["space"]["block_storage"].get("available"), reverse=False)
+
+    aggregates: list = [
+        AggregateSchema(
+            aggregate=a["name"],
+            node=a["node"]["name"],
+            available=a["space"]["block_storage"]["available"],
+            available_str=round_bytes(a["space"]["block_storage"]["available"]),
+        )
+        for a in __aggregates
+    ]
+
+    return aggregates

src/config_upload/__init__.py

@@ -0,0 +1,3 @@
+from src.config_upload.router import router as config_router
+
+__all__ = ["config_router"]

src/config_upload/config.http

@@ -0,0 +1,14 @@
+POST http://127.0.0.1:8000/config
+Content-Type: application/json
+
+{
+  "cluster_list": [
+    {
+      "hostname": "cluster1.demo.netapp.com",
+      "username": "admin",
+      "password": "Netapp1!"
+    }
+  ]
+}
+
+###

src/config_upload/router.py

@@ -0,0 +1,23 @@
+import logging
+
+from fastapi import APIRouter
+
+from .schema import ConfigReturnSchema, ConfigSchema
+
+logger = logging.getLogger("uvicorn")
+
+router = APIRouter(tags=["config_upload"])
+
+
+@router.post(
+    "/config", summary="Upload a configuration", response_model=ConfigReturnSchema
+)
+async def create_config(config: ConfigSchema) -> ConfigSchema:
+    """
+    Endpoint to receive and store configuration data.
+
+    ⚠️ at this time the configuration is not stored anywhere. It's like logging to /dev/null
+    """
+    logger.info("Received configuration data")
+    return config
+

src/config_upload/schema.py

@@ -0,0 +1,21 @@
+# contains the schema definitions for the config_upload service
+from pydantic import BaseModel
+
+
+class ConfigEntrySchema(BaseModel):
+    hostname: str
+    username: str
+    password: str
+
+
+class ConfigOutSchema(BaseModel):
+    hostname: str
+    username: str
+
+
+class ConfigReturnSchema(BaseModel):
+    cluster_list: list[ConfigOutSchema]
+
+
+class ConfigSchema(BaseModel):
+    cluster_list: list[ConfigEntrySchema]

src/config_upload/service.py

@@ -0,0 +1,2 @@
+# contains the business logic for the config_upload service
+async def save_config() -> None: ...

src/database.py

@@ -3,7 +3,7 @@ import logging
 from redis import Redis, ConnectionError
 from typing import List
 from pydantic import TypeAdapter
-from schema import ConfigSchema
+from src.schema import ConfigSchema
 
 
 def setup_db_conn(redishost, redisport: str):

src/example/__init__.py

@@ -1,3 +0,0 @@
-from .router import router as example_router
-
-__all__ = ["example_router"]

src/example/router.py

@@ -1,8 +1,10 @@
-# contains the router for the example endpoint
+# contains the router for the aggregate endpoint
 from fastapi import APIRouter
-from .schema import ExampleSchema
 
-router = APIRouter(tags=["example"])
+from src.example.schema import ExampleSchema
+
+router = APIRouter(tags=["aggregate"])
+
 
 @router.get("/example")
 async def example_endpoint() -> ExampleSchema:

src/example/schema.py

@@ -1,14 +1,18 @@
-# contains the schema definitions for the example service
+# contains the schema definitions for the aggregate service
 from pydantic import BaseModel
+from pathlib import Path
+
 
 class ExampleSchema(BaseModel):
     example_field: str
     another_field: int
 
+
 class ClusterCreds(BaseModel):
     """A structure to hold basic auth cluster credentials for a cluster"""
-    username:      str
+
-    password:      str
+    username: str
-    hostname:      str  = None
+    password: str
+    hostname: str = None
     cert_filepath: Path = None
-    key_filepath:  Path = None
+    key_filepath: Path = None

src/example/service.py

@@ -1,3 +1,3 @@
-# contains the business logic for the example service
+# contains the business logic for the aggregate service
 async def example_service() -> str:
-    return "This is an example service"
+    return "This is an aggregate service"

src/initialize.py

@@ -5,35 +5,36 @@ import yaml
 
 from pathlib import Path
 from dotenv import load_dotenv
-from database import setup_db_conn
+from src.database import setup_db_conn
-from schema import ConfigSchema
+from src.schema import ConfigSchema
 from typing import List
 from pydantic import TypeAdapter
 
+
 def initialize_config():
     load_dotenv()
-    log = logging.getLogger('uvicorn')
+    log = logging.getLogger("uvicorn")
-    ENV_INVENTORYPATH = os.getenv('cluster_inventory_path')
+    ENV_INVENTORYPATH = os.getenv("cluster_inventory_path")
-    ENV_REDISHOST = os.getenv('redis_host')
+    ENV_REDISHOST = os.getenv("redis_host")
-    ENV_REDISPORT = os.getenv('redis_port')
+    ENV_REDISPORT = os.getenv("redis_port")
 
     log.info(f"Found Cluster Inventory file at: {ENV_INVENTORYPATH}")
     if not ENV_INVENTORYPATH or not Path(ENV_INVENTORYPATH).is_file():
         print(f"FATAL: Inventory file {ENV_INVENTORYPATH} is missing or not a file.")
         return False
     try:
-        with open(ENV_INVENTORYPATH, 'r') as f:
+        with open(ENV_INVENTORYPATH, "r") as f:
             inv = yaml.safe_load(f)
             inventory = json.dumps(inv)
     except Exception as e:
         print(f"FATAL: Cannot read inventory file {ENV_INVENTORYPATH}. Err: {e}")
         return False
 
-    print(f'[INFO] Importing configuration to DB...')
+    log.info(f"Importing configuration to DB...")
     try:
         GLOBAL_INVENTORY_VALID = TypeAdapter(List[ConfigSchema]).validate_python(inv)
         redis_conn = setup_db_conn(ENV_REDISHOST, ENV_REDISPORT)
-        redis_conn.hset('cluster_inventory', mapping={'inventory': inventory})
+        redis_conn.hset("cluster_inventory", mapping={"inventory": inventory})
         redis_conn.close()
 
         log.info("Configuration has been loaded.")

src/main.py

@@ -1,30 +1,30 @@
 import os
-import json
 import logging
-import yaml
+import httpx
-
-from pathlib import Path
-from dotenv import load_dotenv
-from redis import Redis
-from contextlib import asynccontextmanager
-
-from pydantic import BaseModel, ValidationError, SecretStr, AnyHttpUrl
-from typing import Optional, Literal, List, Union
 from fastapi import FastAPI
 
+from src.aggregate import aggregate_router
+from src.config_upload import config_router
 
-from database import setup_db_conn, get_inventory_from_redis, get_config_from_db
+from contextlib import asynccontextmanager
+
+from .database import setup_db_conn, get_config_from_db
 from src.initialize import initialize_config
-from utils import setup_logging 
+from .utils import setup_logging
+
+logger = logging.getLogger("uvicorn")
+logger.setLevel("DEBUG")
+logger.info("Starting application")
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    ''' make loading it async'''
+    """make loading it async"""
-    log = logging.getLogger('uvicorn')
+    global shared_redis_conn, requests_client
+    log = logging.getLogger("uvicorn")
     cfg_init_result = initialize_config()
 
-    shared_redis_conn = setup_db_conn(os.getenv('redis_host'), os.getenv('redis_port'))
+    shared_redis_conn = setup_db_conn(os.getenv("redis_host"), os.getenv("redis_port"))
     if not shared_redis_conn:
         log.error("Cannot connect to Redis DB. Exiting...")
         exit(1)
@@ -34,13 +34,16 @@ async def lifespan(app: FastAPI):
     if not cfg_init_result:
         log.error("Configuration initialization failed. Exiting...")
         # exit(1)
-
+    requests_client = httpx.AsyncClient(verify=False)
-    yield
+    yield {"redis_conn": shared_redis_conn, "requests_client": requests_client}
+    await requests_client.aclose()
     log.info("Shutting down FastAPI app...")
 
 
 setup_logging()
-log = logging.getLogger('uvicorn')
+log = logging.getLogger("uvicorn")
 
 log.info("Starting FastAPI app...")
 app = FastAPI(lifespan=lifespan)
+app.include_router(aggregate_router)
+app.include_router(config_router)

src/service.py

@@ -0,0 +1,17 @@
+import logging
+
+from dotenv import dotenv_values
+
+from src.schema import ConfigSchema
+
+logger = logging.getLogger("uvicorn")
+
+
+def load_config() -> ConfigSchema:
+    logger.info("Loading config from .env file")
+    config = dotenv_values(".env")
+    return ConfigSchema(
+        hostname=config["CLUSTER1_HOSTNAME"],
+        username=config["CLUSTER1_USERNAME"],
+        password=config["CLUSTER1_PASSWORD"],
+    )

src/utils.py

@@ -1,9 +1,44 @@
 import logging
+from fastapi import Request
+import httpx
+from src.database import get_config_from_db
+
+
+def round_bytes(size_in_bytes: int) -> str:
+    # Helper function to convert bytes to a human-readable format
+    for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]:
+        if size_in_bytes < 1024:
+            return f"{size_in_bytes:.2f}{unit}"
+        size_in_bytes /= 1024
+    return f"{size_in_bytes:.2f}EB"
+
+
+async def get_data_from_ontap(request: Request, logger, endpoint: str, query_string: str = ""):
+    # get clusters from redis
+
+    redis_conn = request.state.redis_conn
+    config = get_config_from_db(redis_conn)
+    logger.debug("Got the config from REDIS: %s", config)
+
+    results = []
+    client = request.state.requests_client
+    for cluster in config:
+        print(f"\n\n looping, {cluster}")
+        url = f"https://{cluster.hostname}/api/{endpoint}"
+        if query_string:
+            url += f"?{query_string}"
+        try:
+            logger.debug(f"Fetching data from ONTAP: {url}")
+            response = await client.get(url, auth=(cluster.username, cluster.password))
+            response.raise_for_status()
+            results.extend(response.json()["records"])
+        except httpx.HTTPError as e:
+            logger.error(f"HTTP error occurred: {e}")
+            return None
+    return results
+
 
 def setup_logging() -> None:
     """Configure logging for the application"""
-    logging.basicConfig(
+    logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s] [%(levelname)5s] %(message)s")
-        level=logging.DEBUG,
+    print("Logger is initialized.")
-        format="[%(asctime)s] [%(levelname)5s] %(message)s"
-    )
-    print(f"Logger is initialized.")