From 774fa3484c44095a4a4ad30277b34b10480076b6 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 18 Sep 2025 09:04:08 +0200 Subject: [PATCH 1/6] basic creds schema --- src/example/schema.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/example/schema.py b/src/example/schema.py index 47392d6..d0ae38a 100644 --- a/src/example/schema.py +++ b/src/example/schema.py @@ -5,3 +5,11 @@ from pydantic import BaseModel class ExampleSchema(BaseModel): example_field: str another_field: int + +class ClusterCreds(BaseModel): + """A structure to hold basic auth cluster credentials for a cluster""" + username: str + password: str + hostname: str = None + cert_filepath: Path = None + key_filepath: Path = None From cf09ba643158c10c69c7cf2cfa1c5c1a12eaafcc Mon Sep 17 00:00:00 2001 From: root Date: Thu, 18 Sep 2025 09:39:36 +0200 Subject: [PATCH 2/6] req file extended --- initialize.py | 0 requirements.txt | 5 ++++- src/start.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 initialize.py create mode 100644 src/start.py diff --git a/initialize.py b/initialize.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index 8af5e42..5935683 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ fastapi[standard]>=0.116.2 httpx>=0.28.1 redis>=6.4.0 -python-dotenv>=1.1.1 \ No newline at end of file +python-dotenv>=1.1.1 +pydantic +redis[hiredis] +dotenv diff --git a/src/start.py b/src/start.py new file mode 100644 index 0000000..19971bf --- /dev/null +++ b/src/start.py @@ -0,0 +1,38 @@ +import os +import json +from pathlib import Path +from dotenv import load_dotenv +from redis import Redis + +from pydantic import BaseModel, ValidationError, SecretStr, AnyHttpUrl +from typing import Optional, Literal, List, Union + + +def initialize_config(): + load_dotenv() + MY_ENV_VAR = os.getenv('MY_ENV_VAR') + print(MY_ENV_VAR) + +def config_init(redisclient: Redis): + print(f'[INFO] Importing configuration to local DB...') + try: + #redisclient = Redis(host=redishost, port=redisport, decode_responses=True) + inv = { + '1': {'hostname': 'jamaica.muccbc.hq.netapp.com', 'username': '', 'password': ''}, + '2': {'hostname': 'trinidad.muccbc.hq.netapp.com', 'username': '', 'password': ''} + } + + inventory = json.dumps(inv) + redisclient.hset('cluster_inventory', mapping={'inventory': inventory}) + redisclient.close() + return True + + except Exception as e: + print(f"FATAL: Redis DB error: {e}") + return False + +load_dotenv() + +MY_ENV_VAR = os.getenv('MY_ENV_VAR') + +print(MY_ENV_VAR) \ No newline at end of file From 22419ecf845d15d0c28d1635b76e36f8b3a9514f Mon Sep 17 00:00:00 2001 From: root Date: Thu, 18 Sep 2025 11:45:51 +0200 Subject: [PATCH 3/6] read and write with readis --- Dockerfile | 9 +++++ config/inventory.yml | 8 +++++ src/database.py | 30 +++++++++++++++++ src/main.py | 78 ++++++++++++++++++++++++++++++++------------ src/utils.py | 9 +++++ 5 files changed, 113 insertions(+), 21 deletions(-) create mode 100644 Dockerfile create mode 100644 config/inventory.yml create mode 100644 src/database.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..15be366 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM python:latest + +WORKDIR /usr/local/bin +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +COPY src/start.py . + +CMD ["src/start.py"] \ No newline at end of file diff --git a/config/inventory.yml b/config/inventory.yml new file mode 100644 index 0000000..6433930 --- /dev/null +++ b/config/inventory.yml @@ -0,0 +1,8 @@ +- 1: + hostname: '172.16.57.2' + username: 'admin' + password: 'Netapp12' +- 2: + hostname: '172.16.56.2' + username: 'admin' + password: 'Netapp12' diff --git a/src/database.py b/src/database.py new file mode 100644 index 0000000..17d53c5 --- /dev/null +++ b/src/database.py @@ -0,0 +1,30 @@ +import json +import logging +from redis import Redis, ConnectionError + + +def setup_db_conn(redishost, redisport: str): + ''' Setup Redis connection and return it open''' + log = logging.getLogger('uvicorn') + try: + redisclient = Redis(host=redishost, port=redisport, decode_responses=True) + log.info(f"Connected to Redis DB {redishost} on port {redisport}") + return redisclient + except ConnectionError as e: + print(f"FATAL: Redis DB {redishost} is unreachable on port {redisport}. Err: {e}") + return None + except Exception as e: + print(f"FATAL: {e}") + return None + +def get_inventory_from_redis(redisclient: Redis): + ''' Read inventory from Redis ''' + cluster_inv = redisclient.hgetall('cluster_inventory') + if 'inventory' in cluster_inv: + return json.loads(cluster_inv['inventory']) + return {} + +def read_config_from_db(redisclient: Redis): + ''' Load inventory to global vars''' + global_inventory = get_inventory_from_redis(redisclient) + return global_inventory \ No newline at end of file diff --git a/src/main.py b/src/main.py index ea4abdf..294f7e2 100644 --- a/src/main.py +++ b/src/main.py @@ -1,35 +1,71 @@ +import os +import json import logging +import yaml -from fastapi import FastAPI +from pathlib import Path +from dotenv import load_dotenv +from redis import Redis from contextlib import asynccontextmanager -import httpx -from src.aggregate import aggregate_router -from src.service import load_config +from pydantic import BaseModel, ValidationError, SecretStr, AnyHttpUrl +from typing import Optional, Literal, List, Union +from fastapi import FastAPI -logger = logging.getLogger("uvicorn") -logger.info("Starting application") -config = load_config() +from database import setup_db_conn, get_inventory_from_redis, read_config_from_db +from utils import setup_logging + +def initialize_config(): + load_dotenv() + ENV_INVENTORYPATH = os.getenv('cluster_inventory_path') + ENV_REDISHOST = os.getenv('redis_host') + ENV_REDISPORT = os.getenv('redis_port') + + log.info(f"Fount Cluster Inventory file at: {ENV_INVENTORYPATH}") + if not ENV_INVENTORYPATH or not Path(ENV_INVENTORYPATH).is_file(): + print(f"FATAL: Inventory file {ENV_INVENTORYPATH} is missing or not a file.") + return False + try: + with open(ENV_INVENTORYPATH, 'r') as f: + inv = yaml.safe_load(f) + inventory = json.dumps(inv) + except Exception as e: + print(f"FATAL: Cannot read inventory file {ENV_INVENTORYPATH}. Err: {e}") + return False + + print(f'[INFO] Importing configuration to DB...') + try: + redis_conn = setup_db_conn(ENV_REDISHOST, ENV_REDISPORT) + redis_conn.hset('cluster_inventory', mapping={'inventory': inventory}) + redis_conn.close() + + log.info("Configuration has been loaded.") + return True + + except Exception as e: + print(f"FATAL: Redis DB error: {e}") + return False @asynccontextmanager async def lifespan(app: FastAPI): - app.requests_client = httpx.AsyncClient(verify=False) + ''' make loading it async''' + log = logging.getLogger('uvicorn') + cfg_init_result = initialize_config() + + inv_check = read_config_from_db(setup_db_conn(os.getenv('redis_host'), os.getenv('redis_port'))) + log.info(f"Data validity check (DEVELOPER MODE): {inv_check}") + if not cfg_init_result: + log.error("Configuration initialization failed. Exiting...") + exit(1) + yield - await app.requests_client.aclose() + log.info("Shutting down FastAPI app...") +setup_logging() +log = logging.getLogger('uvicorn') + +log.info("Starting FastAPI app...") app = FastAPI(lifespan=lifespan) -app.include_router(aggregate_router) - - -@app.get("/") -async def main(): - return {"Hello": "World"} - - -@app.get("/config") -async def get_config(): - """Endpoint to get the current configuration.""" - return config.model_dump() diff --git a/src/utils.py b/src/utils.py index febeea5..8ab9e69 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,3 +1,4 @@ +import logging import httpx def round_bytes(size_in_bytes: int) -> str: @@ -22,3 +23,11 @@ async def get_data_from_ontap(client, logger, hostname: str, username: str, pass except httpx.HTTPError as e: logger.error(f"HTTP error occurred: {e}") return None + +def setup_logging() -> None: + """Configure logging for the application""" + logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] [%(levelname)5s] %(message)s" + ) + print(f"Logger is initialized.") From d90a18053f81daf2b2436f044cbbee9c74a8d646 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 18 Sep 2025 12:04:10 +0200 Subject: [PATCH 4/6] restructure --- src/initialize.py | 40 ++++++++++++++++++++++++++++++++++++++++ src/main.py | 32 +------------------------------- src/start.py | 38 -------------------------------------- 3 files changed, 41 insertions(+), 69 deletions(-) create mode 100644 src/initialize.py delete mode 100644 src/start.py diff --git a/src/initialize.py b/src/initialize.py new file mode 100644 index 0000000..7621f98 --- /dev/null +++ b/src/initialize.py @@ -0,0 +1,40 @@ +import os +import json +import logging +import yaml + +from pathlib import Path +from dotenv import load_dotenv +from database import setup_db_conn + +def initialize_config(): + load_dotenv() + log = logging.getLogger('uvicorn') + ENV_INVENTORYPATH = os.getenv('cluster_inventory_path') + ENV_REDISHOST = os.getenv('redis_host') + ENV_REDISPORT = os.getenv('redis_port') + + log.info(f"Fount Cluster Inventory file at: {ENV_INVENTORYPATH}") + if not ENV_INVENTORYPATH or not Path(ENV_INVENTORYPATH).is_file(): + print(f"FATAL: Inventory file {ENV_INVENTORYPATH} is missing or not a file.") + return False + try: + with open(ENV_INVENTORYPATH, 'r') as f: + inv = yaml.safe_load(f) + inventory = json.dumps(inv) + except Exception as e: + print(f"FATAL: Cannot read inventory file {ENV_INVENTORYPATH}. Err: {e}") + return False + + print(f'[INFO] Importing configuration to DB...') + try: + redis_conn = setup_db_conn(ENV_REDISHOST, ENV_REDISPORT) + redis_conn.hset('cluster_inventory', mapping={'inventory': inventory}) + redis_conn.close() + + log.info("Configuration has been loaded.") + return True + + except Exception as e: + print(f"FATAL: Redis DB error: {e}") + return False \ No newline at end of file diff --git a/src/main.py b/src/main.py index 294f7e2..3a197a9 100644 --- a/src/main.py +++ b/src/main.py @@ -14,40 +14,10 @@ from fastapi import FastAPI from database import setup_db_conn, get_inventory_from_redis, read_config_from_db +from initialize import initialize_config from utils import setup_logging -def initialize_config(): - load_dotenv() - ENV_INVENTORYPATH = os.getenv('cluster_inventory_path') - ENV_REDISHOST = os.getenv('redis_host') - ENV_REDISPORT = os.getenv('redis_port') - - log.info(f"Fount Cluster Inventory file at: {ENV_INVENTORYPATH}") - if not ENV_INVENTORYPATH or not Path(ENV_INVENTORYPATH).is_file(): - print(f"FATAL: Inventory file {ENV_INVENTORYPATH} is missing or not a file.") - return False - try: - with open(ENV_INVENTORYPATH, 'r') as f: - inv = yaml.safe_load(f) - inventory = json.dumps(inv) - except Exception as e: - print(f"FATAL: Cannot read inventory file {ENV_INVENTORYPATH}. Err: {e}") - return False - - print(f'[INFO] Importing configuration to DB...') - try: - redis_conn = setup_db_conn(ENV_REDISHOST, ENV_REDISPORT) - redis_conn.hset('cluster_inventory', mapping={'inventory': inventory}) - redis_conn.close() - - log.info("Configuration has been loaded.") - return True - - except Exception as e: - print(f"FATAL: Redis DB error: {e}") - return False - @asynccontextmanager async def lifespan(app: FastAPI): ''' make loading it async''' diff --git a/src/start.py b/src/start.py deleted file mode 100644 index 19971bf..0000000 --- a/src/start.py +++ /dev/null @@ -1,38 +0,0 @@ -import os -import json -from pathlib import Path -from dotenv import load_dotenv -from redis import Redis - -from pydantic import BaseModel, ValidationError, SecretStr, AnyHttpUrl -from typing import Optional, Literal, List, Union - - -def initialize_config(): - load_dotenv() - MY_ENV_VAR = os.getenv('MY_ENV_VAR') - print(MY_ENV_VAR) - -def config_init(redisclient: Redis): - print(f'[INFO] Importing configuration to local DB...') - try: - #redisclient = Redis(host=redishost, port=redisport, decode_responses=True) - inv = { - '1': {'hostname': 'jamaica.muccbc.hq.netapp.com', 'username': '', 'password': ''}, - '2': {'hostname': 'trinidad.muccbc.hq.netapp.com', 'username': '', 'password': ''} - } - - inventory = json.dumps(inv) - redisclient.hset('cluster_inventory', mapping={'inventory': inventory}) - redisclient.close() - return True - - except Exception as e: - print(f"FATAL: Redis DB error: {e}") - return False - -load_dotenv() - -MY_ENV_VAR = os.getenv('MY_ENV_VAR') - -print(MY_ENV_VAR) \ No newline at end of file From fe13e49172db27668a4aea9909a41f8f5ae5a7fa Mon Sep 17 00:00:00 2001 From: root Date: Thu, 18 Sep 2025 12:11:52 +0200 Subject: [PATCH 5/6] dev schema --- src/database.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/database.py b/src/database.py index 17d53c5..1c5c842 100644 --- a/src/database.py +++ b/src/database.py @@ -24,7 +24,23 @@ def get_inventory_from_redis(redisclient: Redis): return json.loads(cluster_inv['inventory']) return {} -def read_config_from_db(redisclient: Redis): +def read_config_from_db(redisclient: Redis) -> ConfigSchema: ''' Load inventory to global vars''' - global_inventory = get_inventory_from_redis(redisclient) - return global_inventory \ No newline at end of file + GLOBAL_INVENTORY = get_inventory_from_redis(redisclient) + for item in GLOBAL_INVENTORY: + config = ConfigSchema( + hostname=GLOBAL_INVENTORY[item]['hostname'], + username=GLOBAL_INVENTORY[item]['username'], + password=GLOBAL_INVENTORY[item]['password'], + ) + GLOBAL_INVENTORY += {item: config} + return GLOBAL_INVENTORY + +# def load_config() -> ConfigSchema: +# logger.info("Loading config from .env file") +# config = dotenv_values(".env") +# return ConfigSchema( +# hostname=config["CLUSTER1_HOSTNAME"], +# username=config["CLUSTER1_USERNAME"], +# password=config["CLUSTER1_PASSWORD"], +# ) \ No newline at end of file From 72f738a8164634bab394dfa48785c225db375da2 Mon Sep 17 00:00:00 2001 From: Alexey Date: Thu, 18 Sep 2025 14:00:05 +0200 Subject: [PATCH 6/6] backedn dev --- src/database.py | 30 ++++++++++++------------------ src/initialize.py | 8 ++++++-- src/main.py | 17 +++++++++++------ 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/database.py b/src/database.py index 1c5c842..a278d98 100644 --- a/src/database.py +++ b/src/database.py @@ -1,6 +1,9 @@ import json import logging from redis import Redis, ConnectionError +from typing import List +from pydantic import TypeAdapter +from schema import ConfigSchema def setup_db_conn(redishost, redisport: str): @@ -8,7 +11,11 @@ def setup_db_conn(redishost, redisport: str): log = logging.getLogger('uvicorn') try: redisclient = Redis(host=redishost, port=redisport, decode_responses=True) - log.info(f"Connected to Redis DB {redishost} on port {redisport}") + if redisclient.ping(): + log.info(f"Connected to Redis DB {redishost} on port {redisport}") + else: + log.error(f"Cannot connect to Redis DB {redishost} on port {redisport}") + exit(1) return redisclient except ConnectionError as e: print(f"FATAL: Redis DB {redishost} is unreachable on port {redisport}. Err: {e}") @@ -24,23 +31,10 @@ def get_inventory_from_redis(redisclient: Redis): return json.loads(cluster_inv['inventory']) return {} -def read_config_from_db(redisclient: Redis) -> ConfigSchema: +def get_config_from_db(redisclient: Redis) -> ConfigSchema: ''' Load inventory to global vars''' GLOBAL_INVENTORY = get_inventory_from_redis(redisclient) - for item in GLOBAL_INVENTORY: - config = ConfigSchema( - hostname=GLOBAL_INVENTORY[item]['hostname'], - username=GLOBAL_INVENTORY[item]['username'], - password=GLOBAL_INVENTORY[item]['password'], - ) - GLOBAL_INVENTORY += {item: config} - return GLOBAL_INVENTORY -# def load_config() -> ConfigSchema: -# logger.info("Loading config from .env file") -# config = dotenv_values(".env") -# return ConfigSchema( -# hostname=config["CLUSTER1_HOSTNAME"], -# username=config["CLUSTER1_USERNAME"], -# password=config["CLUSTER1_PASSWORD"], -# ) \ No newline at end of file + GLOBAL_INVENTORY_VALID = TypeAdapter(List[ConfigSchema]).validate_python(GLOBAL_INVENTORY) + + return GLOBAL_INVENTORY_VALID diff --git a/src/initialize.py b/src/initialize.py index 7621f98..dcdd84d 100644 --- a/src/initialize.py +++ b/src/initialize.py @@ -6,6 +6,9 @@ import yaml from pathlib import Path from dotenv import load_dotenv from database import setup_db_conn +from schema import ConfigSchema +from typing import List +from pydantic import TypeAdapter def initialize_config(): load_dotenv() @@ -14,7 +17,7 @@ def initialize_config(): ENV_REDISHOST = os.getenv('redis_host') ENV_REDISPORT = os.getenv('redis_port') - log.info(f"Fount Cluster Inventory file at: {ENV_INVENTORYPATH}") + log.info(f"Found Cluster Inventory file at: {ENV_INVENTORYPATH}") if not ENV_INVENTORYPATH or not Path(ENV_INVENTORYPATH).is_file(): print(f"FATAL: Inventory file {ENV_INVENTORYPATH} is missing or not a file.") return False @@ -28,11 +31,12 @@ def initialize_config(): print(f'[INFO] Importing configuration to DB...') try: + GLOBAL_INVENTORY_VALID = TypeAdapter(List[ConfigSchema]).validate_python(inv) redis_conn = setup_db_conn(ENV_REDISHOST, ENV_REDISPORT) redis_conn.hset('cluster_inventory', mapping={'inventory': inventory}) redis_conn.close() - log.info("Configuration has been loaded.") + log.info("Configuration has been loaded.") return True except Exception as e: diff --git a/src/main.py b/src/main.py index 3a197a9..bec035d 100644 --- a/src/main.py +++ b/src/main.py @@ -13,9 +13,9 @@ from typing import Optional, Literal, List, Union from fastapi import FastAPI -from database import setup_db_conn, get_inventory_from_redis, read_config_from_db -from initialize import initialize_config -from utils import setup_logging +from database import setup_db_conn, get_inventory_from_redis, get_config_from_db +from src.initialize import initialize_config +from utils import setup_logging @asynccontextmanager @@ -23,12 +23,17 @@ async def lifespan(app: FastAPI): ''' make loading it async''' log = logging.getLogger('uvicorn') cfg_init_result = initialize_config() + + shared_redis_conn = setup_db_conn(os.getenv('redis_host'), os.getenv('redis_port')) + if not shared_redis_conn: + log.error("Cannot connect to Redis DB. Exiting...") + exit(1) - inv_check = read_config_from_db(setup_db_conn(os.getenv('redis_host'), os.getenv('redis_port'))) - log.info(f"Data validity check (DEVELOPER MODE): {inv_check}") + inv_check = get_config_from_db(shared_redis_conn) + log.info(f"[DEBUG] Data validity healthcheck (DEVELOPER MODE): {inv_check}") if not cfg_init_result: log.error("Configuration initialization failed. Exiting...") - exit(1) + # exit(1) yield log.info("Shutting down FastAPI app...")