From cca5d924f00d87e65616f226256dc81bd05de874 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 23 Jul 2024 12:50:46 +0200 Subject: [PATCH 01/54] first draft of version 1.0.0 --- README.md | 1 + exaapi/README.md | 21 -------------- exaapi/config.example.py | 19 ------------ exaapi/exa_api_http.py | 31 -------------------- exaapi/exa_api_logger.py | 12 -------- exaapi/exa_api_rabbit.py | 63 ---------------------------------------- exaapi/rabbit_manual.py | 24 --------------- flowapp/__about__.py | 2 +- guarda/README.md | 16 ---------- guarda/config.example.py | 7 ----- guarda/guarda.py | 8 ----- guarda/guarda.service | 15 ---------- setup.py | 12 +++++--- 13 files changed, 10 insertions(+), 221 deletions(-) delete mode 100644 exaapi/README.md delete mode 100644 exaapi/config.example.py delete mode 100755 exaapi/exa_api_http.py delete mode 100644 exaapi/exa_api_logger.py delete mode 100644 exaapi/exa_api_rabbit.py delete mode 100644 exaapi/rabbit_manual.py delete mode 100644 guarda/README.md delete mode 100644 guarda/config.example.py delete mode 100755 guarda/guarda.py delete mode 100644 guarda/guarda.service diff --git a/README.md b/README.md index ba8b1f78..4e0e4891 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Last part of the system is Guarda service. This systemctl service is running in * [Local database instalation notes](./docs/DB_LOCAL.md) ## Change Log +- 1.0.0 . DRAFT - ExaAPI and Guarda modules moved to their own repositories. New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. - 0.8.0 - API keys update. **Run migration scripts to update your DB**. Keys can now have expiration date and readonly flag. Admin can create special keys for certain machinnes. - 0.7.3 - New possibility of external auth proxy. - 0.7.2 - Dashboard and Main menu are now customizable in config. App is ready to be packaged using setup.py. diff --git a/exaapi/README.md b/exaapi/README.md deleted file mode 100644 index cd0fa712..00000000 --- a/exaapi/README.md +++ /dev/null @@ -1,21 +0,0 @@ -#ExaAPI web app - -This is a very simple web application, which needs to be hooked on ExaBGP daemon. Every time this app -gets a new command, it replicates the command to the daemon through the stdout. The registered -daemon is watching the stdout of the ExaAPI service. - -Add this to your ExaBGP config -``` -process flowspec { - run /usr/bin/python3 /home/deploy/www/exaapi/exa_api.py; - encoder json; - } -``` - -It can run on the development Flask server, however there is no security layer in this app. -You should limit the access only from the localhost. - -See [ExaBPG docs](https://github.com/Exa-Networks/exabgp/wiki/Controlling-ExaBGP-:-possible-options-for-process) for more information. - -Our plan is to relace this simple app with message queue in the future. - diff --git a/exaapi/config.example.py b/exaapi/config.example.py deleted file mode 100644 index 06b3c47b..00000000 --- a/exaapi/config.example.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Example of configuration file - -Add your log settings and rename to config.py -""" - -LOG_FILE = "/var/log/exafs/exa_api.log" -LOG_FORMAT = "%(asctime)s: %(message)s" - - -# rabbit mq -# note - rabbit mq must be enabled in main app config -# credentials and queue must be set here for the same values -EXA_API_RABBIT_HOST = "localhost" -EXA_API_RABBIT_PORT = "5672" -EXA_API_RABBIT_PASS = "mysecurepassword" -EXA_API_RABBIT_USER = "myexaapiuser" -EXA_API_RABBIT_VHOST = "/" -EXA_API_RABBIT_QUEUE = "my_exa_api_queue" diff --git a/exaapi/exa_api_http.py b/exaapi/exa_api_http.py deleted file mode 100755 index 2726055b..00000000 --- a/exaapi/exa_api_http.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -""" -ExaBGP HTTP API process -This module is process for ExaBGP -https://github.com/Exa-Networks/exabgp/wiki/Controlling-ExaBGP-:-possible-options-for-process - -Each command received in the POST request is send to stdout and captured by ExaBGP. -""" - -from flask import Flask, request -from sys import stdout - -import exa_api_logger - -app = Flask(__name__) - -logger = exa_api_logger.create() - - -@app.route("/", methods=["POST"]) -def command(): - cmd = request.form["command"] - logger.info(cmd) - stdout.write("%s\n" % cmd) - stdout.flush() - - return "%s\n" % cmd - - -if __name__ == "__main__": - app.run() diff --git a/exaapi/exa_api_logger.py b/exaapi/exa_api_logger.py deleted file mode 100644 index 1c5bb4dd..00000000 --- a/exaapi/exa_api_logger.py +++ /dev/null @@ -1,12 +0,0 @@ -import logging -import config - - -def create(): - logger = logging.getLogger(__name__) - f_format = logging.Formatter(config.LOG_FORMAT) - f_handler = logging.FileHandler(config.LOG_FILE) - f_handler.setFormatter(f_format) - logger.setLevel(logging.INFO) - logger.addHandler(f_handler) - return logger diff --git a/exaapi/exa_api_rabbit.py b/exaapi/exa_api_rabbit.py deleted file mode 100644 index f25c2b89..00000000 --- a/exaapi/exa_api_rabbit.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -""" -ExaBGP RabbitMQ API process -This module is process for ExaBGP -https://github.com/Exa-Networks/exabgp/wiki/Controlling-ExaBGP-:-possible-options-for-process - -Each command received from the queue is send to stdout and captured by ExaBGP. -""" -import pika -import sys -import os -from time import sleep - -import config -import exa_api_logger - -logger = exa_api_logger.create() - - -def callback(ch, method, properties, body): - body = body.decode("utf-8") - logger.info(body) - sys.stdout.write("%s\n" % body) - sys.stdout.flush() - - -def main(): - while True: - user = config.EXA_API_RABBIT_USER - passwd = config.EXA_API_RABBIT_PASS - queue = config.EXA_API_RABBIT_QUEUE - credentials = pika.PlainCredentials(user, passwd) - parameters = pika.ConnectionParameters( - config.EXA_API_RABBIT_HOST, - config.EXA_API_RABBIT_PORT, - config.EXA_API_RABBIT_VHOST, - credentials, - ) - connection = pika.BlockingConnection(parameters) - channel = connection.channel() - - channel.queue_declare(queue=queue) - - channel.basic_consume(queue=queue, on_message_callback=callback, auto_ack=True) - - print(" [*] Waiting for messages. To exit press CTRL+C") - try: - channel.start_consuming() - except KeyboardInterrupt: - channel.stop_consuming() - connection.close() - print("\nInterrupted") - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except pika.exceptions.ConnectionClosedByBroker: - sleep(15) - continue - - -if __name__ == "__main__": - main() diff --git a/exaapi/rabbit_manual.py b/exaapi/rabbit_manual.py deleted file mode 100644 index 0e2b2ce5..00000000 --- a/exaapi/rabbit_manual.py +++ /dev/null @@ -1,24 +0,0 @@ -import pika -import sys - -import config - -user = config.EXA_API_RABBIT_USER -passwd = config.EXA_API_RABBIT_PASS -queue = config.EXA_API_RABBIT_QUEUE -credentials = pika.PlainCredentials(user, passwd) -parameters = pika.ConnectionParameters( - config.EXA_API_RABBIT_HOST, - config.EXA_API_RABBIT_PORT, - config.EXA_API_RABBIT_VHOST, - credentials, -) - -connection = pika.BlockingConnection(parameters) -channel = connection.channel() -channel.queue_declare(queue=queue) -route = sys.argv[1] - -print("got :", route) - -channel.basic_publish(exchange="", routing_key=queue, body=route) diff --git a/flowapp/__about__.py b/flowapp/__about__.py index 59b7f434..1c11a6ec 100755 --- a/flowapp/__about__.py +++ b/flowapp/__about__.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "1.0.0" diff --git a/guarda/README.md b/guarda/README.md deleted file mode 100644 index 7565640d..00000000 --- a/guarda/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Guarda service for ExaBGP - -## As root - -Edit guarda.service file and set correct location of guarda.py. - -Then edit guarda.py and set address of your host. - - -```bash -pip install requests -chmod +x guarda.py -cp guarda.service /usr/lib/systemd/system/guarda.service -systemctl start guarda.service -systemctl enable guarda.service -``` diff --git a/guarda/config.example.py b/guarda/config.example.py deleted file mode 100644 index fc42e34c..00000000 --- a/guarda/config.example.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Example of configuration file - -Add your application URL and rename to config.py -""" - -URL = 'http://127.0.0.1/rules/announce_all' \ No newline at end of file diff --git a/guarda/guarda.py b/guarda/guarda.py deleted file mode 100755 index 262ecc5e..00000000 --- a/guarda/guarda.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/python3 - -import requests -import time -import config - -time.sleep(10) -requests.get(config.URL) diff --git a/guarda/guarda.service b/guarda/guarda.service deleted file mode 100644 index d4cdc756..00000000 --- a/guarda/guarda.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=ExaBGP restart guardian -After=exabgp.service -Requires=exabgp.service -PartOf=exabgp.service -ConditionPathExists=/home/deploy/www/guarda/guarda.py - -[Service] -Type=simple -ExecStart=/usr/bin/python3.6 /home/deploy/www/guarda/guarda.py -StandardOutput=syslog -StandardError=syslog - -[Install] -WantedBy=exabgp.service diff --git a/setup.py b/setup.py index 84584009..b10fb667 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,7 @@ """ -Author(s): Jakub Man +Author(s): +Jiri Vrany +Jakub Man Setuptools configuration """ @@ -13,15 +15,17 @@ setuptools.setup( name="exafs", - version=__version__, # noqa: F821 + version=__version__, # noqa: F821 author="CESNET / Jiri Vrany, Petr Adamec, Josef Verich, Jakub Man", description="Tool for creation, validation, and execution of ExaBGP messages.", url="https://github.com/CESNET/exafs", license="MIT", - py_modules=["flowapp", "exaapi"], + py_modules=[ + "flowapp", + ], packages=setuptools.find_packages(), include_package_data=True, - python_requires=">=3.8", + python_requires=">=3.11", install_requires=[ "Flask>=2.0.2", "Flask-SQLAlchemy>=2.2", From f5720f18fa6003c2ef19cdbedf2de92923e8cb7a Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 10:18:59 +0200 Subject: [PATCH 02/54] why cs in sso map ? --- flowapp/__init__.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/flowapp/__init__.py b/flowapp/__init__.py index f92aeeae..8bdfc16b 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -22,7 +22,6 @@ def create_app(): #: Default attribute map SSO_ATTRIBUTE_MAP = { "eppn": (True, "eppn"), - "cn": (False, "cn"), } # db.init_app(app) @@ -85,7 +84,7 @@ def logout(): @app.route("/ext-login") def ext_login(): - header_name = app.config.get("AUTH_HEADER_NAME", 'X-Authenticated-User') + header_name = app.config.get("AUTH_HEADER_NAME", "X-Authenticated-User") if header_name not in request.headers: return render_template("errors/401.html") @@ -148,9 +147,7 @@ def internal_error(exception): def utility_processor(): def editable_rule(rule): if rule: - validators.editable_range( - rule, models.get_user_nets(session["user_id"]) - ) + validators.editable_range(rule, models.get_user_nets(session["user_id"])) return True return False @@ -176,7 +173,7 @@ def inject_dashboard(): def format_datetime(value): if value is None: return app.config.get("MISSING_DATETIME_MESSAGE", "Never") - + format = "y/MM/dd HH:mm" return babel.dates.format_datetime(value, format) @@ -187,9 +184,7 @@ def _register_user_to_session(uuid: str): session["user_name"] = user.name session["user_id"] = user.id session["user_roles"] = [role.name for role in user.role.all()] - session["user_orgs"] = ", ".join( - org.name for org in user.organization.all() - ) + session["user_orgs"] = ", ".join(org.name for org in user.organization.all()) session["user_role_ids"] = [role.id for role in user.role.all()] session["user_org_ids"] = [org.id for org in user.organization.all()] roles = [i > 1 for i in session["user_role_ids"]] From 7ce6d487881ba3c8ec77671b9a4aa2437137e104 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 13:40:39 +0200 Subject: [PATCH 03/54] SSO Attribute map now in config directly --- config.example.py | 10 ++++++++-- flowapp/__init__.py | 7 ------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/config.example.py b/config.example.py index c4fa7a8d..d091c1cf 100644 --- a/config.example.py +++ b/config.example.py @@ -1,4 +1,4 @@ -class Config(): +class Config: """ Default config options """ @@ -16,7 +16,7 @@ class Config(): # Name of HTTP header containing the UUID of authenticated user. # Only used when HEADER_AUTH is set to True - AUTH_HEADER_NAME = 'X-Authenticated-User' + AUTH_HEADER_NAME = "X-Authenticated-User" # SSO LOGOUT LOGOUT_URL = "https://flowspec.example.com/Shibboleth.sso/Logout" # SQL Alchemy config @@ -77,6 +77,12 @@ class ProductionConfig(Config): LOCAL_IP = "127.0.0.1" # SSO AUTH enabled in produciion SSO_AUTH = True + # Map SSO attributes from ADFS to session keys under session['user'] + SSO_ATTRIBUTE_MAP = { + "eppn": (True, "eppn"), + } + SSO_LOGIN_URL = "/login" + # Set true if you need debug in production DEBUG = False diff --git a/flowapp/__init__.py b/flowapp/__init__.py index 8bdfc16b..dab0ca18 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -18,11 +18,6 @@ def create_app(): app = Flask(__name__) - # Map SSO attributes from ADFS to session keys under session['user'] - #: Default attribute map - SSO_ATTRIBUTE_MAP = { - "eppn": (True, "eppn"), - } # db.init_app(app) migrate.init_app(app, db) @@ -32,8 +27,6 @@ def create_app(): app.config.from_object(InstanceConfig) app.config.setdefault("VERSION", __version__) - app.config.setdefault("SSO_ATTRIBUTE_MAP", SSO_ATTRIBUTE_MAP) - app.config.setdefault("SSO_LOGIN_URL", "/login") # This attaches the *flask_sso* login handler to the SSO_LOGIN_URL, ext = SSO(app=app) From b43c369c6e826682aac1950b709263ce5fe78285 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:01:54 +0200 Subject: [PATCH 04/54] debug SSO --- flowapp/__init__.py | 4 ++++ flowapp/auth.py | 16 ++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/flowapp/__init__.py b/flowapp/__init__.py index dab0ca18..0a393a52 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -28,6 +28,10 @@ def create_app(): app.config.setdefault("VERSION", __version__) + print("Config loaded") + for key, value in app.config.items(): + print(f"{key} = {value}") + # This attaches the *flask_sso* login handler to the SSO_LOGIN_URL, ext = SSO(app=app) diff --git a/flowapp/auth.py b/flowapp/auth.py index c4d942ff..c1426c5a 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -15,6 +15,7 @@ def auth_required(f): def decorated(*args, **kwargs): if not check_auth(get_user()): if current_app.config.get("SSO_AUTH"): + print("SSO AUTH SET BUT FAILS") return redirect("/login") elif current_app.config.get("HEADER_AUTH", False): return redirect("/ext-login") @@ -62,11 +63,7 @@ def decorated(*args, **kwargs): localv4 = current_app.config.get("LOCAL_IP") localv6 = current_app.config.get("LOCAL_IP6") if remote != localv4 and remote != localv6: - print( - "AUTH LOCAL ONLY FAIL FROM {} / local adresses [{}, {}]".format( - remote, localv4, localv6 - ) - ) + print("AUTH LOCAL ONLY FAIL FROM {} / local adresses [{}, {}]".format(remote, localv4, localv6)) abort(403) # Forbidden return f(*args, **kwargs) @@ -97,6 +94,7 @@ def check_auth(uuid): session["app_version"] = __version__ if current_app.config.get("SSO_AUTH"): + print("CHECK AUTH, SS AUTH SET", uuid) # SSO AUTH exist = False if uuid: @@ -104,18 +102,16 @@ def check_auth(uuid): return exist elif current_app.config.get("HEADER_AUTH", False): # External auth (for example apache) - header_name = current_app.config.get("AUTH_HEADER_NAME", 'X-Authenticated-User') + header_name = current_app.config.get("AUTH_HEADER_NAME", "X-Authenticated-User") if header_name not in request.headers or not session.get("user_uuid"): return False return db.session.query(User).filter_by(uuid=request.headers.get(header_name)) else: - # Localhost login / no check + # Localhost login / no check / works only in development environment session["user_email"] = current_app.config["LOCAL_USER_UUID"] session["user_id"] = current_app.config["LOCAL_USER_ID"] session["user_roles"] = current_app.config["LOCAL_USER_ROLES"] - session["user_orgs"] = ", ".join( - org["name"] for org in current_app.config["LOCAL_USER_ORGS"] - ) + session["user_orgs"] = ", ".join(org["name"] for org in current_app.config["LOCAL_USER_ORGS"]) session["user_role_ids"] = current_app.config["LOCAL_USER_ROLE_IDS"] session["user_org_ids"] = current_app.config["LOCAL_USER_ORG_IDS"] roles = [i > 1 for i in session["user_role_ids"]] From 8bafef2a3c7384597d24d7422751d1372bb18c06 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:19:37 +0200 Subject: [PATCH 05/54] debug SSO --- flowapp/instance_config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 9d5a1bfa..96444f20 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -125,3 +125,10 @@ class InstanceConfig: } COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} + + SSO_ATTRIBUTE_MAP = { + "eppn": (True, "eppn"), + "HTTP_X_EPPN": (True, "eppn"), + } + + SSO_LOGIN_URL = "/login" From cb88705bbd08d42cbe49469f13cdfdf53c864a32 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:24:07 +0200 Subject: [PATCH 06/54] debug SSO --- flowapp/instance_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 96444f20..52c1af61 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -127,7 +127,7 @@ class InstanceConfig: COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} SSO_ATTRIBUTE_MAP = { - "eppn": (True, "eppn"), + # "eppn": (True, "eppn"), "HTTP_X_EPPN": (True, "eppn"), } From bc41a77dbccac94d07ce117776855865eb91825e Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:31:41 +0200 Subject: [PATCH 07/54] debug SSO --- flowapp/instance_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 52c1af61..8c7817c3 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -128,7 +128,7 @@ class InstanceConfig: SSO_ATTRIBUTE_MAP = { # "eppn": (True, "eppn"), - "HTTP_X_EPPN": (True, "eppn"), + "eppn": (True, "HTTP_X_EPPN"), } SSO_LOGIN_URL = "/login" From 6beaf942709fcce135265b1fef6732cb3f3d1fc6 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 19:10:31 +0200 Subject: [PATCH 08/54] update app factory to accept config object as parameter --- config.example.py | 12 ++++++++++-- flowapp/__init__.py | 13 ++++++------- flowapp/instance_config.py | 7 ------- run.example.py | 30 +++++++++++------------------- 4 files changed, 27 insertions(+), 35 deletions(-) diff --git a/config.example.py b/config.example.py index d091c1cf..de127dd2 100644 --- a/config.example.py +++ b/config.example.py @@ -75,16 +75,23 @@ class ProductionConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Productionl Database URI" # Public IP of the production machine LOCAL_IP = "127.0.0.1" + LOCAL_IP6 = "2001:718:ff05:107::155" # SSO AUTH enabled in produciion SSO_AUTH = True - # Map SSO attributes from ADFS to session keys under session['user'] SSO_ATTRIBUTE_MAP = { - "eppn": (True, "eppn"), + "eppn": (False, "eppn"), + "HTTP_X_EPPN": (False, "eppn"), } SSO_LOGIN_URL = "/login" # Set true if you need debug in production DEBUG = False + DEVEL = False + + # Set cookie behavior + SESSION_COOKIE_SECURE = (True,) + SESSION_COOKIE_HTTPONLY = (True,) + SESSION_COOKIE_SAMESITE = ("Lax",) class DevelopmentConfig(Config): @@ -95,6 +102,7 @@ class DevelopmentConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Local Database URI" LOCAL_IP = "127.0.0.1" DEBUG = True + DEVEL = True class TestingConfig(Config): diff --git a/flowapp/__init__.py b/flowapp/__init__.py index 0a393a52..b48b175c 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -14,9 +14,10 @@ db = SQLAlchemy() migrate = Migrate() csrf = CSRFProtect() +ext = SSO() -def create_app(): +def create_app(config_object=None): app = Flask(__name__) # db.init_app(app) @@ -25,15 +26,13 @@ def create_app(): # Load the default configuration for dashboard and main menu app.config.from_object(InstanceConfig) + if config_object: + app.config.from_object(config_object) app.config.setdefault("VERSION", __version__) - print("Config loaded") - for key, value in app.config.items(): - print(f"{key} = {value}") - - # This attaches the *flask_sso* login handler to the SSO_LOGIN_URL, - ext = SSO(app=app) + # Init SSO + ext.init_app(app) from flowapp import models, constants, validators from .views.admin import admin diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 8c7817c3..9d5a1bfa 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -125,10 +125,3 @@ class InstanceConfig: } COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} - - SSO_ATTRIBUTE_MAP = { - # "eppn": (True, "eppn"), - "eppn": (True, "HTTP_X_EPPN"), - } - - SSO_LOGIN_URL = "/login" diff --git a/run.example.py b/run.example.py index 2911b5bd..884739cf 100644 --- a/run.example.py +++ b/run.example.py @@ -2,9 +2,12 @@ This is an example of how to run the application. First copy the file as run.py (or whatever you want) Then edit the file to match your needs. + In general you should not need to edit this example file. Only if you want to configure the application main menu and -dashboard. Or in case that you want to add extensions etc. +dashboard. + +Or in case that you want to add extensions etc. """ from os import environ @@ -13,29 +16,18 @@ import config -# Call app factory -app = create_app() - # Configurations -env = environ.get('EXAFS_ENV', 'Production') +env = environ.get("EXAFS_ENV", "Production") -if env == 'devel': - app.config.from_object(config.DevelopmentConfig) - app.config.update( - DEVEL=True - ) +# Call app factory +if env == "devel": + app = create_app(config.DevelopmentConfig) else: - app.config.from_object(config.ProductionConfig) - app.config.update( - SESSION_COOKIE_SECURE=True, - SESSION_COOKIE_HTTPONLY=True, - SESSION_COOKIE_SAMESITE='Lax', - DEVEL=False - ) + app = create_app(config.ProductionConfig) # init database object db.init_app(app) # run app -if __name__ == '__main__': - app.run(host='::', port=8080, debug=True) +if __name__ == "__main__": + app.run(host="::", port=8080, debug=True) From f65f2be8c6ceee972be8b88c2b73fc3da7e69ee5 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 19:12:50 +0200 Subject: [PATCH 09/54] update app factory to accept config object as parameter --- config.example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.example.py b/config.example.py index de127dd2..1583cfa2 100644 --- a/config.example.py +++ b/config.example.py @@ -75,7 +75,7 @@ class ProductionConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Productionl Database URI" # Public IP of the production machine LOCAL_IP = "127.0.0.1" - LOCAL_IP6 = "2001:718:ff05:107::155" + LOCAL_IP6 = "::ffff:127.0.0.1" # SSO AUTH enabled in produciion SSO_AUTH = True SSO_ATTRIBUTE_MAP = { @@ -101,6 +101,7 @@ class DevelopmentConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Local Database URI" LOCAL_IP = "127.0.0.1" + LOCAL_IP6 = "::ffff:127.0.0.1" DEBUG = True DEVEL = True From 0c46ea40ba2161f4f8155e8c4a279b2426cb04a9 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 23 Jul 2024 12:50:46 +0200 Subject: [PATCH 10/54] first draft of version 1.0.0 --- README.md | 3 +- exaapi/README.md | 21 -------------- exaapi/config.example.py | 19 ------------ exaapi/exa_api_http.py | 31 -------------------- exaapi/exa_api_logger.py | 12 -------- exaapi/exa_api_rabbit.py | 63 ---------------------------------------- exaapi/rabbit_manual.py | 24 --------------- flowapp/__about__.py | 2 +- guarda/README.md | 16 ---------- guarda/config.example.py | 7 ----- guarda/guarda.py | 8 ----- guarda/guarda.service | 15 ---------- setup.py | 12 +++++--- 13 files changed, 11 insertions(+), 222 deletions(-) delete mode 100644 exaapi/README.md delete mode 100644 exaapi/config.example.py delete mode 100755 exaapi/exa_api_http.py delete mode 100644 exaapi/exa_api_logger.py delete mode 100644 exaapi/exa_api_rabbit.py delete mode 100644 exaapi/rabbit_manual.py delete mode 100644 guarda/README.md delete mode 100644 guarda/config.example.py delete mode 100755 guarda/guarda.py delete mode 100644 guarda/guarda.service diff --git a/README.md b/README.md index 269047c2..17fea880 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,10 @@ Last part of the system is Guarda service. This systemctl service is running in * [Local database instalation notes](./docs/DB_LOCAL.md) ## Change Log +- 1.0.0 . DRAFT - ExaAPI and Guarda modules moved to their own repositories. New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. - 0.8.1 application is using Flask-Session stored in DB using SQL Alchemy driver. This can be configured for other drivers, however server side session is required for the application proper function. -- 0.8.0 - API keys update. **Run migration scripts to update your DB**. Keys can now have expiration date and readonly flag. Admin can create special keys for certain machines. +- 0.8.0 - API keys update. **Run migration scripts to update your DB**. Keys can now have expiration date and readonly flag. Admin can create special keys for certain machinnes. - 0.7.3 - New possibility of external auth proxy. - 0.7.2 - Dashboard and Main menu are now customizable in config. App is ready to be packaged using setup.py. - 0.7.0 - ExaAPI now have two options - HTTP or RabbitMQ. ExaAPI process has been renamed, update of ExaBGP process value is needed for this version. diff --git a/exaapi/README.md b/exaapi/README.md deleted file mode 100644 index cd0fa712..00000000 --- a/exaapi/README.md +++ /dev/null @@ -1,21 +0,0 @@ -#ExaAPI web app - -This is a very simple web application, which needs to be hooked on ExaBGP daemon. Every time this app -gets a new command, it replicates the command to the daemon through the stdout. The registered -daemon is watching the stdout of the ExaAPI service. - -Add this to your ExaBGP config -``` -process flowspec { - run /usr/bin/python3 /home/deploy/www/exaapi/exa_api.py; - encoder json; - } -``` - -It can run on the development Flask server, however there is no security layer in this app. -You should limit the access only from the localhost. - -See [ExaBPG docs](https://github.com/Exa-Networks/exabgp/wiki/Controlling-ExaBGP-:-possible-options-for-process) for more information. - -Our plan is to relace this simple app with message queue in the future. - diff --git a/exaapi/config.example.py b/exaapi/config.example.py deleted file mode 100644 index 06b3c47b..00000000 --- a/exaapi/config.example.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Example of configuration file - -Add your log settings and rename to config.py -""" - -LOG_FILE = "/var/log/exafs/exa_api.log" -LOG_FORMAT = "%(asctime)s: %(message)s" - - -# rabbit mq -# note - rabbit mq must be enabled in main app config -# credentials and queue must be set here for the same values -EXA_API_RABBIT_HOST = "localhost" -EXA_API_RABBIT_PORT = "5672" -EXA_API_RABBIT_PASS = "mysecurepassword" -EXA_API_RABBIT_USER = "myexaapiuser" -EXA_API_RABBIT_VHOST = "/" -EXA_API_RABBIT_QUEUE = "my_exa_api_queue" diff --git a/exaapi/exa_api_http.py b/exaapi/exa_api_http.py deleted file mode 100755 index 2726055b..00000000 --- a/exaapi/exa_api_http.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -""" -ExaBGP HTTP API process -This module is process for ExaBGP -https://github.com/Exa-Networks/exabgp/wiki/Controlling-ExaBGP-:-possible-options-for-process - -Each command received in the POST request is send to stdout and captured by ExaBGP. -""" - -from flask import Flask, request -from sys import stdout - -import exa_api_logger - -app = Flask(__name__) - -logger = exa_api_logger.create() - - -@app.route("/", methods=["POST"]) -def command(): - cmd = request.form["command"] - logger.info(cmd) - stdout.write("%s\n" % cmd) - stdout.flush() - - return "%s\n" % cmd - - -if __name__ == "__main__": - app.run() diff --git a/exaapi/exa_api_logger.py b/exaapi/exa_api_logger.py deleted file mode 100644 index 1c5bb4dd..00000000 --- a/exaapi/exa_api_logger.py +++ /dev/null @@ -1,12 +0,0 @@ -import logging -import config - - -def create(): - logger = logging.getLogger(__name__) - f_format = logging.Formatter(config.LOG_FORMAT) - f_handler = logging.FileHandler(config.LOG_FILE) - f_handler.setFormatter(f_format) - logger.setLevel(logging.INFO) - logger.addHandler(f_handler) - return logger diff --git a/exaapi/exa_api_rabbit.py b/exaapi/exa_api_rabbit.py deleted file mode 100644 index f25c2b89..00000000 --- a/exaapi/exa_api_rabbit.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -""" -ExaBGP RabbitMQ API process -This module is process for ExaBGP -https://github.com/Exa-Networks/exabgp/wiki/Controlling-ExaBGP-:-possible-options-for-process - -Each command received from the queue is send to stdout and captured by ExaBGP. -""" -import pika -import sys -import os -from time import sleep - -import config -import exa_api_logger - -logger = exa_api_logger.create() - - -def callback(ch, method, properties, body): - body = body.decode("utf-8") - logger.info(body) - sys.stdout.write("%s\n" % body) - sys.stdout.flush() - - -def main(): - while True: - user = config.EXA_API_RABBIT_USER - passwd = config.EXA_API_RABBIT_PASS - queue = config.EXA_API_RABBIT_QUEUE - credentials = pika.PlainCredentials(user, passwd) - parameters = pika.ConnectionParameters( - config.EXA_API_RABBIT_HOST, - config.EXA_API_RABBIT_PORT, - config.EXA_API_RABBIT_VHOST, - credentials, - ) - connection = pika.BlockingConnection(parameters) - channel = connection.channel() - - channel.queue_declare(queue=queue) - - channel.basic_consume(queue=queue, on_message_callback=callback, auto_ack=True) - - print(" [*] Waiting for messages. To exit press CTRL+C") - try: - channel.start_consuming() - except KeyboardInterrupt: - channel.stop_consuming() - connection.close() - print("\nInterrupted") - try: - sys.exit(0) - except SystemExit: - os._exit(0) - except pika.exceptions.ConnectionClosedByBroker: - sleep(15) - continue - - -if __name__ == "__main__": - main() diff --git a/exaapi/rabbit_manual.py b/exaapi/rabbit_manual.py deleted file mode 100644 index 0e2b2ce5..00000000 --- a/exaapi/rabbit_manual.py +++ /dev/null @@ -1,24 +0,0 @@ -import pika -import sys - -import config - -user = config.EXA_API_RABBIT_USER -passwd = config.EXA_API_RABBIT_PASS -queue = config.EXA_API_RABBIT_QUEUE -credentials = pika.PlainCredentials(user, passwd) -parameters = pika.ConnectionParameters( - config.EXA_API_RABBIT_HOST, - config.EXA_API_RABBIT_PORT, - config.EXA_API_RABBIT_VHOST, - credentials, -) - -connection = pika.BlockingConnection(parameters) -channel = connection.channel() -channel.queue_declare(queue=queue) -route = sys.argv[1] - -print("got :", route) - -channel.basic_publish(exchange="", routing_key=queue, body=route) diff --git a/flowapp/__about__.py b/flowapp/__about__.py index 6ed043c8..1c11a6ec 100755 --- a/flowapp/__about__.py +++ b/flowapp/__about__.py @@ -1 +1 @@ -__version__ = "0.8.1" +__version__ = "1.0.0" diff --git a/guarda/README.md b/guarda/README.md deleted file mode 100644 index 7565640d..00000000 --- a/guarda/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Guarda service for ExaBGP - -## As root - -Edit guarda.service file and set correct location of guarda.py. - -Then edit guarda.py and set address of your host. - - -```bash -pip install requests -chmod +x guarda.py -cp guarda.service /usr/lib/systemd/system/guarda.service -systemctl start guarda.service -systemctl enable guarda.service -``` diff --git a/guarda/config.example.py b/guarda/config.example.py deleted file mode 100644 index fc42e34c..00000000 --- a/guarda/config.example.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Example of configuration file - -Add your application URL and rename to config.py -""" - -URL = 'http://127.0.0.1/rules/announce_all' \ No newline at end of file diff --git a/guarda/guarda.py b/guarda/guarda.py deleted file mode 100755 index 262ecc5e..00000000 --- a/guarda/guarda.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/python3 - -import requests -import time -import config - -time.sleep(10) -requests.get(config.URL) diff --git a/guarda/guarda.service b/guarda/guarda.service deleted file mode 100644 index d4cdc756..00000000 --- a/guarda/guarda.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=ExaBGP restart guardian -After=exabgp.service -Requires=exabgp.service -PartOf=exabgp.service -ConditionPathExists=/home/deploy/www/guarda/guarda.py - -[Service] -Type=simple -ExecStart=/usr/bin/python3.6 /home/deploy/www/guarda/guarda.py -StandardOutput=syslog -StandardError=syslog - -[Install] -WantedBy=exabgp.service diff --git a/setup.py b/setup.py index 84584009..b10fb667 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,7 @@ """ -Author(s): Jakub Man +Author(s): +Jiri Vrany +Jakub Man Setuptools configuration """ @@ -13,15 +15,17 @@ setuptools.setup( name="exafs", - version=__version__, # noqa: F821 + version=__version__, # noqa: F821 author="CESNET / Jiri Vrany, Petr Adamec, Josef Verich, Jakub Man", description="Tool for creation, validation, and execution of ExaBGP messages.", url="https://github.com/CESNET/exafs", license="MIT", - py_modules=["flowapp", "exaapi"], + py_modules=[ + "flowapp", + ], packages=setuptools.find_packages(), include_package_data=True, - python_requires=">=3.8", + python_requires=">=3.11", install_requires=[ "Flask>=2.0.2", "Flask-SQLAlchemy>=2.2", From d47fc4ad6363925eb08a5593c34b9d2c3200789a Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 10:18:59 +0200 Subject: [PATCH 11/54] why cs in sso map ? --- flowapp/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flowapp/__init__.py b/flowapp/__init__.py index fb96b098..f5ae3612 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -25,7 +25,6 @@ def create_app(config_object=None): # SSO configuration SSO_ATTRIBUTE_MAP = { "eppn": (True, "eppn"), - "cn": (False, "cn"), } app.config.setdefault("SSO_ATTRIBUTE_MAP", SSO_ATTRIBUTE_MAP) app.config.setdefault("SSO_LOGIN_URL", "/login") From 20a4dbe0e058df9aa2892439c6d63a52047b5d10 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 13:40:39 +0200 Subject: [PATCH 12/54] SSO Attribute map now in config directly --- config.example.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/config.example.py b/config.example.py index c4fa7a8d..d091c1cf 100644 --- a/config.example.py +++ b/config.example.py @@ -1,4 +1,4 @@ -class Config(): +class Config: """ Default config options """ @@ -16,7 +16,7 @@ class Config(): # Name of HTTP header containing the UUID of authenticated user. # Only used when HEADER_AUTH is set to True - AUTH_HEADER_NAME = 'X-Authenticated-User' + AUTH_HEADER_NAME = "X-Authenticated-User" # SSO LOGOUT LOGOUT_URL = "https://flowspec.example.com/Shibboleth.sso/Logout" # SQL Alchemy config @@ -77,6 +77,12 @@ class ProductionConfig(Config): LOCAL_IP = "127.0.0.1" # SSO AUTH enabled in produciion SSO_AUTH = True + # Map SSO attributes from ADFS to session keys under session['user'] + SSO_ATTRIBUTE_MAP = { + "eppn": (True, "eppn"), + } + SSO_LOGIN_URL = "/login" + # Set true if you need debug in production DEBUG = False From fbc780ab6f45b8c9d40f5964d445c984a879559f Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:01:54 +0200 Subject: [PATCH 13/54] debug SSO --- flowapp/auth.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/flowapp/auth.py b/flowapp/auth.py index c4d942ff..c1426c5a 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -15,6 +15,7 @@ def auth_required(f): def decorated(*args, **kwargs): if not check_auth(get_user()): if current_app.config.get("SSO_AUTH"): + print("SSO AUTH SET BUT FAILS") return redirect("/login") elif current_app.config.get("HEADER_AUTH", False): return redirect("/ext-login") @@ -62,11 +63,7 @@ def decorated(*args, **kwargs): localv4 = current_app.config.get("LOCAL_IP") localv6 = current_app.config.get("LOCAL_IP6") if remote != localv4 and remote != localv6: - print( - "AUTH LOCAL ONLY FAIL FROM {} / local adresses [{}, {}]".format( - remote, localv4, localv6 - ) - ) + print("AUTH LOCAL ONLY FAIL FROM {} / local adresses [{}, {}]".format(remote, localv4, localv6)) abort(403) # Forbidden return f(*args, **kwargs) @@ -97,6 +94,7 @@ def check_auth(uuid): session["app_version"] = __version__ if current_app.config.get("SSO_AUTH"): + print("CHECK AUTH, SS AUTH SET", uuid) # SSO AUTH exist = False if uuid: @@ -104,18 +102,16 @@ def check_auth(uuid): return exist elif current_app.config.get("HEADER_AUTH", False): # External auth (for example apache) - header_name = current_app.config.get("AUTH_HEADER_NAME", 'X-Authenticated-User') + header_name = current_app.config.get("AUTH_HEADER_NAME", "X-Authenticated-User") if header_name not in request.headers or not session.get("user_uuid"): return False return db.session.query(User).filter_by(uuid=request.headers.get(header_name)) else: - # Localhost login / no check + # Localhost login / no check / works only in development environment session["user_email"] = current_app.config["LOCAL_USER_UUID"] session["user_id"] = current_app.config["LOCAL_USER_ID"] session["user_roles"] = current_app.config["LOCAL_USER_ROLES"] - session["user_orgs"] = ", ".join( - org["name"] for org in current_app.config["LOCAL_USER_ORGS"] - ) + session["user_orgs"] = ", ".join(org["name"] for org in current_app.config["LOCAL_USER_ORGS"]) session["user_role_ids"] = current_app.config["LOCAL_USER_ROLE_IDS"] session["user_org_ids"] = current_app.config["LOCAL_USER_ORG_IDS"] roles = [i > 1 for i in session["user_role_ids"]] From 5c45a086d06532e9f29202ce53b1f1950591949f Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:19:37 +0200 Subject: [PATCH 14/54] debug SSO --- flowapp/instance_config.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 9d5a1bfa..96444f20 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -125,3 +125,10 @@ class InstanceConfig: } COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} + + SSO_ATTRIBUTE_MAP = { + "eppn": (True, "eppn"), + "HTTP_X_EPPN": (True, "eppn"), + } + + SSO_LOGIN_URL = "/login" From 8518db8fd31ef8a710e788b6c274b78512b1a12e Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:24:07 +0200 Subject: [PATCH 15/54] debug SSO --- flowapp/instance_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 96444f20..52c1af61 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -127,7 +127,7 @@ class InstanceConfig: COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} SSO_ATTRIBUTE_MAP = { - "eppn": (True, "eppn"), + # "eppn": (True, "eppn"), "HTTP_X_EPPN": (True, "eppn"), } From 2623ccc8afe91bd7c2e74fd8e453b266bac402ae Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 14:31:41 +0200 Subject: [PATCH 16/54] debug SSO --- flowapp/instance_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 52c1af61..8c7817c3 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -128,7 +128,7 @@ class InstanceConfig: SSO_ATTRIBUTE_MAP = { # "eppn": (True, "eppn"), - "HTTP_X_EPPN": (True, "eppn"), + "eppn": (True, "HTTP_X_EPPN"), } SSO_LOGIN_URL = "/login" From 85d23664e5f3e93b3f0dc1d5949490e84fb3f383 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 19:10:31 +0200 Subject: [PATCH 17/54] update app factory to accept config object as parameter --- config.example.py | 12 ++++++++++-- flowapp/instance_config.py | 7 ------- run.example.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/config.example.py b/config.example.py index d091c1cf..de127dd2 100644 --- a/config.example.py +++ b/config.example.py @@ -75,16 +75,23 @@ class ProductionConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Productionl Database URI" # Public IP of the production machine LOCAL_IP = "127.0.0.1" + LOCAL_IP6 = "2001:718:ff05:107::155" # SSO AUTH enabled in produciion SSO_AUTH = True - # Map SSO attributes from ADFS to session keys under session['user'] SSO_ATTRIBUTE_MAP = { - "eppn": (True, "eppn"), + "eppn": (False, "eppn"), + "HTTP_X_EPPN": (False, "eppn"), } SSO_LOGIN_URL = "/login" # Set true if you need debug in production DEBUG = False + DEVEL = False + + # Set cookie behavior + SESSION_COOKIE_SECURE = (True,) + SESSION_COOKIE_HTTPONLY = (True,) + SESSION_COOKIE_SAMESITE = ("Lax",) class DevelopmentConfig(Config): @@ -95,6 +102,7 @@ class DevelopmentConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Local Database URI" LOCAL_IP = "127.0.0.1" DEBUG = True + DEVEL = True class TestingConfig(Config): diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 8c7817c3..9d5a1bfa 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -125,10 +125,3 @@ class InstanceConfig: } COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} - - SSO_ATTRIBUTE_MAP = { - # "eppn": (True, "eppn"), - "eppn": (True, "HTTP_X_EPPN"), - } - - SSO_LOGIN_URL = "/login" diff --git a/run.example.py b/run.example.py index e19c804e..b3dd0c4e 100644 --- a/run.example.py +++ b/run.example.py @@ -40,4 +40,4 @@ # run app if __name__ == "__main__": - app.run(host="127.0.0.1", port=8000, debug=True) + app.run(host="::", port=8080, debug=True) From 858071a84c735c926589258509b9947450799296 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 25 Jul 2024 19:12:50 +0200 Subject: [PATCH 18/54] update app factory to accept config object as parameter --- config.example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.example.py b/config.example.py index de127dd2..1583cfa2 100644 --- a/config.example.py +++ b/config.example.py @@ -75,7 +75,7 @@ class ProductionConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Productionl Database URI" # Public IP of the production machine LOCAL_IP = "127.0.0.1" - LOCAL_IP6 = "2001:718:ff05:107::155" + LOCAL_IP6 = "::ffff:127.0.0.1" # SSO AUTH enabled in produciion SSO_AUTH = True SSO_ATTRIBUTE_MAP = { @@ -101,6 +101,7 @@ class DevelopmentConfig(Config): SQLALCHEMY_DATABASE_URI = "Your Local Database URI" LOCAL_IP = "127.0.0.1" + LOCAL_IP6 = "::ffff:127.0.0.1" DEBUG = True DEVEL = True From 5de4703f8aaf31070f9d2305f64845b287e167a0 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 23 Sep 2024 12:38:03 +0200 Subject: [PATCH 19/54] init --- flowapp/__init__.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/flowapp/__init__.py b/flowapp/__init__.py index 97f66a92..f5ae3612 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -16,15 +16,11 @@ migrate = Migrate() csrf = CSRFProtect() ext = SSO() -<<<<<<< HEAD sess = Session() -======= ->>>>>>> f65f2be8c6ceee972be8b88c2b73fc3da7e69ee5 def create_app(config_object=None): app = Flask(__name__) -<<<<<<< HEAD # SSO configuration SSO_ATTRIBUTE_MAP = { @@ -32,8 +28,6 @@ def create_app(config_object=None): } app.config.setdefault("SSO_ATTRIBUTE_MAP", SSO_ATTRIBUTE_MAP) app.config.setdefault("SSO_LOGIN_URL", "/login") -======= ->>>>>>> f65f2be8c6ceee972be8b88c2b73fc3da7e69ee5 # extension init migrate.init_app(app, db) From c7c23e1c79c8c72f1e7a2647159182df276ed98f Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 14 Oct 2024 17:58:04 +0200 Subject: [PATCH 20/54] add loguru for better logging control --- README.md | 3 -- flowapp/__init__.py | 7 ++++- flowapp/auth.py | 6 ++-- flowapp/output.py | 13 ++++---- flowapp/views/admin.py | 46 +++++++---------------------- flowapp/views/api_common.py | 59 ++++++++++--------------------------- flowapp/views/api_keys.py | 11 ++----- flowapp/views/rules.py | 8 ++--- requirements.txt | 1 + run.example.py | 3 -- 10 files changed, 50 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 107dd3a8..17fea880 100644 --- a/README.md +++ b/README.md @@ -53,11 +53,8 @@ Last part of the system is Guarda service. This systemctl service is running in ## Change Log - 1.0.0 . DRAFT - ExaAPI and Guarda modules moved to their own repositories. New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. -<<<<<<< HEAD - 0.8.1 application is using Flask-Session stored in DB using SQL Alchemy driver. This can be configured for other drivers, however server side session is required for the application proper function. -======= ->>>>>>> f65f2be8c6ceee972be8b88c2b73fc3da7e69ee5 - 0.8.0 - API keys update. **Run migration scripts to update your DB**. Keys can now have expiration date and readonly flag. Admin can create special keys for certain machinnes. - 0.7.3 - New possibility of external auth proxy. - 0.7.2 - Dashboard and Main menu are now customizable in config. App is ready to be packaged using setup.py. diff --git a/flowapp/__init__.py b/flowapp/__init__.py index f5ae3612..cb915771 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import babel +from loguru import logger from flask import Flask, redirect, render_template, session, url_for, request from flask_sso import SSO @@ -22,6 +23,10 @@ def create_app(config_object=None): app = Flask(__name__) + # logger init + logger.remove() + app.logger = logger + # SSO configuration SSO_ATTRIBUTE_MAP = { "eppn": (True, "eppn"), @@ -145,7 +150,7 @@ def not_found(error): @app.errorhandler(500) def internal_error(exception): - app.logger.error(exception) + app.logger.exception(exception) return render_template("errors/500.html"), 500 @app.context_processor diff --git a/flowapp/auth.py b/flowapp/auth.py index c1426c5a..d1659af1 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -15,7 +15,7 @@ def auth_required(f): def decorated(*args, **kwargs): if not check_auth(get_user()): if current_app.config.get("SSO_AUTH"): - print("SSO AUTH SET BUT FAILS") + current_app.logger.warning("SSO AUTH SET BUT FAILS") return redirect("/login") elif current_app.config.get("HEADER_AUTH", False): return redirect("/ext-login") @@ -63,7 +63,7 @@ def decorated(*args, **kwargs): localv4 = current_app.config.get("LOCAL_IP") localv6 = current_app.config.get("LOCAL_IP6") if remote != localv4 and remote != localv6: - print("AUTH LOCAL ONLY FAIL FROM {} / local adresses [{}, {}]".format(remote, localv4, localv6)) + current_app.logger.warning(f"AUTH LOCAL ONLY FAIL FROM {remote} / local adresses [{localv4}, {localv6}]") abort(403) # Forbidden return f(*args, **kwargs) @@ -94,7 +94,7 @@ def check_auth(uuid): session["app_version"] = __version__ if current_app.config.get("SSO_AUTH"): - print("CHECK AUTH, SS AUTH SET", uuid) + current_app.logger.warning("CHECK AUTH, SS AUTH SET", uuid) # SSO AUTH exist = False if uuid: diff --git a/flowapp/output.py b/flowapp/output.py index 7f2dcc79..9bd6e2e2 100644 --- a/flowapp/output.py +++ b/flowapp/output.py @@ -1,6 +1,7 @@ """ Module for message announcing and logging """ + from datetime import datetime import requests @@ -36,16 +37,14 @@ def announce_to_http(route): """ if not current_app.config["TESTING"]: try: - resp = requests.post( - current_app.config["EXA_API_URL"], data={"command": route} - ) + resp = requests.post(current_app.config["EXA_API_URL"], data={"command": route}) resp.raise_for_status() except requests.exceptions.HTTPError as err: - print("ExaAPI HTTP Error: ", err) + current_app.logger.error("ExaAPI HTTP Error: ", err) except requests.exceptions.RequestException as ce: - print("Connection to ExaAPI failed: ", ce) + current_app.logger.error("Connection to ExaAPI failed: ", ce) else: - print("Testing:", route) + current_app.logger.debug(f"Testing: {route}") def announce_to_rabbitmq(route): @@ -69,7 +68,7 @@ def announce_to_rabbitmq(route): channel.queue_declare(queue=queue) channel.basic_publish(exchange="", routing_key=queue, body=route) else: - print("Testing:", route) + current_app.logger.debug("Testing: {route}") def log_route(user_id, route_model, rule_type, author): diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index f142a4ae..fab9373d 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta import secrets -from flask import Blueprint, render_template, redirect, flash, request, session, url_for +from flask import Blueprint, render_template, redirect, flash, request, session, url_for, current_app from sqlalchemy.exc import IntegrityError from ..forms import ASPathForm, MachineApiKeyForm, UserForm, ActionForm, OrganizationForm, CommunityForm @@ -68,15 +68,13 @@ def add_machine_key(): form = MachineApiKeyForm(request.form, key=generated) if request.method == "POST" and form.validate(): - print("Form validated") - # import ipdb; ipdb.set_trace() model = MachineApiKey( machine=form.machine.data, key=form.key.data, expires=form.expires.data, readonly=form.readonly.data, comment=form.comment.data, - user_id=session["user_id"] + user_id=session["user_id"], ) db.session.add(model) @@ -87,10 +85,7 @@ def add_machine_key(): else: for field, errors in form.errors.items(): for error in errors: - print( - "Error in the %s field - %s" - % (getattr(form, field).label.text, error) - ) + current_app.logger.debug("Error in the %s field - %s" % (getattr(form, field).label.text, error)) return render_template("forms/machine_api_key.html", form=form, generated_key=generated) @@ -117,12 +112,8 @@ def delete_machine_key(key_id): @admin_required def user(): form = UserForm(request.form) - form.role_ids.choices = [ - (g.id, g.name) for g in db.session.query(Role).order_by("name") - ] - form.org_ids.choices = [ - (g.id, g.name) for g in db.session.query(Organization).order_by("name") - ] + form.role_ids.choices = [(g.id, g.name) for g in db.session.query(Role).order_by("name")] + form.org_ids.choices = [(g.id, g.name) for g in db.session.query(Organization).order_by("name")] if request.method == "POST" and form.validate(): # test if user is unique @@ -157,12 +148,8 @@ def user(): def edit_user(user_id): user = db.session.query(User).get(user_id) form = UserForm(request.form, obj=user) - form.role_ids.choices = [ - (g.id, g.name) for g in db.session.query(Role).order_by("name") - ] - form.org_ids.choices = [ - (g.id, g.name) for g in db.session.query(Organization).order_by("name") - ] + form.role_ids.choices = [(g.id, g.name) for g in db.session.query(Role).order_by("name")] + form.org_ids.choices = [(g.id, g.name) for g in db.session.query(Organization).order_by("name")] if request.method == "POST" and form.validate(): user.update(form) @@ -195,7 +182,6 @@ def delete_user(user_id): except IntegrityError as e: message = "User {} owns some rules, can not be deleted!".format(username) alert_type = "alert-danger" - print(e) flash(message, alert_type) return redirect(url_for("admin.users")) @@ -233,9 +219,7 @@ def organization(): flash("Organization saved") return redirect(url_for("admin.organizations")) else: - flash( - "Organization {} already exists".format(form.name.data), "alert-danger" - ) + flash("Organization {} already exists".format(form.name.data), "alert-danger") action_url = url_for("admin.organization") return render_template( @@ -391,9 +375,7 @@ def action(): return redirect(url_for("admin.actions")) else: flash( - "Action with name {} or command {} already exists".format( - form.name.data, form.command.data - ), + "Action with name {} or command {} already exists".format(form.name.data, form.command.data), "alert-danger", ) @@ -411,7 +393,6 @@ def action(): @admin_required def edit_action(action_id): action = db.session.query(Action).get(action_id) - print(action.role_id) form = ActionForm(request.form, obj=action) if request.method == "POST" and form.validate(): form.populate_obj(action) @@ -480,9 +461,7 @@ def community(): flash("Community saved", "alert-success") return redirect(url_for("admin.communities")) else: - flash( - f"Community with name {form.name.data} already exists", "alert-danger" - ) + flash(f"Community with name {form.name.data} already exists", "alert-danger") community_url = url_for("admin.community") return render_template( @@ -498,7 +477,6 @@ def community(): @admin_required def edit_community(community_id): community = db.session.query(Community).get(community_id) - print(community.role_id) form = CommunityForm(request.form, obj=community) if request.method == "POST" and form.validate(): form.populate_obj(community) @@ -527,9 +505,7 @@ def delete_community(community_id): try: db.session.commit() except IntegrityError: - message = "Community {} is in use in some rules, can not be deleted!".format( - aname - ) + message = "Community {} is in use in some rules, can not be deleted!".format(aname) alert_type = "alert-danger" flash(message, alert_type) diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index 163449ce..864a0222 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -51,9 +51,7 @@ def decorated(*args, **kwargs): return jsonify({"message": "auth token is missing"}), 401 try: - data = jwt.decode( - token, current_app.config.get("JWT_SECRET"), algorithms=["HS256"] - ) + data = jwt.decode(token, current_app.config.get("JWT_SECRET"), algorithms=["HS256"]) current_user = data["user"] except jwt.DecodeError: return jsonify({"message": "auth token is invalid"}), 403 @@ -86,9 +84,7 @@ def authorize(user_key): return jsonify({"message": "auth token is expired"}), 401 # check if the key is not used by different machine - if model and ipaddress.ip_address(model.machine) == ipaddress.ip_address( - request.remote_addr - ): + if model and ipaddress.ip_address(model.machine) == ipaddress.ip_address(request.remote_addr): payload = { "user": { "uuid": model.user.uuid, @@ -114,25 +110,24 @@ def check_readonly(func): Check if the token is readonly Used in api endpoints """ + @wraps(func) def decorated_function(*args, **kwargs): # Access read only flag from first of the args - print("ARGS", args) - print("KWARGS", kwargs) current_user = kwargs.get("current_user", False) read_only = current_user.get("readonly", False) if read_only: return jsonify({"message": "read only token can't perform this action"}), 403 return func(*args, **kwargs) + return decorated_function # endpints + def index(current_user, key_map): - prefered_tf = ( - request.args.get(TIME_FORMAT_ARG) if request.args.get(TIME_FORMAT_ARG) else "" - ) + prefered_tf = request.args.get(TIME_FORMAT_ARG) if request.args.get(TIME_FORMAT_ARG) else "" net_ranges = get_user_nets(current_user["id"]) rules4 = db.session.query(Flowspec4).order_by(Flowspec4.expires.desc()).all() @@ -156,26 +151,14 @@ def index(current_user, key_map): user_actions = get_user_actions(current_user["role_ids"]) user_actions = [act[0] for act in user_actions] - rules4_editable, rules4_visible = flowspec.filter_rules_action( - user_actions, rules4 - ) - rules6_editable, rules6_visible = flowspec.filter_rules_action( - user_actions, rules6 - ) + rules4_editable, rules4_visible = flowspec.filter_rules_action(user_actions, rules4) + rules6_editable, rules6_visible = flowspec.filter_rules_action(user_actions, rules6) payload = { - key_map["ipv4_rules"]: [ - rule.to_dict(prefered_tf) for rule in rules4_editable - ], - key_map["ipv6_rules"]: [ - rule.to_dict(prefered_tf) for rule in rules6_editable - ], - key_map["ipv4_rules_readonly"]: [ - rule.to_dict(prefered_tf) for rule in rules4_visible - ], - key_map["ipv6_rules_readonly"]: [ - rule.to_dict(prefered_tf) for rule in rules6_visible - ], + key_map["ipv4_rules"]: [rule.to_dict(prefered_tf) for rule in rules4_editable], + key_map["ipv6_rules"]: [rule.to_dict(prefered_tf) for rule in rules6_editable], + key_map["ipv4_rules_readonly"]: [rule.to_dict(prefered_tf) for rule in rules4_visible], + key_map["ipv6_rules_readonly"]: [rule.to_dict(prefered_tf) for rule in rules6_visible], key_map["rtbh_rules"]: [rule.to_dict(prefered_tf) for rule in rules_rtbh], } return jsonify(payload) @@ -225,9 +208,7 @@ def create_ipv4(current_user): # if the form is not valid, we should return 404 with errors if not form.validate(): - print("F EXPIRES", form.expires) form_errors = get_form_errors(form) - print("VALIDATION", form_errors) if form_errors: return jsonify(form_errors), 400 @@ -235,9 +216,7 @@ def create_ipv4(current_user): if model: model.expires = form.expires.data - flash_message = ( - "Existing IPv4 Rule found. Expiration time was updated to new value." - ) + flash_message = "Existing IPv4 Rule found. Expiration time was updated to new value." else: model = Flowspec4( source=form.source.data, @@ -300,9 +279,7 @@ def create_ipv6(current_user): if model: model.expires = form.expires.data - flash_message = ( - "Existing IPv6 Rule found. Expiration time was updated to new value." - ) + flash_message = "Existing IPv6 Rule found. Expiration time was updated to new value." else: model = Flowspec6( source=form.source.data, @@ -364,9 +341,7 @@ def create_rtbh(current_user): if model: model.expires = form.expires.data - flash_message = ( - "Existing RTBH Rule found. Expiration time was updated to new value." - ) + flash_message = "Existing RTBH Rule found. Expiration time was updated to new value." else: model = RTBH( ipv4=form.ipv4.data, @@ -438,9 +413,7 @@ def get_rule(current_user, model, rule_id): :param model: rule model :return: json """ - prefered_tf = ( - request.args.get(TIME_FORMAT_ARG) if request.args.get(TIME_FORMAT_ARG) else "" - ) + prefered_tf = request.args.get(TIME_FORMAT_ARG) if request.args.get(TIME_FORMAT_ARG) else "" if model: if check_access_rights(current_user, model.user_id): diff --git a/flowapp/views/api_keys.py b/flowapp/views/api_keys.py index 45cc276b..b7b511ab 100644 --- a/flowapp/views/api_keys.py +++ b/flowapp/views/api_keys.py @@ -62,7 +62,7 @@ def add(): expires=form.expires.data, readonly=form.readonly.data, comment=form.comment.data, - user_id=session["user_id"] + user_id=session["user_id"], ) db.session.add(model) @@ -73,10 +73,7 @@ def add(): else: for field, errors in form.errors.items(): for error in errors: - print( - "Error in the %s field - %s" - % (getattr(form, field).label.text, error) - ) + current_app.logger.debug("Error in the %s field - %s" % (getattr(form, field).label.text, error)) return render_template("forms/api_key.html", form=form, generated_key=generated) @@ -89,9 +86,7 @@ def delete(key_id): :param key_id: integer """ key_list = request.cookies.get(COOKIE_KEY) - key_list = jwt.decode( - key_list, current_app.config.get("JWT_SECRET"), algorithms=["HS256"] - ) + key_list = jwt.decode(key_list, current_app.config.get("JWT_SECRET"), algorithms=["HS256"]) model = db.session.query(ApiKey).get(key_id) if model.id not in key_list["keys"]: diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 714f6221..783128b6 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from operator import ge, lt -from flask import Blueprint, flash, redirect, render_template, request, session, url_for +from flask import Blueprint, current_app, flash, redirect, render_template, request, session, url_for from flowapp import constants, db, messages from flowapp.auth import ( @@ -465,7 +465,7 @@ def ipv4_rule(): else: for field, errors in form.errors.items(): for error in errors: - print("Error in the %s field - %s" % (getattr(form, field).label.text, error)) + current_app.logger.debug("Error in the %s field - %s" % (getattr(form, field).label.text, error)) default_expires = datetime.now() + timedelta(days=7) form.expires.data = default_expires @@ -535,7 +535,7 @@ def ipv6_rule(): else: for field, errors in form.errors.items(): for error in errors: - print("Error in the %s field - %s" % (getattr(form, field).label.text, error)) + current_app.logger.debug("Error in the %s field - %s" % (getattr(form, field).label.text, error)) default_expires = datetime.now() + timedelta(days=7) form.expires.data = default_expires @@ -600,7 +600,7 @@ def rtbh_rule(): else: for field, errors in form.errors.items(): for error in errors: - print("Error in the %s field - %s" % (getattr(form, field).label.text, error)) + current_app.logger.debug("Error in the %s field - %s" % (getattr(form, field).label.text, error)) default_expires = datetime.now() + timedelta(days=7) form.expires.data = default_expires diff --git a/requirements.txt b/requirements.txt index 876a5c61..e26e9e9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ babel>=2.7.0 email_validator>=1.1 pika>=1.3.0 mysqlclient>=2.0.0 +loguru diff --git a/run.example.py b/run.example.py index 75fec464..b3dd0c4e 100644 --- a/run.example.py +++ b/run.example.py @@ -3,13 +3,10 @@ First copy the file as run.py (or whatever you want) Then edit the file to match your needs. -<<<<<<< HEAD From version 0.8.1 the application is using Flask-Session stored in DB using SQL Alchemy driver. This can be configured for other drivers, however server side session is required for the application. -======= ->>>>>>> f65f2be8c6ceee972be8b88c2b73fc3da7e69ee5 In general you should not need to edit this example file. Only if you want to configure the application main menu and dashboard. From d8a94dcb4da6288c0ed13300d2dae571a39c9cf9 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 14 Oct 2024 19:36:09 +0200 Subject: [PATCH 21/54] update docs --- README.md | 16 +++++++++------- docs/guarda-service/README.md | 14 ++++++++++++++ docs/guarda-service/guarda.service | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 docs/guarda-service/README.md create mode 100644 docs/guarda-service/guarda.service diff --git a/README.md b/README.md index 17fea880..b54b8c1c 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,15 @@ See how is ExaFS integrated into the network in the picture below. The central part of the ExaFS is a web application, written in Python3.6 with Flask framework. It provides a user interface for ExaBGP rule CRUD operations. The application also provides the REST API with CRUD operations for the configuration rules. The web app uses Shibboleth authorization; the REST API is using token-based authorization. -The app creates the ExaBGP commands and forwards them to ExaAPI. All rules are carefully validated, and only valid rules are stored in the database and sent to the ExaBGP connector. +The app creates the ExaBGP commands and forwards them to ExaBGP process. All rules are carefully validated, and only valid rules are stored in the database and sent to the ExaBGP connector. -This second part of the system is another web application that replicates the received command to the stdout. The connection between ExaBGP daemon and stdout of ExaAPI is specified in the ExaBGP config. +This second part of the system is another application that replicates the received command to the stdout. The connection between ExaBGP daemon and stdout of ExaAPI (ExaBGP process) is specified in the ExaBGP config. + +This API was a part of the project, but now has been moved to own repository. You can use [pip package exabgp-process](https://pypi.org/project/exabgp-process/) or clone the git repo. Or you can create your own version. -Every time this API gets a command from ExaFS, it replicates this command to the ExaBGP daemon through the stdout. The registered daemon then updates the ExaBGP table – create, modify or remove the rule from command. -Last part of the system is Guarda service. This systemctl service is running in the host system and gets a notification on each restart of ExaBGP service via systemctl WantedBy config option. For every restart of ExaBGP the Guarda service will put all the valid and active rules to the ExaBGP rules table again. +Every time this process gets a command from ExaFS, it replicates this command to the ExaBGP service through the stdout. The registered service then updates the ExaBGP table – create, modify or remove the rule from command. + +You may also need to monitor the ExaBGP and renew the commands after restart / shutdown. In docs you can find and example of system service named Guarda. This systemctl service is running in the host system and gets a notification on each restart of ExaBGP service via systemctl WantedBy config option. For every restart of ExaBGP the Guarda service will put all the valid and active rules to the ExaBGP rules table again. ## DOCS * [Install notes](./docs/INSTALL.md) @@ -52,9 +55,8 @@ Last part of the system is Guarda service. This systemctl service is running in * [Local database instalation notes](./docs/DB_LOCAL.md) ## Change Log -- 1.0.0 . DRAFT - ExaAPI and Guarda modules moved to their own repositories. New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. -- 0.8.1 application is using Flask-Session stored in DB using SQL Alchemy driver. This can be configured for other -drivers, however server side session is required for the application proper function. +- 1.0.0 . DRAFT - ExaAPI and Guarda modules moved outside of the project to their own repositories. ExaAPI is now available also as a [pip package exabgp-process](https://pypi.org/project/exabgp-process/). New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. +- 0.8.1 application is using Flask-Session stored in DB using SQL Alchemy driver. This can be configured for other drivers, however server side session is required for the application proper function. - 0.8.0 - API keys update. **Run migration scripts to update your DB**. Keys can now have expiration date and readonly flag. Admin can create special keys for certain machinnes. - 0.7.3 - New possibility of external auth proxy. - 0.7.2 - Dashboard and Main menu are now customizable in config. App is ready to be packaged using setup.py. diff --git a/docs/guarda-service/README.md b/docs/guarda-service/README.md new file mode 100644 index 00000000..45f53446 --- /dev/null +++ b/docs/guarda-service/README.md @@ -0,0 +1,14 @@ +# Guarda Service for ExaBGP + +This is a systemd service designed to monitor ExaBGP and reapply commands after a restart or shutdown. The guarda.service runs on the host system and is triggered whenever the ExaBGP service restarts, thanks to the WantedBy configuration in systemd. After each restart, the Guarda service will reapply all valid and active rules to the ExaBGP rules table. + +## Usage (as root) + +First, set the environment variable with the correct URL for your installation. The announce_all endpoint is only accessible from localhost within the app, so ensure that your configuration includes the correct local IP address. + +```bash +export GUARDA_URL=http://127.0.0.1:8080/rules/announce_all +cp guarda.service /usr/lib/systemd/system/guarda.service +systemctl start guarda.service +systemctl enable guarda.service +``` diff --git a/docs/guarda-service/guarda.service b/docs/guarda-service/guarda.service new file mode 100644 index 00000000..07eabe4b --- /dev/null +++ b/docs/guarda-service/guarda.service @@ -0,0 +1,14 @@ +[Unit] +Description=ExaBGP restart guardian +After=exabgp.service +Requires=exabgp.service +PartOf=exabgp.service + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'sleep 10; curl -s $GUARDA_URL' +StandardOutput=syslog +StandardError=syslog + +[Install] +WantedBy=exabgp.service From aeb31168228deb382a47d70aec3f71fa8ec650f2 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 15 Oct 2024 12:45:32 +0200 Subject: [PATCH 22/54] new format of output message, now with author and source. New type Route for the output and RuleTypes and RouteSources for identification --- flowapp/__init__.py | 19 ++++++-- flowapp/constants.py | 3 +- flowapp/output.py | 35 +++++++++++--- flowapp/utils.py | 8 ++-- flowapp/views/api_common.py | 49 +++++++++++++------- flowapp/views/dashboard.py | 51 +++++---------------- flowapp/views/rules.py | 91 ++++++++++++++++++++++++++----------- 7 files changed, 158 insertions(+), 98 deletions(-) diff --git a/flowapp/__init__.py b/flowapp/__init__.py index cb915771..bf6c77c9 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- import babel +import logging from loguru import logger from flask import Flask, redirect, render_template, session, url_for, request +from flask.logging import default_handler from flask_sso import SSO from flask_sqlalchemy import SQLAlchemy from flask_wtf.csrf import CSRFProtect @@ -20,13 +22,16 @@ sess = Session() +class InterceptHandler(logging.Handler): + + def emit(self, record): + logger_opt = logger.opt(depth=6, exception=record.exc_info, colors=True) + logger_opt.log(record.levelname, record.getMessage()) + + def create_app(config_object=None): app = Flask(__name__) - # logger init - logger.remove() - app.logger = logger - # SSO configuration SSO_ATTRIBUTE_MAP = { "eppn": (True, "eppn"), @@ -71,6 +76,10 @@ def create_app(config_object=None): app.register_blueprint(api_v3, url_prefix="/api/v3") app.register_blueprint(dashboard, url_prefix="/dashboard") + # register loguru as handler + app.logger.removeHandler(default_handler) + app.logger.addHandler(InterceptHandler()) + @ext.login_handler def login(user_info): try: @@ -109,6 +118,8 @@ def ext_login(): @app.route("/") @auth_required def index(): + + logger.debug("That's it, beautiful and simple logging!") try: rtype = session[constants.TYPE_ARG] except KeyError: diff --git a/flowapp/constants.py b/flowapp/constants.py index 28bc8f2c..74e3352f 100644 --- a/flowapp/constants.py +++ b/flowapp/constants.py @@ -1,6 +1,7 @@ """ This module contains constant values used in application """ + from operator import ge, lt DEFAULT_SORT = "expires" @@ -22,7 +23,7 @@ RULES_KEY = "rules" -RULE_TYPES = {"ipv4": 4, "ipv6": 6, "rtbh": 1} +RULE_TYPES_DICT = {"ipv4": 4, "ipv6": 6, "rtbh": 1} DEFAULT_COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} ANNOUNCE = 1 diff --git a/flowapp/output.py b/flowapp/output.py index 9bd6e2e2..29c34ff7 100644 --- a/flowapp/output.py +++ b/flowapp/output.py @@ -2,10 +2,12 @@ Module for message announcing and logging """ +from dataclasses import dataclass, asdict from datetime import datetime import requests import pika +import json from flask import current_app from flowapp import db, messages @@ -16,19 +18,40 @@ 4: messages.create_ipv4, 6: messages.create_ipv6, } -RULE_TYPES = {"RTBH": 1, "IPv4": 4, "IPv6": 6} -def announce_route(route): +class RuleTypes: + RTBH = 1 + IPv4 = 4 + IPv6 = 6 + + +class RouteSources: + UI = "UI" + API = "API" + + +@dataclass +class Route: + author: str + source: RouteSources + command: str + + def __dict__(self): + return asdict(self) + + +def announce_route(route: Route): """ - Dispatch route to ExaBGP API + Dispatch route as dict to ExaBGP API API must be set in app config.py defaults to HTTP API """ + current_app.logger.debug(asdict(route)) if current_app.config.get("EXA_API") == "RABBIT": - announce_to_rabbitmq(route) + announce_to_rabbitmq(asdict(route)) else: - announce_to_http(route) + announce_to_http(asdict(route)) def announce_to_http(route): @@ -66,7 +89,7 @@ def announce_to_rabbitmq(route): connection = pika.BlockingConnection(parameters) channel = connection.channel() channel.queue_declare(queue=queue) - channel.basic_publish(exchange="", routing_key=queue, body=route) + channel.basic_publish(exchange="", routing_key=queue, body=json.dumps(route)) else: current_app.logger.debug("Testing: {route}") diff --git a/flowapp/utils.py b/flowapp/utils.py index 50f0f065..ad15b615 100644 --- a/flowapp/utils.py +++ b/flowapp/utils.py @@ -7,7 +7,7 @@ TIME_US, TIME_STMP, TIME_FORMAT_ARG, - RULE_TYPES, + RULE_TYPES_DICT, FORM_TIME_PATTERN, ) @@ -17,7 +17,7 @@ def other_rtypes(rtype): get rtype and return list of remaining rtypes for example get ipv4 and return [ipv6, rtbh] """ - result = list(RULE_TYPES.keys()) + result = list(RULE_TYPES_DICT.keys()) try: result.remove(rtype) except ValueError: @@ -151,9 +151,7 @@ def flash_errors(form): """ for field, errors in form.errors.items(): for error in errors: - flash( - "Error in the %s field - %s" % (getattr(form, field).label.text, error) - ) + flash("Error in the %s field - %s" % (getattr(form, field).label.text, error)) def active_css_rstate(rtype, rstate): diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index 864a0222..330c46fe 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -28,12 +28,7 @@ output_date_format, ) from flowapp.auth import check_access_rights -from flowapp.output import ( - RULE_TYPES, - announce_route, - log_route, - log_withdraw, -) +from flowapp.output import RuleTypes, announce_route, log_route, log_withdraw, Route, RouteSources from flowapp import db, validators, flowspec, messages @@ -242,15 +237,20 @@ def create_ipv4(current_user): # announce route if model is in active state if model.rstate_id == 1: - route = messages.create_ipv4(model, ANNOUNCE) + command = messages.create_ipv4(model, ANNOUNCE) + route = Route( + author=f"{current_user['uuid']} / {current_user['org']}", + source=RouteSources.API, + command=command, + ) announce_route(route) # log changes log_route( current_user["id"], model, - RULE_TYPES["IPv4"], - "{} / {}".format(current_user["uuid"], current_user["org"]), + RuleTypes.IPv4, + f"{current_user['uuid']} / {current_user['org']}", ) pref_format = output_date_format(json_request_data, form.expires.pref_format) @@ -304,15 +304,20 @@ def create_ipv6(current_user): # announce routes if model.rstate_id == 1: - route = messages.create_ipv6(model, ANNOUNCE) + command = messages.create_ipv6(model, ANNOUNCE) + route = Route( + author=f"{current_user['uuid']} / {current_user['org']}", + source=RouteSources.API, + command=command, + ) announce_route(route) # log changes log_route( current_user["id"], model, - RULE_TYPES["IPv6"], - "{} / {}".format(current_user["uuid"], current_user["org"]), + RuleTypes.IPv6, + f"{current_user['uuid']} / {current_user['org']}", ) pref_format = output_date_format(json_request_data, form.expires.pref_format) @@ -360,14 +365,19 @@ def create_rtbh(current_user): # announce routes if model.rstate_id == 1: - route = messages.create_rtbh(model, ANNOUNCE) + command = messages.create_rtbh(model, ANNOUNCE) + route = Route( + author=f"{current_user['uuid']} / {current_user['org']}", + source=RouteSources.API, + command=command, + ) announce_route(route) # log changes log_route( current_user["id"], model, - RULE_TYPES["RTBH"], - "{} / {}".format(current_user["uuid"], current_user["org"]), + RuleTypes.RTBH, + f"{current_user['uuid']} / {current_user['org']}", ) pref_format = output_date_format(json_request_data, form.expires.pref_format) @@ -467,7 +477,12 @@ def delete_rule(current_user, rule_id, model_name, route_model, rule_type): if model: if check_access_rights(current_user, model.user_id): # withdraw route - route = route_model(model, WITHDRAW) + command = route_model(model, WITHDRAW) + route = Route( + author=f"{current_user['uuid']} / {current_user['org']}", + source=RouteSources.API, + command=command, + ) announce_route(route) log_withdraw( @@ -475,7 +490,7 @@ def delete_rule(current_user, rule_id, model_name, route_model, rule_type): route, rule_type, model.id, - "{} / {}".format(current_user["uuid"], current_user["org"]), + f"{current_user['uuid']} / {current_user['org']}", ) # delete from db db.session.delete(model) diff --git a/flowapp/views/dashboard.py b/flowapp/views/dashboard.py index 1f63159c..dff36d98 100644 --- a/flowapp/views/dashboard.py +++ b/flowapp/views/dashboard.py @@ -52,6 +52,7 @@ def index(rtype=None, rstate="active"): :param rstate: :return: view from view factory """ + # set first key of dashboard config as default rtype if not rtype: rtype = next(iter(current_app.config["DASHBOARD"].keys())) @@ -74,31 +75,13 @@ def index(rtype=None, rstate="active"): # get the macros for the current rule type from config # warning no checks here, if the config is set to non existing macro the app will crash - macro_file = ( - current_app.config["DASHBOARD"].get(rtype).get("macro_file", "macros.html") - ) - macro_tbody = ( - current_app.config["DASHBOARD"].get(rtype).get("macro_tbody", "build_ip_tbody") - ) - macro_thead = ( - current_app.config["DASHBOARD"] - .get(rtype) - .get("macro_thead", "build_rules_thead") - ) - macro_tfoot = ( - current_app.config["DASHBOARD"] - .get(rtype) - .get("macro_tfoot", "build_group_buttons_tfoot") - ) + macro_file = current_app.config["DASHBOARD"].get(rtype).get("macro_file", "macros.html") + macro_tbody = current_app.config["DASHBOARD"].get(rtype).get("macro_tbody", "build_ip_tbody") + macro_thead = current_app.config["DASHBOARD"].get(rtype).get("macro_thead", "build_rules_thead") + macro_tfoot = current_app.config["DASHBOARD"].get(rtype).get("macro_tfoot", "build_group_buttons_tfoot") - data_handler_module = ( - current_app.config["DASHBOARD"].get(rtype).get("data_handler", models) - ) - data_handler_method = ( - current_app.config["DASHBOARD"] - .get(rtype) - .get("data_handler_method", "get_ip_rules") - ) + data_handler_module = current_app.config["DASHBOARD"].get(rtype).get("data_handler", models) + data_handler_method = current_app.config["DASHBOARD"].get(rtype).get("data_handler_method", "get_ip_rules") # get search query, sort order and sort key from request or session get_search_query = request.args.get(SEARCH_ARG, session.get(SEARCH_ARG, "")) @@ -213,9 +196,7 @@ def create_dashboard_table_head( tstring = tstring + f"from '{macro_file}' import {macro_name}" tstring = tstring + " %} {{" tstring = ( - tstring - + f" {macro_name}(rules_columns, rtype, rstate, sort_key, sort_order, search_query, group_op) " - + "}}" + tstring + f" {macro_name}(rules_columns, rtype, rstate, sort_key, sort_order, search_query, group_op) " + "}}" ) dashboard_table_head = render_template_string( @@ -232,9 +213,7 @@ def create_dashboard_table_head( return dashboard_table_head -def create_dashboard_table_foot( - colspan=10, macro_file="macros.html", macro_name="build_group_buttons_tfoot" -): +def create_dashboard_table_foot(colspan=10, macro_file="macros.html", macro_name="build_group_buttons_tfoot"): """ create the table foot for the dashboard using a jinja2 macro :param colspan: the number of columns @@ -276,9 +255,7 @@ def create_admin_response( :return: """ - dashboard_table_body = create_dashboard_table_body( - rules, rtype, macro_file=macro_file, macro_name=macro_tbody - ) + dashboard_table_body = create_dashboard_table_body(rules, rtype, macro_file=macro_file, macro_name=macro_tbody) dashboard_table_head = create_dashboard_table_head( rules_columns=table_columns, @@ -345,16 +322,12 @@ def create_user_response( net_ranges = models.get_user_nets(session["user_id"]) if rtype == "rtbh": - rules_editable, read_only_rules = validators.split_rtbh_rules_for_user( - net_ranges, rules - ) + rules_editable, read_only_rules = validators.split_rtbh_rules_for_user(net_ranges, rules) else: user_rules, read_only_rules = validators.split_rules_for_user(net_ranges, rules) user_actions = models.get_user_actions(session["user_role_ids"]) user_actions = [act[0] for act in user_actions] - rules_editable, rules_visible = flowspec.filter_rules_action( - user_actions, user_rules - ) + rules_editable, rules_visible = flowspec.filter_rules_action(user_actions, user_rules) read_only_rules = read_only_rules + rules_visible # we don't want the read only rules if they are not active diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 783128b6..f17520d6 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -27,13 +27,7 @@ get_user_nets, insert_initial_communities, ) -from flowapp.output import ( - ROUTE_MODELS, - RULE_TYPES, - announce_route, - log_route, - log_withdraw, -) +from flowapp.output import ROUTE_MODELS, RuleTypes, announce_route, log_route, log_withdraw, RouteSources, Route from flowapp.utils import ( flash_errors, get_state_by_time, @@ -100,18 +94,28 @@ def reactivate_rule(rule_type, rule_id): if model.rstate_id == 1: # announce route - route = route_model(model, constants.ANNOUNCE) + command = route_model(model, constants.ANNOUNCE) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) # log changes log_route( session["user_id"], model, rule_type, - "{} / {}".format(session["user_email"], session["user_orgs"]), + f"{session['user_email']} / {session['user_orgs']}", ) else: # withdraw route - route = route_model(model, constants.WITHDRAW) + command = route_model(model, constants.WITHDRAW) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) # log changes log_withdraw( @@ -169,7 +173,12 @@ def delete_rule(rule_type, rule_id): model = db.session.query(model_name).get(rule_id) if model.id in session[constants.RULES_KEY]: # withdraw route - route = route_model(model, constants.WITHDRAW) + command = route_model(model, constants.WITHDRAW) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) log_withdraw( @@ -234,7 +243,7 @@ def group_delete(): """ rule_type = session[constants.TYPE_ARG] model_name = DATA_MODELS_NAMED[rule_type] - rule_type_int = constants.RULE_TYPES[rule_type] + rule_type_int = constants.RULE_TYPES_DICT[rule_type] route_model = ROUTE_MODELS[rule_type_int] rules = [str(x) for x in session[constants.RULES_KEY]] to_delete = request.form.getlist("delete-id") @@ -243,7 +252,12 @@ def group_delete(): for rule_id in to_delete: # withdraw route model = db.session.query(model_name).get(rule_id) - route = route_model(model, constants.WITHDRAW) + command = route_model(model, constants.WITHDRAW) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) log_withdraw( @@ -284,7 +298,7 @@ def group_update(): form_name = DATA_FORMS_NAMED[rule_type] to_update = request.form.getlist("delete-id") rule_type = session[constants.TYPE_ARG] - rule_type_int = constants.RULE_TYPES[rule_type] + rule_type_int = constants.RULE_TYPES_DICT[rule_type] rules = [str(x) for x in session[constants.RULES_KEY]] # redirect bad request if not set(to_update).issubset(set(rules)) or is_admin(session["user_roles"]): @@ -366,14 +380,19 @@ def group_update_save(rule_type): if model.rstate_id == 1: # announce route - route = route_model(model, constants.ANNOUNCE) + command = route_model(model, constants.ANNOUNCE) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) # log changes log_route( session["user_id"], model, rule_type, - "{} / {}".format(session["user_email"], session["user_orgs"]), + f"{session['user_email']} / {session['user_orgs']}", ) else: # withdraw route @@ -450,15 +469,20 @@ def ipv4_rule(): # announce route if model is in active state if model.rstate_id == 1: - route = messages.create_ipv4(model, constants.ANNOUNCE) + command = messages.create_ipv4(model, constants.ANNOUNCE) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) # log changes log_route( session["user_id"], model, - RULE_TYPES["IPv4"], - "{} / {}".format(session["user_email"], session["user_orgs"]), + RuleTypes.IPv4, + f"{session['user_email']} / {session['user_orgs']}", ) return redirect(url_for("index")) @@ -520,15 +544,20 @@ def ipv6_rule(): # announce routes if model.rstate_id == 1: - route = messages.create_ipv6(model, constants.ANNOUNCE) + command = messages.create_ipv6(model, constants.ANNOUNCE) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) # log changes log_route( session["user_id"], model, - RULE_TYPES["IPv6"], - "{} / {}".format(session["user_email"], session["user_orgs"]), + RuleTypes.IPv6, + f"{session['user_email']} / {session['user_orgs']}", ) return redirect(url_for("index")) @@ -586,14 +615,19 @@ def rtbh_rule(): flash(flash_message, "alert-success") # announce routes if model.rstate_id == 1: - route = messages.create_rtbh(model, constants.ANNOUNCE) + command = messages.create_rtbh(model, constants.ANNOUNCE) + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) # log changes log_route( session["user_id"], model, - RULE_TYPES["RTBH"], - "{} / {}".format(session["user_email"], session["user_orgs"]), + RuleTypes.RTBH, + f"{session['user_email']} / {session['user_orgs']}", ) return redirect(url_for("index")) @@ -687,7 +721,12 @@ def announce_all_routes(action=constants.ANNOUNCE): messages_all.extend(messages_v6) messages_all.extend(messages_rtbh) - for route in messages_all: + for command in messages_all: + route = Route( + author=f"{session['user_email']} / {session['user_orgs']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) if action == constants.WITHDRAW: From 29ff188b1179cfe2d98402d8dfd26402036f7e08 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 15 Oct 2024 13:03:34 +0200 Subject: [PATCH 23/54] fixed app context in test_forms, fixed bug in log_withdrawroute --- flowapp/tests/test_forms.py | 22 +++++++++++++++------- flowapp/views/api_common.py | 2 +- flowapp/views/rules.py | 8 ++++---- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/flowapp/tests/test_forms.py b/flowapp/tests/test_forms.py index 481354c9..76c98960 100644 --- a/flowapp/tests/test_forms.py +++ b/flowapp/tests/test_forms.py @@ -1,16 +1,24 @@ import pytest +from flask import Flask import flowapp.forms @pytest.fixture() -def ip_form(field_class): +def app(): + app = Flask(__name__) + app.secret_key = "test" + return app - form = flowapp.forms.IPForm() - form.source = field_class() - form.dest = field_class() - form.source_mask = field_class() - form.dest_mask = field_class() - return form + +@pytest.fixture() +def ip_form(app, field_class): + with app.test_request_context(): # Push the request context + form = flowapp.forms.IPForm() + form.source = field_class() + form.dest = field_class() + form.source_mask = field_class() + form.dest_mask = field_class() + return form def test_ip_form_created(ip_form): diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index 330c46fe..f819c5da 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -487,7 +487,7 @@ def delete_rule(current_user, rule_id, model_name, route_model, rule_type): log_withdraw( current_user["id"], - route, + route.command, rule_type, model.id, f"{current_user['uuid']} / {current_user['org']}", diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index f17520d6..3a3162f6 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -120,7 +120,7 @@ def reactivate_rule(rule_type, rule_id): # log changes log_withdraw( session["user_id"], - route, + route.command, rule_type, model.id, "{} / {}".format(session["user_email"], session["user_orgs"]), @@ -183,7 +183,7 @@ def delete_rule(rule_type, rule_id): log_withdraw( session["user_id"], - route, + route.command, rule_type, model.id, "{} / {}".format(session["user_email"], session["user_orgs"]), @@ -262,7 +262,7 @@ def group_delete(): log_withdraw( session["user_id"], - route, + route.command, rule_type_int, model.id, "{} / {}".format(session["user_email"], session["user_orgs"]), @@ -401,7 +401,7 @@ def group_update_save(rule_type): # log changes log_withdraw( session["user_id"], - route, + route.command, rule_type, model.id, "{} / {}".format(session["user_email"], session["user_orgs"]), From 46cd85993c5265e221e7e6dc4307bf7836d6f998 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 15 Oct 2024 17:12:39 +0200 Subject: [PATCH 24/54] modify organization model, add limit for number of rules --- .gitignore | 1 + flowapp/models.py | 78 +++++------------------- flowapp/tests/test_api_v3.py | 15 ++--- migrations/versions/1b25723059d6_.py | 44 -------------- migrations/versions/2bd0e800ab1c_.py | 36 ------------ migrations/versions/3003649af016_.py | 58 ------------------ migrations/versions/4af5ae4bae1c_.py | 38 ------------ migrations/versions/4fa1bacabe4d_.py | 28 --------- migrations/versions/5945c1418f0f_.py | 36 ------------ migrations/versions/67bb6c1b3898_.py | 45 -------------- migrations/versions/701711e8c4f4_.py | 88 ---------------------------- migrations/versions/76856add9483_.py | 38 ------------ migrations/versions/7a816ca986b3_.py | 32 ---------- migrations/versions/b3efc4d93b12_.py | 35 ----------- migrations/versions/d88d6bb3ae9b_.py | 43 -------------- migrations/versions/e25fdf3278bf_.py | 36 ------------ 16 files changed, 21 insertions(+), 630 deletions(-) delete mode 100644 migrations/versions/1b25723059d6_.py delete mode 100644 migrations/versions/2bd0e800ab1c_.py delete mode 100644 migrations/versions/3003649af016_.py delete mode 100644 migrations/versions/4af5ae4bae1c_.py delete mode 100644 migrations/versions/4fa1bacabe4d_.py delete mode 100644 migrations/versions/5945c1418f0f_.py delete mode 100644 migrations/versions/67bb6c1b3898_.py delete mode 100644 migrations/versions/701711e8c4f4_.py delete mode 100644 migrations/versions/76856add9483_.py delete mode 100644 migrations/versions/7a816ca986b3_.py delete mode 100644 migrations/versions/b3efc4d93b12_.py delete mode 100644 migrations/versions/d88d6bb3ae9b_.py delete mode 100644 migrations/versions/e25fdf3278bf_.py diff --git a/.gitignore b/.gitignore index 436de6f0..b91be95b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ config.py run.py +migrations/versions/ # PyCharm .idea/ diff --git a/flowapp/models.py b/flowapp/models.py index c274e932..91950092 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -15,9 +15,7 @@ user_organization = db.Table( "user_organization", db.Column("user_id", db.Integer, db.ForeignKey("user.id"), nullable=False), - db.Column( - "organization_id", db.Integer, db.ForeignKey("organization.id"), nullable=False - ), + db.Column("organization_id", db.Integer, db.ForeignKey("organization.id"), nullable=False), db.PrimaryKeyConstraint("user_id", "organization_id"), ) @@ -37,9 +35,7 @@ class User(db.Model): machineapikeys = db.relationship("MachineApiKey", back_populates="user", lazy="dynamic") role = db.relationship("Role", secondary=user_role, lazy="dynamic", backref="user") - organization = db.relationship( - "Organization", secondary=user_organization, lazy="dynamic", backref="user" - ) + organization = db.relationship("Organization", secondary=user_organization, lazy="dynamic", backref="user") def __init__(self, uuid, name=None, phone=None, email=None, comment=None): self.uuid = uuid @@ -130,6 +126,7 @@ class Organization(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(150), unique=True) arange = db.Column(db.Text) + rule_limit = db.Column(db.Integer, default=0) def __init__(self, name, arange): self.name = name @@ -186,9 +183,7 @@ class Community(db.Model): role_id = db.Column(db.Integer, db.ForeignKey("role.id"), nullable=False) role = db.relationship("Role", backref="community") - def __init__( - self, name, comm, larcomm, extcomm, description, as_path=False, role_id=2 - ): + def __init__(self, name, comm, larcomm, extcomm, description, as_path=False, role_id=2): self.name = name self.comm = comm self.larcomm = larcomm @@ -665,11 +660,7 @@ def insert_initial_actions(table, conn, *args, **kwargs): role_id=2, ) ) - conn.execute( - table.insert().values( - name="Discard", command="discard", description="Discard", role_id=2 - ) - ) + conn.execute(table.insert().values(name="Discard", command="discard", description="Discard", role_id=2)) @event.listens_for(Community.__table__, "after_create") @@ -715,16 +706,8 @@ def insert_initial_roles(table, conn, *args, **kwargs): @event.listens_for(Organization.__table__, "after_create") def insert_initial_organizations(table, conn, *args, **kwargs): - conn.execute( - table.insert().values( - name="TU Liberec", arange="147.230.0.0/16\n2001:718:1c01::/48" - ) - ) - conn.execute( - table.insert().values( - name="Cesnet", arange="147.230.0.0/16\n2001:718:1c01::/48" - ) - ) + conn.execute(table.insert().values(name="TU Liberec", arange="147.230.0.0/16\n2001:718:1c01::/48")) + conn.execute(table.insert().values(name="Cesnet", arange="147.230.0.0/16\n2001:718:1c01::/48")) @event.listens_for(Rstate.__table__, "after_create") @@ -832,9 +815,7 @@ def insert_users(users): db.session.commit() -def insert_user( - uuid, role_ids, org_ids, name=None, phone=None, email=None, comment=None -): +def insert_user(uuid, role_ids, org_ids, name=None, phone=None, email=None, comment=None): """ insert new user with multiple roles and organizations :param uuid: string unique user id (eppn or similar) @@ -894,9 +875,7 @@ def get_user_communities(user_roles): if max_role == 3: communities = db.session.query(Community).order_by("id") else: - communities = ( - db.session.query(Community).filter_by(role_id=max_role).order_by("id") - ) + communities = db.session.query(Community).filter_by(role_id=max_role).order_by("id") return [(g.id, g.name) for g in communities] @@ -909,9 +888,7 @@ def get_existing_action(name=None, command=None): :param command: string action command :return: action id """ - action = Action.query.filter( - (Action.name == name) | (Action.command == command) - ).first() + action = Action.query.filter((Action.name == name) | (Action.command == command)).first() return action.id if hasattr(action, "id") else None @@ -943,10 +920,7 @@ def get_ip_rules(rule_type, rule_state, sort="expires", order="desc"): sorting_ip4 = getattr(sorter_ip4, order) if comp_func: rules4 = ( - db.session.query(Flowspec4) - .filter(comp_func(Flowspec4.expires, today)) - .order_by(sorting_ip4()) - .all() + db.session.query(Flowspec4).filter(comp_func(Flowspec4.expires, today)).order_by(sorting_ip4()).all() ) else: rules4 = db.session.query(Flowspec4).order_by(sorting_ip4()).all() @@ -958,10 +932,7 @@ def get_ip_rules(rule_type, rule_state, sort="expires", order="desc"): sorting_ip6 = getattr(sorter_ip6, order) if comp_func: rules6 = ( - db.session.query(Flowspec6) - .filter(comp_func(Flowspec6.expires, today)) - .order_by(sorting_ip6()) - .all() + db.session.query(Flowspec6).filter(comp_func(Flowspec6.expires, today)).order_by(sorting_ip6()).all() ) else: rules6 = db.session.query(Flowspec6).order_by(sorting_ip6()).all() @@ -973,12 +944,7 @@ def get_ip_rules(rule_type, rule_state, sort="expires", order="desc"): sorting_rtbh = getattr(sorter_rtbh, order) if comp_func: - rules_rtbh = ( - db.session.query(RTBH) - .filter(comp_func(RTBH.expires, today)) - .order_by(sorting_rtbh()) - .all() - ) + rules_rtbh = db.session.query(RTBH).filter(comp_func(RTBH.expires, today)).order_by(sorting_rtbh()).all() else: rules_rtbh = db.session.query(RTBH).order_by(sorting_rtbh()).all() @@ -995,25 +961,13 @@ def get_user_rules_ids(user_id, rule_type): """ if rule_type == "ipv4": - rules4 = ( - db.session.query(Flowspec4.id) - .filter_by(user_id=user_id) - .all() - ) + rules4 = db.session.query(Flowspec4.id).filter_by(user_id=user_id).all() return [int(x[0]) for x in rules4] if rule_type == "ipv6": - rules6 = ( - db.session.query(Flowspec6.id) - .order_by(Flowspec6.expires.desc()) - .all() - ) + rules6 = db.session.query(Flowspec6.id).order_by(Flowspec6.expires.desc()).all() return [int(x[0]) for x in rules6] if rule_type == "rtbh": - rules_rtbh = ( - db.session.query(RTBH.id) - .filter_by(user_id=user_id) - .all() - ) + rules_rtbh = db.session.query(RTBH.id).filter_by(user_id=user_id).all() return [int(x[0]) for x in rules_rtbh] diff --git a/flowapp/tests/test_api_v3.py b/flowapp/tests/test_api_v3.py index 75abb73c..78b75432 100644 --- a/flowapp/tests/test_api_v3.py +++ b/flowapp/tests/test_api_v3.py @@ -164,7 +164,7 @@ def test_validation_rtbh_rule(client, db, jwt_token): data = json.loads(req.data) assert req.status_code == 400 assert data["message"] == "error - invalid request data" - assert type(data["validation_errors"]) == dict + assert type(data["validation_errors"]) is dict assert "ipv6" in data["validation_errors"] assert "ipv4" in data["validation_errors"] @@ -226,10 +226,7 @@ def test_all_validation_errors(client, db, jwt_token): """ test that creating with invalid data returns 400 and errors """ - req = client.post( - f"{V_PREFIX}/rules/ipv4", headers={"x-access-token": jwt_token}, json={"action": 2} - ) - data = json.loads(req.data) + req = client.post(f"{V_PREFIX}/rules/ipv4", headers={"x-access-token": jwt_token}, json={"action": 2}) assert req.status_code == 400 @@ -252,9 +249,7 @@ def test_validate_v6rule(client, db, jwt_token): data = json.loads(req.data) assert req.status_code == 400 assert len(data["validation_errors"]) > 0 - assert sorted(data["validation_errors"].keys()) == sorted( - ["action", "next_header", "dest", "source"] - ) + assert sorted(data["validation_errors"].keys()) == sorted(["action", "next_header", "dest", "source"]) # assert data['validation_errors'][0].startswith('Error in the Action') # assert data['validation_errors'][1].startswith('Error in the Source') # assert data['validation_errors'][2].startswith('Error in the Next Header') @@ -277,9 +272,7 @@ def test_timestamp_param(client, db, jwt_token): """ test that url param for time format works as expected """ - req = client.get( - f"{V_PREFIX}/rules?time_format=timestamp", headers={"x-access-token": jwt_token} - ) + req = client.get(f"{V_PREFIX}/rules?time_format=timestamp", headers={"x-access-token": jwt_token}) assert req.status_code == 200 diff --git a/migrations/versions/1b25723059d6_.py b/migrations/versions/1b25723059d6_.py deleted file mode 100644 index b3b328ac..00000000 --- a/migrations/versions/1b25723059d6_.py +++ /dev/null @@ -1,44 +0,0 @@ -"""empty message - -Revision ID: 1b25723059d6 -Revises: e25fdf3278bf -Create Date: 2019-08-23 12:49:57.512115 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '1b25723059d6' -down_revision = 'e25fdf3278bf' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('RTBH', 'community_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('RTBH', 'rstate_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('RTBH', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('RTBH', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('RTBH', 'rstate_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('RTBH', 'community_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - # ### end Alembic commands ### diff --git a/migrations/versions/2bd0e800ab1c_.py b/migrations/versions/2bd0e800ab1c_.py deleted file mode 100644 index bfea502d..00000000 --- a/migrations/versions/2bd0e800ab1c_.py +++ /dev/null @@ -1,36 +0,0 @@ -"""empty message - -Revision ID: 2bd0e800ab1c -Revises: 4fa1bacabe4d -Create Date: 2022-04-29 14:35:23.856715 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '2bd0e800ab1c' -down_revision = '4fa1bacabe4d' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('as_path', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('prefix', sa.String(length=120), nullable=True), - sa.Column('as_path', sa.String(length=250), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('prefix') - ) - op.add_column('community', sa.Column('as_path', sa.Boolean(), nullable=True)) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('community', 'as_path') - op.drop_table('as_path') - # ### end Alembic commands ### diff --git a/migrations/versions/3003649af016_.py b/migrations/versions/3003649af016_.py deleted file mode 100644 index b1623b8e..00000000 --- a/migrations/versions/3003649af016_.py +++ /dev/null @@ -1,58 +0,0 @@ -"""empty message - -Revision ID: 3003649af016 -Revises: 701711e8c4f4 -Create Date: 2019-08-26 08:43:04.219577 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '3003649af016' -down_revision = '701711e8c4f4' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('community', 'comm', - existing_type=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=120), - type_=sa.String(length=2047), - existing_nullable=True) - op.alter_column('community', 'description', - existing_type=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=260), - type_=sa.String(length=255), - existing_nullable=True) - op.alter_column('community', 'extcomm', - existing_type=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=120), - type_=sa.String(length=2047), - existing_nullable=True) - op.alter_column('community', 'larcomm', - existing_type=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=120), - type_=sa.String(length=2047), - existing_nullable=True) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('community', 'larcomm', - existing_type=sa.String(length=2047), - type_=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=120), - existing_nullable=True) - op.alter_column('community', 'extcomm', - existing_type=sa.String(length=2047), - type_=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=120), - existing_nullable=True) - op.alter_column('community', 'description', - existing_type=sa.String(length=255), - type_=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=260), - existing_nullable=True) - op.alter_column('community', 'comm', - existing_type=sa.String(length=2047), - type_=mysql.VARCHAR(collation=u'utf8_unicode_ci', length=120), - existing_nullable=True) - # ### end Alembic commands ### diff --git a/migrations/versions/4af5ae4bae1c_.py b/migrations/versions/4af5ae4bae1c_.py deleted file mode 100644 index 15017fb6..00000000 --- a/migrations/versions/4af5ae4bae1c_.py +++ /dev/null @@ -1,38 +0,0 @@ -"""empty message - -Revision ID: 4af5ae4bae1c -Revises: 67bb6c1b3898 -Create Date: 2024-03-27 18:19:35.721215 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '4af5ae4bae1c' -down_revision = '67bb6c1b3898' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('api_key', schema=None) as batch_op: - batch_op.add_column(sa.Column('comment', sa.String(length=255), nullable=True)) - - with op.batch_alter_table('machine_api_key', schema=None) as batch_op: - batch_op.add_column(sa.Column('readonly', sa.Boolean(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('machine_api_key', schema=None) as batch_op: - batch_op.drop_column('readonly') - - with op.batch_alter_table('api_key', schema=None) as batch_op: - batch_op.drop_column('comment') - - # ### end Alembic commands ### diff --git a/migrations/versions/4fa1bacabe4d_.py b/migrations/versions/4fa1bacabe4d_.py deleted file mode 100644 index 918364c0..00000000 --- a/migrations/versions/4fa1bacabe4d_.py +++ /dev/null @@ -1,28 +0,0 @@ -"""empty message - -Revision ID: 4fa1bacabe4d -Revises: 5945c1418f0f -Create Date: 2022-04-20 12:30:48.123941 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '4fa1bacabe4d' -down_revision = '5945c1418f0f' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('flowspec4', sa.Column('fragment', sa.String(length=255), nullable=True)) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('flowspec4', 'fragment') - # ### end Alembic commands ### diff --git a/migrations/versions/5945c1418f0f_.py b/migrations/versions/5945c1418f0f_.py deleted file mode 100644 index 6a825986..00000000 --- a/migrations/versions/5945c1418f0f_.py +++ /dev/null @@ -1,36 +0,0 @@ -"""empty message - -Revision ID: 5945c1418f0f -Revises: 3003649af016 -Create Date: 2021-03-09 12:57:56.549338 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '5945c1418f0f' -down_revision = '3003649af016' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('log', sa.Column('author', sa.String(length=1000), nullable=True)) - op.alter_column('log', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.drop_constraint('log_ibfk_1', 'log', type_='foreignkey') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_foreign_key('log_ibfk_1', 'log', 'user', ['user_id'], ['id']) - op.alter_column('log', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.drop_column('log', 'author') - # ### end Alembic commands ### diff --git a/migrations/versions/67bb6c1b3898_.py b/migrations/versions/67bb6c1b3898_.py deleted file mode 100644 index ec0d3e08..00000000 --- a/migrations/versions/67bb6c1b3898_.py +++ /dev/null @@ -1,45 +0,0 @@ -"""empty message - -Revision ID: 67bb6c1b3898 -Revises: 2bd0e800ab1c -Create Date: 2024-03-27 18:13:10.688958 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '67bb6c1b3898' -down_revision = '2bd0e800ab1c' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('machine_api_key', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('machine', sa.String(length=255), nullable=True), - sa.Column('key', sa.String(length=255), nullable=True), - sa.Column('expires', sa.DateTime(), nullable=True), - sa.Column('comment', sa.String(length=255), nullable=True), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), - sa.PrimaryKeyConstraint('id') - ) - with op.batch_alter_table('api_key', schema=None) as batch_op: - batch_op.add_column(sa.Column('readonly', sa.Boolean(), nullable=True)) - batch_op.add_column(sa.Column('expires', sa.DateTime(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('api_key', schema=None) as batch_op: - batch_op.drop_column('expires') - batch_op.drop_column('readonly') - - op.drop_table('machine_api_key') - # ### end Alembic commands ### diff --git a/migrations/versions/701711e8c4f4_.py b/migrations/versions/701711e8c4f4_.py deleted file mode 100644 index 2cb69a66..00000000 --- a/migrations/versions/701711e8c4f4_.py +++ /dev/null @@ -1,88 +0,0 @@ -"""empty message - -Revision ID: 701711e8c4f4 -Revises: 1b25723059d6 -Create Date: 2019-08-23 13:05:45.140334 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '701711e8c4f4' -down_revision = '1b25723059d6' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('action', 'role_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False, - existing_server_default=sa.text(u"'0'")) - op.alter_column('api_key', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('community', 'role_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('flowspec4', 'action_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('flowspec4', 'rstate_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('flowspec4', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('flowspec6', 'action_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('flowspec6', 'rstate_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('flowspec6', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - op.alter_column('log', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('log', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('flowspec6', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('flowspec6', 'rstate_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('flowspec6', 'action_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('flowspec4', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('flowspec4', 'rstate_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('flowspec4', 'action_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('community', 'role_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('api_key', 'user_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True) - op.alter_column('action', 'role_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True, - existing_server_default=sa.text(u"'0'")) - # ### end Alembic commands ### diff --git a/migrations/versions/76856add9483_.py b/migrations/versions/76856add9483_.py deleted file mode 100644 index 41002034..00000000 --- a/migrations/versions/76856add9483_.py +++ /dev/null @@ -1,38 +0,0 @@ -"""empty message - -Revision ID: 76856add9483 -Revises: b3efc4d93b12 -Create Date: 2019-01-28 10:22:17.904055 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '76856add9483' -down_revision = 'b3efc4d93b12' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('community', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=120), nullable=True), - sa.Column('command', sa.String(length=120), nullable=True), - sa.Column('description', sa.String(length=260), nullable=True), - sa.Column('role_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['role_id'], ['role.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('command'), - sa.UniqueConstraint('name') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('community') - # ### end Alembic commands ### diff --git a/migrations/versions/7a816ca986b3_.py b/migrations/versions/7a816ca986b3_.py deleted file mode 100644 index d0982d6c..00000000 --- a/migrations/versions/7a816ca986b3_.py +++ /dev/null @@ -1,32 +0,0 @@ -"""empty message - -Revision ID: 7a816ca986b3 -Revises: 76856add9483 -Create Date: 2019-01-28 11:59:08.217024 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = '7a816ca986b3' -down_revision = '76856add9483' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('RTBH', sa.Column('community_id', sa.Integer(), nullable=True)) - op.create_foreign_key(None, 'RTBH', 'community', ['community_id'], ['id']) - op.drop_column('RTBH', 'community') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('RTBH', sa.Column('community', mysql.VARCHAR(length=255), nullable=True)) - op.drop_constraint(None, 'RTBH', type_='foreignkey') - op.drop_column('RTBH', 'community_id') - # ### end Alembic commands ### diff --git a/migrations/versions/b3efc4d93b12_.py b/migrations/versions/b3efc4d93b12_.py deleted file mode 100644 index 6f8450f1..00000000 --- a/migrations/versions/b3efc4d93b12_.py +++ /dev/null @@ -1,35 +0,0 @@ -"""empty message - -Revision ID: b3efc4d93b12 -Revises: d88d6bb3ae9b -Create Date: 2018-11-15 13:51:05.759008 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'b3efc4d93b12' -down_revision = 'd88d6bb3ae9b' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('api_key', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('machine', sa.String(length=255), nullable=True), - sa.Column('key', sa.String(length=255), nullable=True), - sa.Column('user_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('api_key') - # ### end Alembic commands ### diff --git a/migrations/versions/d88d6bb3ae9b_.py b/migrations/versions/d88d6bb3ae9b_.py deleted file mode 100644 index 64af69bc..00000000 --- a/migrations/versions/d88d6bb3ae9b_.py +++ /dev/null @@ -1,43 +0,0 @@ -"""empty message - -Revision ID: d88d6bb3ae9b -Revises: -Create Date: 2018-11-15 13:30:47.322461 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = 'd88d6bb3ae9b' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('rulestate') - op.alter_column('action', 'role_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=True, - existing_server_default=sa.text(u"'0'")) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column('action', 'role_id', - existing_type=mysql.INTEGER(display_width=11), - nullable=False, - existing_server_default=sa.text(u"'0'")) - op.create_table('rulestate', - sa.Column('id', mysql.INTEGER(display_width=10, unsigned=True), autoincrement=True, nullable=False), - sa.Column('description', mysql.VARCHAR(collation=u'utf8mb4_unicode_ci', length=260), nullable=False), - sa.PrimaryKeyConstraint('id'), - mysql_collate=u'utf8mb4_unicode_ci', - mysql_default_charset=u'utf8mb4', - mysql_engine=u'InnoDB' - ) - # ### end Alembic commands ### diff --git a/migrations/versions/e25fdf3278bf_.py b/migrations/versions/e25fdf3278bf_.py deleted file mode 100644 index 08cf9723..00000000 --- a/migrations/versions/e25fdf3278bf_.py +++ /dev/null @@ -1,36 +0,0 @@ -"""empty message - -Revision ID: e25fdf3278bf -Revises: 7a816ca986b3 -Create Date: 2019-08-22 13:39:34.983566 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = 'e25fdf3278bf' -down_revision = '7a816ca986b3' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('community', sa.Column('comm', sa.String(length=120), nullable=True)) - op.add_column('community', sa.Column('extcomm', sa.String(length=120), nullable=True)) - op.add_column('community', sa.Column('larcomm', sa.String(length=120), nullable=True)) - op.drop_index('command', table_name='community') - op.drop_column('community', 'command') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('community', sa.Column('command', mysql.VARCHAR(collation=u'utf8_unicode_ci', length=120), nullable=True)) - op.create_index('command', 'community', ['command'], unique=True) - op.drop_column('community', 'larcomm') - op.drop_column('community', 'extcomm') - op.drop_column('community', 'comm') - # ### end Alembic commands ### From 420018a2ed25cd74c4eeb84de5a7332fb8ca2b69 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 15 Oct 2024 18:15:23 +0200 Subject: [PATCH 25/54] new migrations init, add rule_limit to organization, migrate db, update from and template for organization --- .gitignore | 1 - flowapp/forms.py | 145 +++++------------- flowapp/models.py | 3 +- flowapp/templates/pages/orgs.html | 2 + flowapp/views/admin.py | 4 +- migrations/README | 1 + migrations/README.md | 10 -- migrations/alembic.ini | 7 +- migrations/env.py | 66 +++++--- ...224c9f1f_add_rule_limit_to_organization.py | 32 ++++ 10 files changed, 133 insertions(+), 138 deletions(-) create mode 100644 migrations/README delete mode 100644 migrations/README.md create mode 100644 migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py diff --git a/.gitignore b/.gitignore index b91be95b..436de6f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ config.py run.py -migrations/versions/ # PyCharm .idea/ diff --git a/flowapp/forms.py b/flowapp/forms.py index ae83a1e2..4818a6e1 100644 --- a/flowapp/forms.py +++ b/flowapp/forms.py @@ -55,7 +55,7 @@ class MultiFormatDateTimeLocalField(DateTimeField): def __init__(self, *args, **kwargs): kwargs.setdefault("format", "%Y-%m-%dT%H:%M") - self.unlimited = kwargs.pop('unlimited', False) + self.unlimited = kwargs.pop("unlimited", False) self.pref_format = None super().__init__(*args, **kwargs) @@ -92,9 +92,7 @@ class UserForm(FlaskForm): ], ) - email = StringField( - "Email", validators=[Optional(), Email("Please provide valid email")] - ) + email = StringField("Email", validators=[Optional(), Email("Please provide valid email")]) comment = StringField("Notice", validators=[Optional()]) @@ -102,9 +100,7 @@ class UserForm(FlaskForm): phone = StringField("Contact phone", validators=[Optional()]) - role_ids = SelectMultipleField( - "Role", coerce=int, validators=[DataRequired("Select at last one role")] - ) + role_ids = SelectMultipleField("Role", coerce=int, validators=[DataRequired("Select at last one role")]) org_ids = SelectMultipleField( "Organization", @@ -124,13 +120,13 @@ class ApiKeyForm(FlaskForm): validators=[DataRequired(), IPAddress(message="provide valid IP address")], ) - comment = TextAreaField( - "Your comment for this key", validators=[Optional(), Length(max=255)] - ) + comment = TextAreaField("Your comment for this key", validators=[Optional(), Length(max=255)]) expires = MultiFormatDateTimeLocalField( "Key expiration. Leave blank for non expring key (not-recomended).", - format=FORM_TIME_PATTERN, validators=[Optional()], unlimited=True + format=FORM_TIME_PATTERN, + validators=[Optional()], + unlimited=True, ) readonly = BooleanField("Read only key", default=False) @@ -150,13 +146,13 @@ class MachineApiKeyForm(FlaskForm): validators=[DataRequired(), IPAddress(message="provide valid IP address")], ) - comment = TextAreaField( - "Your comment for this key", validators=[Optional(), Length(max=255)] - ) + comment = TextAreaField("Your comment for this key", validators=[Optional(), Length(max=255)]) expires = MultiFormatDateTimeLocalField( "Key expiration. Leave blank for non expring key (not-recomended).", - format=FORM_TIME_PATTERN, validators=[Optional()], unlimited=True + format=FORM_TIME_PATTERN, + validators=[Optional()], + unlimited=True, ) readonly = BooleanField("Read only key", default=False) @@ -172,6 +168,14 @@ class OrganizationForm(FlaskForm): name = StringField("Organization name", validators=[Optional(), Length(max=150)]) + rule_limit = IntegerField( + "Maximum number of rules, 0 for unlimited", + validators=[ + Optional(), + NumberRange(min=0, max=1000, message="invalid mask value (0-1000)"), + ], + ) + arange = TextAreaField( "Organization Adress Range - one range per row", validators=[Optional(), NetRangeString()], @@ -214,9 +218,7 @@ class CommunityForm(FlaskForm): used in Admin """ - name = StringField( - "Community short name", validators=[Length(max=120), DataRequired()] - ) + name = StringField("Community short name", validators=[Length(max=120), DataRequired()]) comm = StringField("Community value", validators=[Length(max=2046)]) @@ -320,31 +322,19 @@ def validate(self): # if none is set, validation fails # if one is set, validation passes if self.ipv4.data and self.ipv6.data: - self.ipv4.errors.append( - "IPv4 and IPv6 are mutually exclusive in RTBH rule." - ) - self.ipv6.errors.append( - "IPv4 and IPv6 are mutually exclusive in RTBH rule." - ) + self.ipv4.errors.append("IPv4 and IPv6 are mutually exclusive in RTBH rule.") + self.ipv6.errors.append("IPv4 and IPv6 are mutually exclusive in RTBH rule.") result = False - if self.ipv4.data and not address_with_mask( - self.ipv4.data, self.ipv4_mask.data - ): + if self.ipv4.data and not address_with_mask(self.ipv4.data, self.ipv4_mask.data): self.ipv4.errors.append( - "This is not valid combination of address {} and mask {}.".format( - self.ipv4.data, self.ipv4_mask.data - ) + "This is not valid combination of address {} and mask {}.".format(self.ipv4.data, self.ipv4_mask.data) ) result = False - if self.ipv6.data and not address_with_mask( - self.ipv6.data, self.ipv6_mask.data - ): + if self.ipv6.data and not address_with_mask(self.ipv6.data, self.ipv6_mask.data): self.ipv6.errors.append( - "This is not valid combination of address {} and mask {}.".format( - self.ipv6.data, self.ipv6_mask.data - ) + "This is not valid combination of address {} and mask {}.".format(self.ipv6.data, self.ipv6_mask.data) ) result = False @@ -352,16 +342,8 @@ def validate(self): ipv4_in_range = address_in_range(self.ipv4.data, self.net_ranges) if not (ipv6_in_range or ipv4_in_range): - self.ipv6.errors.append( - "IPv4 or IPv6 address must be in organization range : {}.".format( - self.net_ranges - ) - ) - self.ipv4.errors.append( - "IPv4 or IPv6 address must be in organization range : {}.".format( - self.net_ranges - ) - ) + self.ipv6.errors.append("IPv4 or IPv6 address must be in organization range : {}.".format(self.net_ranges)) + self.ipv4.errors.append("IPv4 or IPv6 address must be in organization range : {}.".format(self.net_ranges)) result = False return result @@ -381,9 +363,7 @@ def __init__(self, *args, **kwargs): source_mask = None dest = None dest_mask = None - flags = SelectMultipleField( - "TCP flag(s)", choices=TCP_FLAGS, validators=[Optional()] - ) + flags = SelectMultipleField("TCP flag(s)", choices=TCP_FLAGS, validators=[Optional()]) source_port = StringField( "Source port(s) - ; separated ", @@ -406,9 +386,7 @@ def __init__(self, *args, **kwargs): validators=[DataRequired(message="Please select an action for the rule.")], ) - expires = MultiFormatDateTimeLocalField( - "Expires", format="%Y-%m-%dT%H:%M", validators=[InputRequired()] - ) + expires = MultiFormatDateTimeLocalField("Expires", format="%Y-%m-%dT%H:%M", validators=[InputRequired()]) comment = arange = TextAreaField("Comments") @@ -434,9 +412,7 @@ def validate_source_address(self): validate source address, set error message if validation fails :return: boolean validation result """ - if self.source.data and not address_with_mask( - self.source.data, self.source_mask.data - ): + if self.source.data and not address_with_mask(self.source.data, self.source_mask.data): self.source.errors.append( "This is not valid combination of address {} and mask {}.".format( self.source.data, self.source_mask.data @@ -451,13 +427,9 @@ def validate_dest_address(self): validate dest address, set error message if validation fails :return: boolean validation result """ - if self.dest.data and not address_with_mask( - self.dest.data, self.dest_mask.data - ): + if self.dest.data and not address_with_mask(self.dest.data, self.dest_mask.data): self.dest.errors.append( - "This is not valid combination of address {} and mask {}.".format( - self.dest.data, self.dest_mask.data - ) + "This is not valid combination of address {} and mask {}.".format(self.dest.data, self.dest_mask.data) ) return False @@ -473,35 +445,15 @@ def validate_address_ranges(self): if not (self.source.data or self.dest.data): whole_world_member = whole_world_range(self.net_ranges, self.zero_address) if not whole_world_member: - self.source.errors.append( - "Source or dest must be in organization range : {}.".format( - self.net_ranges - ) - ) - self.dest.errors.append( - "Source or dest must be in organization range : {}.".format( - self.net_ranges - ) - ) + self.source.errors.append("Source or dest must be in organization range : {}.".format(self.net_ranges)) + self.dest.errors.append("Source or dest must be in organization range : {}.".format(self.net_ranges)) return False else: - source_in_range = network_in_range( - self.source.data, self.source_mask.data, self.net_ranges - ) - dest_in_range = network_in_range( - self.dest.data, self.dest_mask.data, self.net_ranges - ) + source_in_range = network_in_range(self.source.data, self.source_mask.data, self.net_ranges) + dest_in_range = network_in_range(self.dest.data, self.dest_mask.data, self.net_ranges) if not (source_in_range or dest_in_range): - self.source.errors.append( - "Source or dest must be in organization range : {}.".format( - self.net_ranges - ) - ) - self.dest.errors.append( - "Source or dest must be in organization range : {}.".format( - self.net_ranges - ) - ) + self.source.errors.append("Source or dest must be in organization range : {}.".format(self.net_ranges)) + self.dest.errors.append("Source or dest must be in organization range : {}.".format(self.net_ranges)) return False return True @@ -567,17 +519,8 @@ def validate_ipv_specific(self): :return: boolean validation result """ - if ( - self.flags.data - and self.protocol.data - and len(self.flags.data) > 0 - and self.protocol.data != "tcp" - ): - self.flags.errors.append( - "Can not set TCP flags for protocol {} !".format( - self.protocol.data.upper() - ) - ) + if self.flags.data and self.protocol.data and len(self.flags.data) > 0 and self.protocol.data != "tcp": + self.flags.errors.append("Can not set TCP flags for protocol {} !".format(self.protocol.data.upper())) return False return True @@ -630,11 +573,7 @@ def validate_ipv_specific(self): :return: boolean validation result """ if len(self.flags.data) > 0 and self.next_header.data != "tcp": - self.flags.errors.append( - "Can not set TCP flags for next-header {} !".format( - self.next_header.data.upper() - ) - ) + self.flags.errors.append("Can not set TCP flags for next-header {} !".format(self.next_header.data.upper())) return False return True diff --git a/flowapp/models.py b/flowapp/models.py index 91950092..6568be01 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -128,9 +128,10 @@ class Organization(db.Model): arange = db.Column(db.Text) rule_limit = db.Column(db.Integer, default=0) - def __init__(self, name, arange): + def __init__(self, name, arange, rule_limit=0): self.name = name self.arange = arange + self.rule_limit = rule_limit def __repr__(self): return self.name diff --git a/flowapp/templates/pages/orgs.html b/flowapp/templates/pages/orgs.html index 9d2b2cb2..120b5c43 100644 --- a/flowapp/templates/pages/orgs.html +++ b/flowapp/templates/pages/orgs.html @@ -4,12 +4,14 @@ + {% for org in orgs %} +
NameLimit for rules Adress Ranges action
{{ org.name }}{{ "unlimited" if org.rule_limit == 0 else org.rule_limit }} {% set rows = org.arange.split() %}
    diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index fab9373d..6880d204 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -213,7 +213,7 @@ def organization(): # test if user is unique exist = db.session.query(Organization).filter_by(name=form.name.data).first() if not exist: - org = Organization(name=form.name.data, arange=form.arange.data) + org = Organization(name=form.name.data, arange=form.arange.data, rule_limit=form.rule_limit.data) db.session.add(org) db.session.commit() flash("Organization saved") @@ -240,7 +240,7 @@ def edit_organization(org_id): if request.method == "POST" and form.validate(): form.populate_obj(org) db.session.commit() - flash("Organization updated") + flash("Organization updated", "alert-success") return redirect(url_for("admin.organizations")) action_url = url_for("admin.edit_organization", org_id=org.id) diff --git a/migrations/README b/migrations/README new file mode 100644 index 00000000..0e048441 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Single-database configuration for Flask. diff --git a/migrations/README.md b/migrations/README.md deleted file mode 100644 index f28162a2..00000000 --- a/migrations/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# DB migrations when schema changes - -In top dir run DB Migration script - -### Usage: -``` -flask db migrate # creates migration script -flask db upgrade # upgrades database with migration script -``` -[https://flask-migrate.readthedocs.io/en/latest/](https://flask-migrate.readthedocs.io/en/latest/) \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini index f8ed4801..ec9d45c2 100644 --- a/migrations/alembic.ini +++ b/migrations/alembic.ini @@ -11,7 +11,7 @@ # Logging configuration [loggers] -keys = root,sqlalchemy,alembic +keys = root,sqlalchemy,alembic,flask_migrate [handlers] keys = console @@ -34,6 +34,11 @@ level = INFO handlers = qualname = alembic +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + [handler_console] class = StreamHandler args = (sys.stderr,) diff --git a/migrations/env.py b/migrations/env.py index 23663ff2..4c970927 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,8 +1,9 @@ -from __future__ import with_statement -from alembic import context -from sqlalchemy import engine_from_config, pool -from logging.config import fileConfig import logging +from logging.config import fileConfig + +from flask import current_app + +from alembic import context # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -13,14 +14,30 @@ fileConfig(config.config_file_name) logger = logging.getLogger('alembic.env') + +def get_engine(): + try: + # this works with Flask-SQLAlchemy<3 and Alchemical + return current_app.extensions['migrate'].db.get_engine() + except (TypeError, AttributeError): + # this works with Flask-SQLAlchemy>=3 + return current_app.extensions['migrate'].db.engine + + +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace( + '%', '%%') + except AttributeError: + return str(get_engine().url).replace('%', '%%') + + # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from flask import current_app -config.set_main_option('sqlalchemy.url', - current_app.config.get('SQLALCHEMY_DATABASE_URI')) -target_metadata = current_app.extensions['migrate'].db.metadata +config.set_main_option('sqlalchemy.url', get_engine_url()) +target_db = current_app.extensions['migrate'].db # other values from the config, defined by the needs of env.py, # can be acquired: @@ -28,6 +45,12 @@ # ... etc. +def get_metadata(): + if hasattr(target_db, 'metadatas'): + return target_db.metadatas[None] + return target_db.metadata + + def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -41,7 +64,9 @@ def run_migrations_offline(): """ url = config.get_main_option("sqlalchemy.url") - context.configure(url=url) + context.configure( + url=url, target_metadata=get_metadata(), literal_binds=True + ) with context.begin_transaction(): context.run_migrations() @@ -65,21 +90,22 @@ def process_revision_directives(context, revision, directives): directives[:] = [] logger.info('No changes in schema detected.') - engine = engine_from_config(config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) + conf_args = current_app.extensions['migrate'].configure_args + if conf_args.get("process_revision_directives") is None: + conf_args["process_revision_directives"] = process_revision_directives - connection = engine.connect() - context.configure(connection=connection, - target_metadata=target_metadata, - process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args) + connectable = get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=get_metadata(), + **conf_args + ) - try: with context.begin_transaction(): context.run_migrations() - finally: - connection.close() + if context.is_offline_mode(): run_migrations_offline() diff --git a/migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py b/migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py new file mode 100644 index 00000000..97b623de --- /dev/null +++ b/migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py @@ -0,0 +1,32 @@ +"""add rule_limit to organization + +Revision ID: 7719224c9f1f +Revises: +Create Date: 2024-10-15 15:44:56.937747 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7719224c9f1f' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.add_column(sa.Column('rule_limit', sa.Integer(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.drop_column('rule_limit') + + # ### end Alembic commands ### From cae5efe025e661c5881cb5a00b9c142bc74c74e1 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 17 Oct 2024 18:13:20 +0200 Subject: [PATCH 26/54] WIP - changed login for local_auth, created dialog for choosing the organization for the session. From now only one organization is used for user in single session. --- flowapp/__init__.py | 77 ++++++++++++++---- flowapp/auth.py | 59 ++++++-------- flowapp/constants.py | 6 ++ flowapp/forms.py | 6 ++ flowapp/models.py | 80 +++++++++++++++++++ flowapp/output.py | 6 -- flowapp/templates/forms/ipv4_rule.html | 7 +- flowapp/templates/layouts/default.html | 8 +- flowapp/templates/pages/org_modal.html | 23 ++++++ flowapp/validators.py | 44 +++------- flowapp/views/api_common.py | 4 +- flowapp/views/rules.py | 21 ++++- ...ed7f6_add_rule_counters_to_organization.py | 36 +++++++++ 13 files changed, 282 insertions(+), 95 deletions(-) create mode 100644 flowapp/templates/pages/org_modal.html create mode 100644 migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py diff --git a/flowapp/__init__.py b/flowapp/__init__.py index bf6c77c9..3ef31611 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -86,13 +86,9 @@ def login(user_info): uuid = user_info.get("eppn") except KeyError: uuid = False - return redirect("/") - else: - try: - _register_user_to_session(uuid) - except AttributeError: - pass - return redirect("/") + return render_template("errors/401.html") + + return _handle_login(uuid) @app.route("/logout") def logout(): @@ -108,18 +104,30 @@ def ext_login(): return render_template("errors/401.html") uuid = request.headers.get(header_name) - if uuid: - try: - _register_user_to_session(uuid) - except AttributeError: - return render_template("errors/401.html") - return redirect("/") + if not uuid: + return render_template("errors/401.html") + + return _handle_login(uuid) + + @app.route("/local-login") + def local_login(): + print("Local login started") + if not app.config.get("LOCAL_AUTH", False): + print("Local auth not enabled") + return render_template("errors/401.html") + + uuid = app.config.get("LOCAL_USER_UUID", False) + if not uuid: + print("Local user not set") + return render_template("errors/401.html") + + print(f"Local login with {uuid}") + return _handle_login(uuid) @app.route("/") @auth_required def index(): - logger.debug("That's it, beautiful and simple logging!") try: rtype = session[constants.TYPE_ARG] except KeyError: @@ -150,6 +158,20 @@ def index(): ) ) + @app.route("/select_org", defaults={"org_id": None}) + @app.route("/select_org/") + def select_org(org_id=None): + uuid = session.get("user_uuid") + user = db.session.query(models.User).filter_by(uuid=uuid).first() + orgs = user.organization + if org_id: + org = db.session.query(models.Organization).filter_by(id=org_id).first() + session["user_org_id"] = org.id + session["user_org"] = org.name + return redirect("/") + + return render_template("pages/org_modal.html", orgs=orgs) + @app.teardown_appcontext def shutdown_session(exception=None): db.session.remove() @@ -198,18 +220,41 @@ def format_datetime(value): format = "y/MM/dd HH:mm" return babel.dates.format_datetime(value, format) + def _handle_login(uuid: str): + """ + handles rest of login process + """ + multiple_orgs = False + try: + user, multiple_orgs = _register_user_to_session(uuid) + except AttributeError as e: + app.logger.exception(e) + return render_template("errors/401.html") + + if multiple_orgs: + return redirect(url_for("select_org", org_id=None)) + + # set user org to session + user_org = user.organization.first() + session["user_org"] = user_org.name + session["user_org_id"] = user_org.id + + return redirect("/") + def _register_user_to_session(uuid: str): print(f"Registering user {uuid} to session") user = db.session.query(models.User).filter_by(uuid=uuid).first() + print(f"Got user {user} from DB") session["user_uuid"] = user.uuid session["user_email"] = user.uuid session["user_name"] = user.name session["user_id"] = user.id session["user_roles"] = [role.name for role in user.role.all()] - session["user_orgs"] = ", ".join(org.name for org in user.organization.all()) session["user_role_ids"] = [role.id for role in user.role.all()] - session["user_org_ids"] = [org.id for org in user.organization.all()] roles = [i > 1 for i in session["user_role_ids"]] session["can_edit"] = True if all(roles) and roles else [] + # check if user has multiple organizations and return True if so + print(f"DEBUG SESSION {session}") + return user, len(user.organization.all()) > 1 return app diff --git a/flowapp/auth.py b/flowapp/auth.py index d1659af1..55b1e9fd 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -13,12 +13,18 @@ def auth_required(f): @wraps(f) def decorated(*args, **kwargs): - if not check_auth(get_user()): + user = session.get("user_uuid", False) + if not user: if current_app.config.get("SSO_AUTH"): current_app.logger.warning("SSO AUTH SET BUT FAILS") - return redirect("/login") - elif current_app.config.get("HEADER_AUTH", False): - return redirect("/ext-login") + return redirect(url_for("login")) + + if current_app.config.get("HEADER_AUTH", False): + return redirect(url_for("ext_login")) + + if current_app.config.get("LOCAL_AUTH"): + return redirect(url_for("local_login")) + return f(*args, **kwargs) return decorated @@ -70,18 +76,6 @@ def decorated(*args, **kwargs): return decorated -def get_user(): - """ - get user from session - """ - try: - uuid = session["user_uuid"] - except KeyError: - uuid = False - - return uuid - - def check_auth(uuid): """ This function is every time when someone accessing the endpoint @@ -93,30 +87,25 @@ def check_auth(uuid): session["app_version"] = __version__ - if current_app.config.get("SSO_AUTH"): - current_app.logger.warning("CHECK AUTH, SS AUTH SET", uuid) - # SSO AUTH - exist = False - if uuid: - exist = db.session.query(User).filter_by(uuid=uuid).first() - return exist - elif current_app.config.get("HEADER_AUTH", False): + if current_app.config.get("HEADER_AUTH", False): # External auth (for example apache) header_name = current_app.config.get("AUTH_HEADER_NAME", "X-Authenticated-User") if header_name not in request.headers or not session.get("user_uuid"): return False return db.session.query(User).filter_by(uuid=request.headers.get(header_name)) - else: - # Localhost login / no check / works only in development environment - session["user_email"] = current_app.config["LOCAL_USER_UUID"] - session["user_id"] = current_app.config["LOCAL_USER_ID"] - session["user_roles"] = current_app.config["LOCAL_USER_ROLES"] - session["user_orgs"] = ", ".join(org["name"] for org in current_app.config["LOCAL_USER_ORGS"]) - session["user_role_ids"] = current_app.config["LOCAL_USER_ROLE_IDS"] - session["user_org_ids"] = current_app.config["LOCAL_USER_ORG_IDS"] - roles = [i > 1 for i in session["user_role_ids"]] - session["can_edit"] = True if all(roles) and roles else [] - return True + + if current_app.config.get("SSO_AUTH"): + current_app.logger.warning("CHECK AUTH, SSO AUTH SET UUID : {uuid}") + elif current_app.config.get("LOCAL_AUTH"): + uuid = current_app.config.get("LOCAL_USER_UUID", False) + current_app.logger.warning(f"CHECK AUTH, LOCAL AUTH SET UUID: {uuid}") + + exist = False + if uuid: + exist = db.session.query(User).filter_by(uuid=uuid).first() + + current_app.logger.debug(f"CHECK AUTH RETURN, UUID: {uuid}, EXIST: {exist}") + return exist def check_access_rights(current_user, model_id): diff --git a/flowapp/constants.py b/flowapp/constants.py index 74e3352f..3c22be30 100644 --- a/flowapp/constants.py +++ b/flowapp/constants.py @@ -56,3 +56,9 @@ ] FORM_TIME_PATTERN = "%Y-%m-%dT%H:%M" + + +class RuleTypes: + RTBH = 1 + IPv4 = 4 + IPv6 = 6 diff --git a/flowapp/forms.py b/flowapp/forms.py index 4818a6e1..14b817fb 100644 --- a/flowapp/forms.py +++ b/flowapp/forms.py @@ -386,6 +386,12 @@ def __init__(self, *args, **kwargs): validators=[DataRequired(message="Please select an action for the rule.")], ) + organization = SelectField( + "Organization", + coerce=int, + validators=[DataRequired(message="Please select one of your organizations.")], + ) + expires = MultiFormatDateTimeLocalField("Expires", format="%Y-%m-%dT%H:%M", validators=[InputRequired()]) comment = arange = TextAreaField("Comments") diff --git a/flowapp/models.py b/flowapp/models.py index 6568be01..e001bc87 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -2,6 +2,7 @@ from sqlalchemy import event from datetime import datetime from flowapp import db, utils +from flowapp.constants import RuleTypes # models and tables @@ -127,6 +128,9 @@ class Organization(db.Model): name = db.Column(db.String(150), unique=True) arange = db.Column(db.Text) rule_limit = db.Column(db.Integer, default=0) + ipv4count = db.Column(db.Integer, default=0) + ipv6count = db.Column(db.Integer, default=0) + rtbhcount = db.Column(db.Integer, default=0) def __init__(self, name, arange, rule_limit=0): self.name = name @@ -136,6 +140,31 @@ def __init__(self, name, arange, rule_limit=0): def __repr__(self): return self.name + def get_users(self): + """ + Returns all users associated with this organization. + """ + # self.user is the backref from the user_organization relationship + return self.user + + def count_rules(self, rule_type): + """ + Count the number of rules of a certain type for this organization + :param rule_type: RuleTypes + :return: integer + """ + if rule_type == RuleTypes.IPv4: + ctr = 0 + for user in self.user: + for rule in user.flowspec4: + if rule.rstate_id == 1: + ctr += 1 + return ctr + if rule_type == RuleTypes.IPv6: + return self.ipv6count + if rule_type == RuleTypes.RTBH: + return self.rtbhcount + class ASPath(db.Model): id = db.Column(db.Integer, primary_key=True) @@ -719,6 +748,37 @@ def insert_initial_rulestates(table, conn, *args, **kwargs): # Misc functions +def check_rule_limit(org_id: int, rule_type: RuleTypes): + """ + Check if the organization has reached the rule limit + :param org_id: integer organization id + :param rule_type: RuleType rule type + :return: boolean + """ + org = Organization.query.filter_by(id=org_id).first() + if rule_type == RuleTypes.IPv4: + return org.ipv4count >= org.rule_limit + if rule_type == RuleTypes.IPv6: + return org.ipv6count >= org.rule_limit + if rule_type == RuleTypes.RTBH: + return org.rtbhcount >= org.rule_limit + + +def increment_rule_count(rule_type: RuleTypes, org_id: int): + """ + Increment rule count for organization + :param rule_type: RuleType rule type + :param org_id: integer organization id + :return: None + """ + org = Organization.query.filter_by(id=org_id).first() + if rule_type == RuleTypes.IPv4: + org.ipv4count += 1 + if rule_type == RuleTypes.IPv6: + org.ipv6count += 1 + if rule_type == RuleTypes.RTBH: + org.rtbhcount += 1 + db.session.commit() def get_ipv4_model_if_exists(form_data, rstate_id=1): @@ -855,6 +915,26 @@ def get_user_nets(user_id): return result +def get_user_orgs_choices(user_id): + """ + Return list of orgs as choices for form + """ + user = db.session.query(User).filter_by(id=user_id).first() + orgs = user.organization + + return [(g.id, g.name) for g in orgs] + + +def get_user_orgs_limits(user_id): + """ + Return list of orgs as choices for form + """ + user = db.session.query(User).filter_by(id=user_id).first() + orgs = user.organization + + return [(g.id, g.name) for g in orgs] + + def get_user_actions(user_roles): """ Return list of actions based on current user role diff --git a/flowapp/output.py b/flowapp/output.py index 29c34ff7..e2868277 100644 --- a/flowapp/output.py +++ b/flowapp/output.py @@ -20,12 +20,6 @@ } -class RuleTypes: - RTBH = 1 - IPv4 = 4 - IPv6 = 6 - - class RouteSources: UI = "UI" API = "API" diff --git a/flowapp/templates/forms/ipv4_rule.html b/flowapp/templates/forms/ipv4_rule.html index c1c7d233..700048e9 100644 --- a/flowapp/templates/forms/ipv4_rule.html +++ b/flowapp/templates/forms/ipv4_rule.html @@ -52,10 +52,10 @@

    {{ title or 'New'}} IPv4 rule

    -
    +
    {{ render_field(form.action) }}
    -
    +
    @@ -76,6 +76,9 @@

    {{ title or 'New'}} IPv4 rule

    {% endif %}
    +
    + {{ render_field(form.organization) }} +
    diff --git a/flowapp/templates/layouts/default.html b/flowapp/templates/layouts/default.html index 2d3f732c..dd0dd4c9 100644 --- a/flowapp/templates/layouts/default.html +++ b/flowapp/templates/layouts/default.html @@ -13,10 +13,14 @@ {% block title %}{% endblock %} - + + + + @@ -59,7 +63,7 @@
{{ session['user_name']}} <{{ session['user_email'] }}>, - role: {{ session['user_roles']|join(", ") }}, org: {{ session['user_orgs'] }} + role: {{ session['user_roles']|join(", ") }}, org: {{ session['user_org'] }} diff --git a/flowapp/templates/pages/org_modal.html b/flowapp/templates/pages/org_modal.html new file mode 100644 index 00000000..ea16ce13 --- /dev/null +++ b/flowapp/templates/pages/org_modal.html @@ -0,0 +1,23 @@ +{% extends 'layouts/default.html' %} +{% block title %}ExaFS - choose our organization{% endblock %} +{% block content %} + + + {% endblock %} \ No newline at end of file diff --git a/flowapp/validators.py b/flowapp/validators.py index f4569f86..63061062 100644 --- a/flowapp/validators.py +++ b/flowapp/validators.py @@ -30,9 +30,9 @@ def split_rules_for_user(net_ranges, rules): user_rules = [] rest_rules = [] for rule in rules: - if network_in_range( - rule.source, rule.source_mask, net_ranges - ) or network_in_range(rule.dest, rule.dest_mask, net_ranges): + if network_in_range(rule.source, rule.source_mask, net_ranges) or network_in_range( + rule.dest, rule.dest_mask, net_ranges + ): user_rules.append(rule) else: rest_rules.append(rule) @@ -85,9 +85,7 @@ def address_in_range(address, net_ranges): result = False for adr_range in net_ranges: try: - result = result or ipaddress.ip_address(address) in ipaddress.ip_network( - adr_range - ) + result = result or ipaddress.ip_address(address) in ipaddress.ip_network(adr_range) except ValueError: return False @@ -105,9 +103,7 @@ def network_in_range(address, mask, net_ranges): network = "{}/{}".format(address, mask) for adr_range in net_ranges: try: - result = result or subnet_of( - ipaddress.ip_network(network), ipaddress.ip_network(adr_range) - ) + result = result or subnet_of(ipaddress.ip_network(network), ipaddress.ip_network(adr_range)) except TypeError: # V4 can't be a subnet of V6 and vice versa pass except ValueError: @@ -127,9 +123,7 @@ def range_in_network(address, mask, net_ranges): network = "{}/{}".format(address, mask) for adr_range in net_ranges: try: - result = result or supernet_of( - ipaddress.ip_network(network), ipaddress.ip_network(adr_range) - ) + result = result or supernet_of(ipaddress.ip_network(network), ipaddress.ip_network(adr_range)) except ValueError: return False @@ -147,9 +141,7 @@ def whole_world_range(net_ranges, address="0.0.0.0"): try: for adr_range in net_ranges: - result = result or ipaddress.ip_address(address) in ipaddress.ip_network( - adr_range - ) + result = result or ipaddress.ip_address(address) in ipaddress.ip_network(adr_range) except ValueError: return False @@ -203,11 +195,7 @@ def __init__(self, message=None, max_values=constants.MAX_COMMA_VALUES): def __call__(self, form, field): field_data = field.data.split(";") if len(field_data) > self.max_values: - raise ValidationError( - "{} maximum {} comma separated values".format( - self.message, self.max_values - ) - ) + raise ValidationError("{} maximum {} comma separated values".format(self.message, self.max_values)) try: for port_string in field_data: flowspec.to_exabgp_string(port_string, constants.MAX_PORT) @@ -265,9 +253,7 @@ def __call__(self, form, field): result = False for address in field.data.split("/"): for adr_range in self.net_ranges: - result = result or ipaddress.ip_address( - address - ) in ipaddress.ip_network(adr_range) + result = result or ipaddress.ip_address(address) in ipaddress.ip_network(adr_range) if not result: raise ValidationError(self.message) @@ -285,7 +271,7 @@ def __init__(self, message=None): def __call__(self, form, field): try: - address = ipaddress.ip_address(field.data) + ipaddress.ip_address(field.data) except ValueError: raise ValidationError(self.message + str(field.data)) @@ -323,6 +309,7 @@ def __call__(self, form, field): except ValueError: raise ValidationError(self.message + str(field.data)) + def editable_range(rule, net_ranges): """ check if the rule is editable for user @@ -352,14 +339,9 @@ def _is_subnet_of(a, b): # Always false if one is v4 and the other is v6. if a._version != b._version: raise TypeError("%s and %s are not of the same version" % (a, b)) - return ( - b.network_address <= a.network_address - and b.broadcast_address >= a.broadcast_address - ) + return b.network_address <= a.network_address and b.broadcast_address >= a.broadcast_address except AttributeError: - raise TypeError( - "Unable to test subnet containment " "between %s and %s" % (a, b) - ) + raise TypeError("Unable to test subnet containment " "between %s and %s" % (a, b)) def subnet_of(net_a, net_b): diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index f819c5da..3ae96fb5 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -5,7 +5,7 @@ from functools import wraps from datetime import datetime, timedelta -from flowapp.constants import WITHDRAW, ANNOUNCE, TIME_FORMAT_ARG +from flowapp.constants import WITHDRAW, ANNOUNCE, TIME_FORMAT_ARG, RuleTypes from flowapp.models import ( RTBH, Flowspec4, @@ -28,7 +28,7 @@ output_date_format, ) from flowapp.auth import check_access_rights -from flowapp.output import RuleTypes, announce_route, log_route, log_withdraw, Route, RouteSources +from flowapp.output import announce_route, log_route, log_withdraw, Route, RouteSources from flowapp import db, validators, flowspec, messages diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 3a3162f6..196392dd 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -12,6 +12,7 @@ localhost_only, user_or_admin_required, ) +from flowapp.constants import RuleTypes from flowapp.forms import IPv4Form, IPv6Form, RTBHForm from flowapp.models import ( RTBH, @@ -19,15 +20,19 @@ Community, Flowspec4, Flowspec6, + User, get_ipv4_model_if_exists, get_ipv6_model_if_exists, get_rtbh_model_if_exists, get_user_actions, get_user_communities, get_user_nets, + get_user_orgs_choices, + get_user_orgs_limits, + increment_rule_count, insert_initial_communities, ) -from flowapp.output import ROUTE_MODELS, RuleTypes, announce_route, log_route, log_withdraw, RouteSources, Route +from flowapp.output import ROUTE_MODELS, announce_route, log_route, log_withdraw, RouteSources, Route from flowapp.utils import ( flash_errors, get_state_by_time, @@ -433,17 +438,27 @@ def ipv4_rule(): user_actions = [ (0, "---- select action ----"), ] + user_actions + user_orgs = get_user_orgs_choices(session["user_id"]) + form.action.choices = user_actions form.action.default = 0 form.net_ranges = net_ranges + form.organization.choices = user_orgs + current_app.logger.debug("User orgs: %s", user_orgs) + user = db.session.query(User).get(session["user_id"]) + for org in user.organization: + count = org.count_rules(RuleTypes.IPv4) + current_app.logger.debug(f"Org: {org.name}, Count: {count}") if request.method == "POST" and form.validate(): model = get_ipv4_model_if_exists(form.data, 1) if model: + increment = False model.expires = round_to_ten_minutes(form.expires.data) flash_message = "Existing IPv4 Rule found. Expiration time was updated to new value." else: + increment = True model = Flowspec4( source=form.source.data, source_mask=form.source_mask.data, @@ -467,6 +482,10 @@ def ipv4_rule(): db.session.commit() flash(flash_message, "alert-success") + # increment counter if new rule was added + if increment: + increment_rule_count(RuleTypes.IPv4, form.organization.data) + # announce route if model is in active state if model.rstate_id == 1: command = messages.create_ipv4(model, constants.ANNOUNCE) diff --git a/migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py b/migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py new file mode 100644 index 00000000..f6b6e107 --- /dev/null +++ b/migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py @@ -0,0 +1,36 @@ +"""add rule counters to organization + +Revision ID: 58ac38ced7f6 +Revises: 7719224c9f1f +Create Date: 2024-10-15 16:53:48.222167 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '58ac38ced7f6' +down_revision = '7719224c9f1f' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.add_column(sa.Column('ipv4count', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('ipv6count', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('rtbhcount', sa.Integer(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.drop_column('rtbhcount') + batch_op.drop_column('ipv6count') + batch_op.drop_column('ipv4count') + + # ### end Alembic commands ### From 99510fa4483263890929372a2aaf9887a8985268 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 17 Oct 2024 19:07:06 +0200 Subject: [PATCH 27/54] WIP - limit for each type in org form and table --- flowapp/__init__.py | 4 + flowapp/forms.py | 28 ++++--- flowapp/models.py | 41 ++++------ flowapp/templates/forms/ipv4_rule.html | 1 - flowapp/templates/forms/ipv6_rule.html | 6 +- flowapp/templates/forms/org.html | 35 +++++++++ flowapp/templates/pages/orgs.html | 14 ++-- flowapp/views/admin.py | 12 ++- flowapp/views/rules.py | 4 +- ...imit_for_each_type_of_rule_org_id_as_a_.py | 74 +++++++++++++++++++ 10 files changed, 171 insertions(+), 48 deletions(-) create mode 100644 flowapp/templates/forms/org.html create mode 100644 migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py diff --git a/flowapp/__init__.py b/flowapp/__init__.py index 3ef31611..606ad695 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -220,6 +220,10 @@ def format_datetime(value): format = "y/MM/dd HH:mm" return babel.dates.format_datetime(value, format) + @app.template_filter("unlimited") + def unlimited_filter(value): + return "unlimited" if value == 0 else value + def _handle_login(uuid: str): """ handles rest of login process diff --git a/flowapp/forms.py b/flowapp/forms.py index 14b817fb..8a1396b2 100644 --- a/flowapp/forms.py +++ b/flowapp/forms.py @@ -105,7 +105,7 @@ class UserForm(FlaskForm): org_ids = SelectMultipleField( "Organization", coerce=int, - validators=[DataRequired("Select at last one Organization")], + validators=[DataRequired("We prefer one Organization per user, but it's possible select more")], ) @@ -168,8 +168,24 @@ class OrganizationForm(FlaskForm): name = StringField("Organization name", validators=[Optional(), Length(max=150)]) - rule_limit = IntegerField( - "Maximum number of rules, 0 for unlimited", + limit_flowspec4 = IntegerField( + "Maximum number of IPv4 rules, 0 for unlimited", + validators=[ + Optional(), + NumberRange(min=0, max=1000, message="invalid mask value (0-1000)"), + ], + ) + + limit_flowspec6 = IntegerField( + "Maximum number of IPv6 rules, 0 for unlimited", + validators=[ + Optional(), + NumberRange(min=0, max=1000, message="invalid mask value (0-1000)"), + ], + ) + + limit_rtbh = IntegerField( + "Maximum number of RTBH rules, 0 for unlimited", validators=[ Optional(), NumberRange(min=0, max=1000, message="invalid mask value (0-1000)"), @@ -386,12 +402,6 @@ def __init__(self, *args, **kwargs): validators=[DataRequired(message="Please select an action for the rule.")], ) - organization = SelectField( - "Organization", - coerce=int, - validators=[DataRequired(message="Please select one of your organizations.")], - ) - expires = MultiFormatDateTimeLocalField("Expires", format="%Y-%m-%dT%H:%M", validators=[InputRequired()]) comment = arange = TextAreaField("Comments") diff --git a/flowapp/models.py b/flowapp/models.py index e001bc87..3f54ad0a 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -127,15 +127,16 @@ class Organization(db.Model): id = db.Column(db.Integer, primary_key=True) name = db.Column(db.String(150), unique=True) arange = db.Column(db.Text) - rule_limit = db.Column(db.Integer, default=0) - ipv4count = db.Column(db.Integer, default=0) - ipv6count = db.Column(db.Integer, default=0) - rtbhcount = db.Column(db.Integer, default=0) + limit_flowspec4 = db.Column(db.Integer, default=0) + limit_flowspec6 = db.Column(db.Integer, default=0) + limit_rtbh = db.Column(db.Integer, default=0) - def __init__(self, name, arange, rule_limit=0): + def __init__(self, name, arange, limit_flowspec4=0, limit_flowspec6=0, limit_rtbh=0): self.name = name self.arange = arange - self.rule_limit = rule_limit + self.limit_flowspec4 = limit_flowspec4 + self.limit_flowspec6 = limit_flowspec6 + self.limit_rtbh = limit_rtbh def __repr__(self): return self.name @@ -147,24 +148,6 @@ def get_users(self): # self.user is the backref from the user_organization relationship return self.user - def count_rules(self, rule_type): - """ - Count the number of rules of a certain type for this organization - :param rule_type: RuleTypes - :return: integer - """ - if rule_type == RuleTypes.IPv4: - ctr = 0 - for user in self.user: - for rule in user.flowspec4: - if rule.rstate_id == 1: - ctr += 1 - return ctr - if rule_type == RuleTypes.IPv6: - return self.ipv6count - if rule_type == RuleTypes.RTBH: - return self.rtbhcount - class ASPath(db.Model): id = db.Column(db.Integer, primary_key=True) @@ -250,6 +233,8 @@ class RTBH(db.Model): created = db.Column(db.DateTime) user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User", backref="rtbh") + org_id = db.Column(db.Integer, db.ForeignKey("organization.id"), nullable=False) + org = db.relationship("Organization", backref="rtbh") rstate_id = db.Column(db.Integer, db.ForeignKey("rstate.id"), nullable=False) rstate = db.relationship("Rstate", backref="RTBH") @@ -380,6 +365,8 @@ class Flowspec4(db.Model): action = db.relationship("Action", backref="flowspec4") user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User", backref="flowspec4") + org_id = db.Column(db.Integer, db.ForeignKey("organization.id"), nullable=False) + org = db.relationship("Organization", backref="flowspec4") rstate_id = db.Column(db.Integer, db.ForeignKey("rstate.id"), nullable=False) rstate = db.relationship("Rstate", backref="flowspec4") @@ -534,6 +521,8 @@ class Flowspec6(db.Model): action = db.relationship("Action", backref="flowspec6") user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User", backref="flowspec6") + org_id = db.Column(db.Integer, db.ForeignKey("organization.id"), nullable=False) + org = db.relationship("Organization", backref="flowspec6") rstate_id = db.Column(db.Integer, db.ForeignKey("rstate.id"), nullable=False) rstate = db.relationship("Rstate", backref="flowspec6") @@ -650,14 +639,16 @@ class Log(db.Model): rule_type = db.Column(db.Integer) rule_id = db.Column(db.Integer) user_id = db.Column(db.Integer) + org_id = db.Column(db.Integer, nullable=True) - def __init__(self, time, task, user_id, rule_type, rule_id, author): + def __init__(self, time, task, user_id, rule_type, rule_id, author, org_id=None): self.time = time self.task = task self.rule_type = rule_type self.rule_id = rule_id self.user_id = user_id self.author = author + self.org_id = org_id # DDL diff --git a/flowapp/templates/forms/ipv4_rule.html b/flowapp/templates/forms/ipv4_rule.html index 700048e9..6cc9021f 100644 --- a/flowapp/templates/forms/ipv4_rule.html +++ b/flowapp/templates/forms/ipv4_rule.html @@ -77,7 +77,6 @@

{{ title or 'New'}} IPv4 rule

- {{ render_field(form.organization) }}
diff --git a/flowapp/templates/forms/ipv6_rule.html b/flowapp/templates/forms/ipv6_rule.html index 8929c99e..62198b35 100644 --- a/flowapp/templates/forms/ipv6_rule.html +++ b/flowapp/templates/forms/ipv6_rule.html @@ -46,10 +46,10 @@

{{ title or 'New'}} IPv6 rule

-
+
{{ render_field(form.action) }}
-
+
@@ -69,6 +69,8 @@

{{ title or 'New'}} IPv6 rule

{% endif %}
+
+
diff --git a/flowapp/templates/forms/org.html b/flowapp/templates/forms/org.html new file mode 100644 index 00000000..ef059334 --- /dev/null +++ b/flowapp/templates/forms/org.html @@ -0,0 +1,35 @@ +{% extends 'layouts/default.html' %} +{% from 'forms/macros.html' import render_field %} +{% block title %}Add / Edit Organization{% endblock %} +{% block content %} +

{{ title or 'New'}} Organization

+
+ {{ form.hidden_tag() if form.hidden_tag }} +
+
+ {{ render_field(form.name) }} +
+
+
+
+ {{ render_field(form.limit_flowspec4) }} +
+
+ {{ render_field(form.limit_flowspec6) }} +
+
+ {{ render_field(form.limit_rtbh) }} +
+
+
+
+ {{ render_field(form.arange) }} +
+
+
+
+ +
+
+
+{% endblock %} \ No newline at end of file diff --git a/flowapp/templates/pages/orgs.html b/flowapp/templates/pages/orgs.html index 120b5c43..3b251f5d 100644 --- a/flowapp/templates/pages/orgs.html +++ b/flowapp/templates/pages/orgs.html @@ -1,5 +1,5 @@ {% extends 'layouts/default.html' %} -{% block title %}Flowspec Organziations{% endblock %} +{% block title %}Flowspec Organizations{% endblock %} {% block content %} @@ -10,9 +10,13 @@ {% for org in orgs %} - - - + + - + @@ -18,6 +19,8 @@

Your machines and ApiKeys

+ diff --git a/flowapp/templates/pages/limit_reached.html b/flowapp/templates/pages/limit_reached.html new file mode 100644 index 00000000..4129912c --- /dev/null +++ b/flowapp/templates/pages/limit_reached.html @@ -0,0 +1,25 @@ +{% extends 'layouts/default.html' %} +{% block title %}Rule limit reached{% endblock %} +{% block content %} +
+
+

You can't add new / reactivate {{ rule_type }} rule.

+
Rule limit of you organization has been reached.
+
{{ org.name }}{{ "unlimited" if org.rule_limit == 0 else org.rule_limit }} + {{ org.name }} + IPv4: {{ org.limit_flowspec4 | unlimited }}
+ IPv6: {{ org.limit_flowspec6 | unlimited }}
+ RTBH: {{ org.limit_rtbh | unlimited }} + 		{% set rows = org.arange.split() %}
    {% for row in rows %} @@ -20,7 +24,7 @@ {% endfor %}
+ diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index 6880d204..68417d0b 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -213,7 +213,13 @@ def organization(): # test if user is unique exist = db.session.query(Organization).filter_by(name=form.name.data).first() if not exist: - org = Organization(name=form.name.data, arange=form.arange.data, rule_limit=form.rule_limit.data) + org = Organization( + name=form.name.data, + arange=form.arange.data, + limit_flowspec4=form.limit_flowspec4.data, + limit_flowspec6=form.limit_flowspec6.data, + limit_rtbh=form.limit_rtbh.data, + ) db.session.add(org) db.session.commit() flash("Organization saved") @@ -223,7 +229,7 @@ def organization(): action_url = url_for("admin.organization") return render_template( - "forms/simple_form.html", + "forms/org.html", title="Add new organization to Flowspec", form=form, action_url=action_url, @@ -245,7 +251,7 @@ def edit_organization(org_id): action_url = url_for("admin.edit_organization", org_id=org.id) return render_template( - "forms/simple_form.html", + "forms/org.html", title="Editing {}".format(org.name), form=form, action_url=action_url, diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 196392dd..249b6620 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -438,13 +438,11 @@ def ipv4_rule(): user_actions = [ (0, "---- select action ----"), ] + user_actions - user_orgs = get_user_orgs_choices(session["user_id"]) form.action.choices = user_actions form.action.default = 0 form.net_ranges = net_ranges - form.organization.choices = user_orgs - current_app.logger.debug("User orgs: %s", user_orgs) + user = db.session.query(User).get(session["user_id"]) for org in user.organization: count = org.count_rules(RuleTypes.IPv4) diff --git a/migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py b/migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py new file mode 100644 index 00000000..6e463aef --- /dev/null +++ b/migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py @@ -0,0 +1,74 @@ +"""limit for each type of rule, org id as a key for each rule + +Revision ID: e25b385bae2c +Revises: 58ac38ced7f6 +Create Date: 2024-10-17 16:41:01.116727 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'e25b385bae2c' +down_revision = '58ac38ced7f6' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('RTBH', schema=None) as batch_op: + batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) + batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) + + with op.batch_alter_table('flowspec4', schema=None) as batch_op: + batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) + batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) + + with op.batch_alter_table('flowspec6', schema=None) as batch_op: + batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) + batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) + + with op.batch_alter_table('log', schema=None) as batch_op: + batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=True)) + + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.add_column(sa.Column('limit_flowspec4', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('limit_flowspec6', sa.Integer(), nullable=True)) + batch_op.add_column(sa.Column('limit_rtbh', sa.Integer(), nullable=True)) + batch_op.drop_column('ipv6count') + batch_op.drop_column('rule_limit') + batch_op.drop_column('rtbhcount') + batch_op.drop_column('ipv4count') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.add_column(sa.Column('ipv4count', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) + batch_op.add_column(sa.Column('rtbhcount', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) + batch_op.add_column(sa.Column('rule_limit', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) + batch_op.add_column(sa.Column('ipv6count', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) + batch_op.drop_column('limit_rtbh') + batch_op.drop_column('limit_flowspec6') + batch_op.drop_column('limit_flowspec4') + + with op.batch_alter_table('log', schema=None) as batch_op: + batch_op.drop_column('org_id') + + with op.batch_alter_table('flowspec6', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.drop_column('org_id') + + with op.batch_alter_table('flowspec4', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.drop_column('org_id') + + with op.batch_alter_table('RTBH', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.drop_column('org_id') + + # ### end Alembic commands ### From 72758077a88ef16e884c9832d12bf937354ea210 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 21 Oct 2024 15:12:31 +0200 Subject: [PATCH 28/54] finished limit counts in UI and API, ready for testing --- flowapp/__init__.py | 5 + flowapp/auth.py | 9 +- flowapp/constants.py | 1 + flowapp/models.py | 60 ++++---- flowapp/templates/pages/api_key.html | 3 + flowapp/templates/pages/limit_reached.html | 25 +++ flowapp/templates/pages/orgs.html | 37 +++-- flowapp/templates/pages/user_list.html | 19 +++ flowapp/tests/conftest.py | 36 +++-- flowapp/tests/test_api_v3.py | 145 ++++++++++++++++-- flowapp/tests/test_flowapp.py | 20 ++- flowapp/tests/test_login.py | 0 flowapp/tests/test_models.py | 21 ++- flowapp/views/admin.py | 86 ++++++++++- flowapp/views/api_common.py | 29 +++- flowapp/views/api_keys.py | 1 + flowapp/views/dashboard.py | 2 + flowapp/views/rules.py | 122 +++++++++------ ...e99a66ed9f8_organization_in_apikey_and_.py | 42 +++++ 19 files changed, 524 insertions(+), 139 deletions(-) create mode 100644 flowapp/templates/pages/limit_reached.html create mode 100644 flowapp/templates/pages/user_list.html create mode 100644 flowapp/tests/test_login.py create mode 100644 migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py diff --git a/flowapp/__init__.py b/flowapp/__init__.py index 606ad695..411fe71c 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -160,9 +160,14 @@ def index(): @app.route("/select_org", defaults={"org_id": None}) @app.route("/select_org/") + @auth_required def select_org(org_id=None): uuid = session.get("user_uuid") user = db.session.query(models.User).filter_by(uuid=uuid).first() + + if user is None: + return render_template("errors/404.html"), 404 # Handle missing user gracefully + orgs = user.organization if org_id: org = db.session.query(models.Organization).filter_by(id=org_id).first() diff --git a/flowapp/auth.py b/flowapp/auth.py index 55b1e9fd..ee2f8851 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -13,7 +13,7 @@ def auth_required(f): @wraps(f) def decorated(*args, **kwargs): - user = session.get("user_uuid", False) + user = get_user() if not user: if current_app.config.get("SSO_AUTH"): current_app.logger.warning("SSO AUTH SET BUT FAILS") @@ -155,3 +155,10 @@ def is_admin(current_user_roles): return True return False + + +def get_user(): + """ + get user from session or return None + """ + return session.get("user_uuid", None) diff --git a/flowapp/constants.py b/flowapp/constants.py index 3c22be30..5aa52834 100644 --- a/flowapp/constants.py +++ b/flowapp/constants.py @@ -24,6 +24,7 @@ RULES_KEY = "rules" RULE_TYPES_DICT = {"ipv4": 4, "ipv6": 6, "rtbh": 1} +RULE_NAMES_DICT = {4: "ipv4", 6: "ipv6", 1: "rtbh"} DEFAULT_COUNT_MATCH = {"ipv4": 0, "ipv6": 0, "rtbh": 0} ANNOUNCE = 1 diff --git a/flowapp/models.py b/flowapp/models.py index 3f54ad0a..5ec94700 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -3,6 +3,7 @@ from datetime import datetime from flowapp import db, utils from flowapp.constants import RuleTypes +from flask import current_app # models and tables @@ -85,6 +86,8 @@ class ApiKey(db.Model): comment = db.Column(db.String(255)) user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User", back_populates="apikeys") + org_id = db.Column(db.Integer, db.ForeignKey("organization.id"), nullable=False) + org = db.relationship("Organization", backref="apikey") def is_expired(self): if self.expires is None: @@ -102,6 +105,8 @@ class MachineApiKey(db.Model): comment = db.Column(db.String(255)) user_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False) user = db.relationship("User", back_populates="machineapikeys") + org_id = db.Column(db.Integer, db.ForeignKey("organization.id"), nullable=False) + org = db.relationship("Organization", backref="machineapikey") def is_expired(self): if self.expires is None: @@ -247,6 +252,7 @@ def __init__( community_id, expires, user_id, + org_id, comment=None, created=None, rstate_id=1, @@ -258,6 +264,7 @@ def __init__( self.community_id = community_id self.expires = expires self.user_id = user_id + self.org_id = org_id self.comment = comment if created is None: created = datetime.now() @@ -384,6 +391,7 @@ def __init__( fragment, expires, user_id, + org_id, action_id, created=None, comment=None, @@ -402,6 +410,7 @@ def __init__( self.comment = comment self.expires = expires self.user_id = user_id + self.org_id = org_id self.action_id = action_id if created is None: created = datetime.now() @@ -539,6 +548,7 @@ def __init__( packet_len, expires, user_id, + org_id, action_id, created=None, comment=None, @@ -556,6 +566,7 @@ def __init__( self.comment = comment self.expires = expires self.user_id = user_id + self.org_id = org_id self.action_id = action_id if created is None: created = datetime.now() @@ -727,7 +738,6 @@ def insert_initial_roles(table, conn, *args, **kwargs): @event.listens_for(Organization.__table__, "after_create") def insert_initial_organizations(table, conn, *args, **kwargs): - conn.execute(table.insert().values(name="TU Liberec", arange="147.230.0.0/16\n2001:718:1c01::/48")) conn.execute(table.insert().values(name="Cesnet", arange="147.230.0.0/16\n2001:718:1c01::/48")) @@ -746,30 +756,30 @@ def check_rule_limit(org_id: int, rule_type: RuleTypes): :param rule_type: RuleType rule type :return: boolean """ - org = Organization.query.filter_by(id=org_id).first() - if rule_type == RuleTypes.IPv4: - return org.ipv4count >= org.rule_limit - if rule_type == RuleTypes.IPv6: - return org.ipv6count >= org.rule_limit - if rule_type == RuleTypes.RTBH: - return org.rtbhcount >= org.rule_limit - + flowspec_limit = current_app.config.get("FLOWSPEC_MAX_RULES", 9000) + rtbh_limit = current_app.config.get("RTBH_MAX_RULES", 100000) + fs4 = db.session.query(Flowspec4).filter_by(rstate_id=1).count() + fs6 = db.session.query(Flowspec6).filter_by(rstate_id=1).count() + rtbh = db.session.query(RTBH).filter_by(rstate_id=1).count() -def increment_rule_count(rule_type: RuleTypes, org_id: int): - """ - Increment rule count for organization - :param rule_type: RuleType rule type - :param org_id: integer organization id - :return: None - """ + # check the organization limits org = Organization.query.filter_by(id=org_id).first() + if rule_type == RuleTypes.IPv4 and org.limit_flowspec4 > 0: + count = db.session.query(Flowspec4).filter_by(org_id=org_id, rstate_id=1).count() + return count >= org.limit_flowspec4 or fs4 >= flowspec_limit + if rule_type == RuleTypes.IPv6 and org.limit_flowspec6 > 0: + count = db.session.query(Flowspec6).filter_by(org_id=org_id, rstate_id=1).count() + return count >= org.limit_flowspec6 or fs6 >= flowspec_limit + if rule_type == RuleTypes.RTBH and org.limit_rtbh > 0: + count = db.session.query(RTBH).filter_by(org_id=org_id, rstate_id=1).count() + return count >= org.limit_rtbh or rtbh >= rtbh_limit + # check the global limits if the organization limits are not set if rule_type == RuleTypes.IPv4: - org.ipv4count += 1 + return fs4 >= flowspec_limit if rule_type == RuleTypes.IPv6: - org.ipv6count += 1 + return fs6 >= flowspec_limit if rule_type == RuleTypes.RTBH: - org.rtbhcount += 1 - db.session.commit() + return rtbh >= rtbh_limit def get_ipv4_model_if_exists(form_data, rstate_id=1): @@ -916,16 +926,6 @@ def get_user_orgs_choices(user_id): return [(g.id, g.name) for g in orgs] -def get_user_orgs_limits(user_id): - """ - Return list of orgs as choices for form - """ - user = db.session.query(User).filter_by(id=user_id).first() - orgs = user.organization - - return [(g.id, g.name) for g in orgs] - - def get_user_actions(user_roles): """ Return list of actions based on current user role diff --git a/flowapp/templates/pages/api_key.html b/flowapp/templates/pages/api_key.html index cc645887..d4b753c8 100644 --- a/flowapp/templates/pages/api_key.html +++ b/flowapp/templates/pages/api_key.html @@ -6,6 +6,7 @@

Your machines and ApiKeys

Machine address ApiKeyOrganization Expires Read only Action {{ row.key }} + {{ row.org.name }} {{ row.expires|strftime }}
+ + + + + + + + + + + + +
Rule typeCurrent countYour org Limit
IPv4 (Flowspec4){{ count_4 }}{{ org.limit_flowspec4|unlimited }}
IPv6 {Flowspec6){{ count_6 }}{{ org.limit_flowspec6|unlimited }}
RTBH{{ count_rtbh }}{{ org.limit_rtbh|unlimited }}
+

Please delete some unnecesary rules, or contact system Administrator.

+
+
+{% endblock %} \ No newline at end of file diff --git a/flowapp/templates/pages/orgs.html b/flowapp/templates/pages/orgs.html index 3b251f5d..3d2e3df0 100644 --- a/flowapp/templates/pages/orgs.html +++ b/flowapp/templates/pages/orgs.html @@ -1,6 +1,20 @@ {% extends 'layouts/default.html' %} {% block title %}Flowspec Organizations{% endblock %} {% block content %} + + + + + + + + + + + + + +
RTBH All CountFlowspec4 All CountFlowspec6 All Count
{{ rtbh_all_count }} / {{ rtbh_limit }} {{ flowspec4_all_count }} / {{ flowspec_limit }} {{ flowspec6_all_count }} / {{ flowspec_limit }}
@@ -9,12 +23,12 @@ {% for org in orgs %} - + - + {% endfor %} +
Nameaction
{{ org.name }} - IPv4: {{ org.limit_flowspec4 | unlimited }}
- IPv6: {{ org.limit_flowspec6 | unlimited }}
- RTBH: {{ org.limit_rtbh | unlimited }} + IPv4: {{ org.limit_flowspec4 | unlimited }} / {{ flowspec4_counts[org.id] | default(0) }}
+ IPv6: {{ org.limit_flowspec6 | unlimited }} / {{ flowspec6_counts[org.id] | default(0) }}
+ RTBH: {{ org.limit_rtbh | unlimited }} / {{ rtbh_counts[org.id] | default(0) }} 		{% set rows = org.arange.split() %} @@ -25,14 +39,15 @@ - - - - - - + + + + + +
{% endblock %} \ No newline at end of file diff --git a/flowapp/templates/pages/user_list.html b/flowapp/templates/pages/user_list.html new file mode 100644 index 00000000..ba08dc3d --- /dev/null +++ b/flowapp/templates/pages/user_list.html @@ -0,0 +1,19 @@ +{% extends 'layouts/default.html' %} +{% block title %}Flowspec Actions{% endblock %} +{% block content %} +

Updated {{ updated}} rules.

+

Users with multiple organizations

+

Records of users with multilple orgs could not be updated.

+ {% for user, orgs in users.items() %} +

+ {{ user }} +

    + {% for org in orgs %} +
  • {{ org }}
    • + {% endfor %} +
+

+ {% endfor %} + + +{% endblock %} \ No newline at end of file diff --git a/flowapp/tests/conftest.py b/flowapp/tests/conftest.py index 1d5436db..65919af1 100644 --- a/flowapp/tests/conftest.py +++ b/flowapp/tests/conftest.py @@ -1,6 +1,7 @@ """ PyTest configuration file for all tests """ + import os import json import pytest @@ -12,6 +13,7 @@ from datetime import datetime import flowapp.models + TESTDB = "test_project.db" TESTDB_PATH = "/tmp/{}".format(TESTDB) TEST_DATABASE_URI = "sqlite:///" + TESTDB_PATH @@ -63,20 +65,11 @@ def app(request): JWT_SECRET="testing", API_KEY="testkey", SECRET_KEY="testkeysession", - LOCAL_USER_UUID="jiri.vrany@tul.cz", - LOCAL_USER_ID=1, - LOCAL_USER_ROLES=["admin"], - LOCAL_USER_ORGS=[ - {"name": "TU Liberec", "arange": "147.230.0.0/16\n2001:718:1c01::/48"} - ], - # Defined in Role model / default 1 - view, 2 - normal user, 3 - admin - LOCAL_USER_ROLE_IDS=[3], - # Defined in Organization model - LOCAL_USER_ORG_IDS=[1], + LOCAL_USER_UUID="jiri.vrany@cesnet.cz", + LOCAL_AUTH=True, ) print("\n----- CREATE FLASK APPLICATION\n") - context = _app.app_context() context.push() yield _app @@ -115,9 +108,8 @@ def db(app, request): _db.create_all() users = [ - {"name": "jiri.vrany@tul.cz", "role_id": 3, "org_id": 1}, - {"name": "petr.adamec@tul.cz", "role_id": 3, "org_id": 1}, - {"name": "adamec@cesnet.cz", "role_id": 3, "org_id": 2}, + {"name": "jiri.vrany@cesnet.cz", "role_id": 3, "org_id": 1}, + {"name": "petr.adamec@cesnet.cz", "role_id": 3, "org_id": 1}, ] print("#: inserting users") flowapp.models.insert_users(users) @@ -139,7 +131,7 @@ def jwt_token(client, app, db, request): mkey = "testkey" with app.app_context(): - model = flowapp.models.ApiKey(machine="127.0.0.1", key=mkey, user_id=1) + model = flowapp.models.ApiKey(machine="127.0.0.1", key=mkey, user_id=1, org_id=1) db.session.add(model) db.session.commit() @@ -159,7 +151,7 @@ def expired_auth_token(client, app, db, request): test_key = "expired_test_key" expired_date = datetime.strptime("2019-01-01", "%Y-%m-%d") with app.app_context(): - model = flowapp.models.ApiKey(machine="127.0.0.1", key=test_key, user_id=1, expires=expired_date) + model = flowapp.models.ApiKey(machine="127.0.0.1", key=test_key, user_id=1, expires=expired_date, org_id=1) db.session.add(model) db.session.commit() @@ -173,7 +165,7 @@ def readonly_jwt_token(client, app, db, request): """ readonly_key = "readonly-testkey" with app.app_context(): - model = flowapp.models.ApiKey(machine="127.0.0.1", key=readonly_key, user_id=1, readonly=True) + model = flowapp.models.ApiKey(machine="127.0.0.1", key=readonly_key, user_id=1, readonly=True, org_id=1) db.session.add(model) db.session.commit() @@ -183,3 +175,13 @@ def readonly_jwt_token(client, app, db, request): token = client.get(url, headers=headers) data = json.loads(token.data) return data["token"] + + +@pytest.fixture(scope="session") +def auth_client(client): + """ + Get the test_client from the app, for the whole test session. + """ + print("\n----- CREATE AUTHENTICATED FLASK TEST CLIENT\n") + client.get("/local-login") + return client diff --git a/flowapp/tests/test_api_v3.py b/flowapp/tests/test_api_v3.py index 78b75432..d2b53998 100644 --- a/flowapp/tests/test_api_v3.py +++ b/flowapp/tests/test_api_v3.py @@ -1,5 +1,7 @@ import json +from flowapp.models import Flowspec4, Organization + V_PREFIX = "/api/v3" @@ -65,7 +67,7 @@ def test_create_v4rule(client, db, jwt_token): data = json.loads(req.data) assert data["rule"] assert data["rule"]["id"] == 1 - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" def test_delete_v4rule(client, db, jwt_token): @@ -117,7 +119,7 @@ def test_create_rtbh_rule(client, db, jwt_token): assert req.status_code == 201 assert data["rule"] assert data["rule"]["id"] == 1 - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" def test_delete_rtbh_rule(client, db, jwt_token): @@ -189,7 +191,7 @@ def test_create_v6rule(client, db, jwt_token): assert req.status_code == 201 assert data["rule"] assert data["rule"]["id"] == "1" - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" def test_validation_v4rule(client, db, jwt_token): @@ -302,7 +304,7 @@ def test_update_existing_v4rule_with_timestamp(client, db, jwt_token): data = json.loads(req.data) assert data["rule"] assert data["rule"]["id"] == 2 - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" assert data["rule"]["expires"] == 1444913400 @@ -327,7 +329,7 @@ def test_create_v4rule_with_timestamp(client, db, jwt_token): data = json.loads(req.data) assert data["rule"] assert data["rule"]["id"] == 3 - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" assert data["rule"]["expires"] == 1444913400 @@ -351,7 +353,7 @@ def test_update_existing_v6rule_with_timestamp(client, db, jwt_token): assert req.status_code == 201 assert data["rule"] assert data["rule"]["id"] == "1" - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" assert data["rule"]["expires"] == 1444913400 @@ -368,15 +370,16 @@ def test_create_v6rule_with_timestamp(client, db, jwt_token): "source": "2001:718:1C01:1111::", "source_mask": 64, "source_port": "", - "expires": "1444913400", + "expires": "2549952908", }, ) data = json.loads(req.data) assert req.status_code == 201 assert data["rule"] assert data["rule"]["id"] == "2" - assert data["rule"]["user"] == "jiri.vrany@tul.cz" - assert data["rule"]["expires"] == 1444913400 + assert data["rule"]["rstate"] == "active rule" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" + assert data["rule"]["expires"] == 2549953200 def test_update_existing_rtbh_rule_with_timestamp(client, db, jwt_token): @@ -397,7 +400,7 @@ def test_update_existing_rtbh_rule_with_timestamp(client, db, jwt_token): assert req.status_code == 201 assert data["rule"] assert data["rule"]["id"] == 1 - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" assert data["rule"]["expires"] == 1444913400 @@ -419,5 +422,125 @@ def test_create_rtbh_rule_with_timestamp(client, db, jwt_token): assert req.status_code == 201 assert data["rule"] assert data["rule"]["id"] == 2 - assert data["rule"]["user"] == "jiri.vrany@tul.cz" + assert data["rule"]["user"] == "jiri.vrany@cesnet.cz" assert data["rule"]["expires"] == 1444913400 + + +def test_create_v4rule_lmit(client, db, app, jwt_token): + """ + test that limit checkt for v4 works + """ + with app.app_context(): + org = db.session.query(Organization).filter_by(id=1).first() + org.limit_flowspec4 = 2 + db.session.commit() + + # count + count = db.session.query(Flowspec4).count() + print("COUNT", count) + + sources = ["147.230.42.17", "147.230.42.118"] + codes = [201, 403] + + for source, code in zip(sources, codes): + data = { + "action": 1, + "protocol": "tcp", + "source": source, + "source_mask": 32, + "source_port": "", + "expires": "10/15/2050 14:46", + } + req = client.post( + f"{V_PREFIX}/rules/ipv4", + headers={"x-access-token": jwt_token}, + json=data, + ) + + assert req.status_code == code + + +def test_create_v6rule_lmit(client, db, app, jwt_token): + """ + test that limit check for v6 works + """ + with app.app_context(): + org = db.session.query(Organization).filter_by(id=1).first() + org.limit_flowspec6 = 3 + db.session.commit() + + sources = ["2001:718:1C01:1111::", "2001:718:1C01:1112::"] + codes = [201, 403] + + for source, code in zip(sources, codes): + data = { + "action": 1, + "next_header": "tcp", + "source": source, + "source_mask": 64, + "source_port": "", + "expires": "10/15/2050 14:46", + } + req = client.post( + f"{V_PREFIX}/rules/ipv6", + headers={"x-access-token": jwt_token}, + json=data, + ) + + assert req.status_code == code + + +def test_create_rtbh_lmit(client, db, app, jwt_token): + """ + test that limit check for v6 works + """ + with app.app_context(): + org = db.session.query(Organization).filter_by(id=1).first() + org.limit_rtbh = 2 + db.session.commit() + + sources = ["147.230.17.42", "147.230.17.43"] + codes = [201, 403] + + for source, code in zip(sources, codes): + data = { + "community": 1, + "ipv4": source, + "ipv4_mask": 32, + "expires": "10/25/2050 14:46", + } + req = client.post(f"{V_PREFIX}/rules/rtbh", headers={"x-access-token": jwt_token}, json=data) + + assert req.status_code == code + + +def test_update_existing_v4rule_with_timestamp_limit(client, db, app, jwt_token): + """ + test that update with different data passes + """ + with app.app_context(): + # count + count = db.session.query(Flowspec4).filter_by(org_id=1, rstate_id=1).count() + print("COUNT in update", count) + + org = db.session.query(Organization).filter_by(id=1).first() + org.limit_flowspec4 = count + db.session.commit() + + req = client.post( + f"{V_PREFIX}/rules/ipv4", + headers={"x-access-token": jwt_token}, + json={ + "action": 2, + "protocol": "tcp", + "source": "147.230.17.17", + "source_mask": 32, + "source_port": "", + "expires": "2552634908", + }, + ) + + assert req.status_code == 403 + data = json.loads(req.data) + assert data["message"] + assert data["message"].startswith("rule limit reached for") diff --git a/flowapp/tests/test_flowapp.py b/flowapp/tests/test_flowapp.py index 37cc90fd..f71e4aeb 100644 --- a/flowapp/tests/test_flowapp.py +++ b/flowapp/tests/test_flowapp.py @@ -1,6 +1,14 @@ -def test_create_survey(client, db): - """ - test that creating with valid data returns 201 - """ - req = client.get("/rules/add_ipv4_rule") - assert req.status_code == 200 +def test_dashboard_not_auth(client): + + response = client.get("/dashboard/ipv4/active/?sort=expires&order=desc") + + # Expecting a 302 redirect to login + assert response.status_code == 302 + + +def test_dashboard(auth_client): + + response = auth_client.get("/dashboard/ipv4/active/?sort=expires&order=desc") + + # Check that the request is successful and renders the correct template + assert response.status_code == 200 # Expecting a 200 OK if the user is authenticated diff --git a/flowapp/tests/test_login.py b/flowapp/tests/test_login.py new file mode 100644 index 00000000..e69de29b diff --git a/flowapp/tests/test_models.py b/flowapp/tests/test_models.py index b9357f83..89599453 100644 --- a/flowapp/tests/test_models.py +++ b/flowapp/tests/test_models.py @@ -22,7 +22,8 @@ def test_insert_ipv4(db): fragment="", action_id=1, expires=datetime.now(), - user_id=4, + user_id=1, + org_id=1, rstate_id=1, ) db.session.add(model) @@ -48,7 +49,8 @@ def test_get_ipv4_model_if_exists(db): packet_len="", action_id=1, expires=datetime.now(), - user_id=4, + user_id=1, + org_id=1, rstate_id=1, ) db.session.add(model) @@ -90,7 +92,8 @@ def test_get_ipv6_model_if_exists(db): packet_len="", action_id=1, expires=datetime.now(), - user_id=4, + user_id=1, + org_id=1, rstate_id=1, ) db.session.add(model) @@ -131,7 +134,8 @@ def test_ipv4_eq(db): packet_len="", action_id=1, expires="123", - user_id=4, + user_id=1, + org_id=1, rstate_id=1, ) @@ -149,6 +153,7 @@ def test_ipv4_eq(db): action_id=1, expires="123456", user_id=1, + org_id=1, rstate_id=1, ) @@ -172,7 +177,8 @@ def test_ipv4_ne(db): packet_len="", action_id=1, expires="123", - user_id=4, + user_id=1, + org_id=1, rstate_id=1, ) @@ -190,6 +196,7 @@ def test_ipv4_ne(db): action_id=1, expires="123456", user_id=1, + org_id=1, rstate_id=1, ) @@ -207,7 +214,8 @@ def test_rtbj_eq(db): ipv6_mask="", community_id=1, expires="123", - user_id=4, + user_id=1, + org_id=1, rstate_id=1, ) @@ -219,6 +227,7 @@ def test_rtbj_eq(db): community_id=1, expires="123456", user_id=1, + org_id=1, rstate_id=1, ) diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index 68417d0b..9791f2b6 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta import secrets +from sqlalchemy import func from flask import Blueprint, render_template, redirect, flash, request, session, url_for, current_app from sqlalchemy.exc import IntegrityError @@ -18,6 +19,9 @@ Community, get_existing_community, Log, + Flowspec4, + Flowspec6, + RTBH, ) from ..auth import auth_required, admin_required from flowapp import db @@ -161,7 +165,7 @@ def edit_user(user_id): return render_template( "forms/simple_form.html", - title="Editing {}".format(user.email), + title=f"Editing {user.email}", form=form, action_url=action_url, ) @@ -179,8 +183,8 @@ def delete_user(user_id): alert_type = "alert-success" try: db.session.commit() - except IntegrityError as e: - message = "User {} owns some rules, can not be deleted!".format(username) + except IntegrityError: + message = f"User {username} owns some rules, can not be deleted!" alert_type = "alert-danger" flash(message, alert_type) @@ -199,8 +203,51 @@ def users(): @auth_required @admin_required def organizations(): - orgs = db.session.query(Organization).all() - return render_template("pages/orgs.html", orgs=orgs) + # Query all organizations and eager load RTBH relationships + orgs = db.session.query(Organization).options(db.joinedload(Organization.rtbh)).all() + + # Get RTBH counts with rstate_id=1 for all organizations in one query + rtbh_counts_query = ( + db.session.query(RTBH.org_id, func.count(RTBH.id)).filter(RTBH.rstate_id == 1).group_by(RTBH.org_id).all() + ) + + flowspec4_count_query = ( + db.session.query(Flowspec4.org_id, func.count(Flowspec4.id)) + .filter(Flowspec4.rstate_id == 1) + .group_by(Flowspec4.org_id) + .all() + ) + + flowspec6_count_query = ( + db.session.query(Flowspec6.org_id, func.count(Flowspec6.id)) + .filter(Flowspec6.rstate_id == 1) + .group_by(Flowspec6.org_id) + .all() + ) + + flowspec4_all_count = db.session.query(Flowspec4).filter(Flowspec4.rstate_id == 1).count() + flowspec6_all_count = db.session.query(Flowspec6).filter(Flowspec6.rstate_id == 1).count() + rtbh_all_count = db.session.query(RTBH).filter(RTBH.rstate_id == 1).count() + flowspec_limit = current_app.config.get("FLOWSPEC_MAX_RULES", 9000) + rtbh_limit = current_app.config.get("RTBH_MAX_RULES", 100000) + + # Convert query result to a dictionary {org_id: count} + rtbh_counts = {org_id: count for org_id, count in rtbh_counts_query} + flowspec4_counts = {org_id: count for org_id, count in flowspec4_count_query} + flowspec6_counts = {org_id: count for org_id, count in flowspec6_count_query} + + return render_template( + "pages/orgs.html", + orgs=orgs, + rtbh_counts=rtbh_counts, + flowspec4_counts=flowspec4_counts, + flowspec6_counts=flowspec6_counts, + rtbh_all_count=rtbh_all_count, + flowspec4_all_count=flowspec4_all_count, + flowspec6_all_count=flowspec6_all_count, + flowspec_limit=flowspec_limit, + rtbh_limit=rtbh_limit, + ) @admin.route("/organization", methods=["GET", "POST"]) @@ -516,3 +563,32 @@ def delete_community(community_id): flash(message, alert_type) return redirect(url_for("admin.communities")) + + +@admin.route("/set-org-for-rules", methods=["GET"]) +@auth_required +@admin_required +def update_rules_set_org(): + + # Get all flowspec records where org_id is NULL (if this is needed) + models = [Flowspec4, Flowspec6, RTBH] + user_with_multiple_orgs = {} + for model in models: + rules = model.query.filter(model.org_id == 0).all() + print(f"Found {len(rules)} records with org_id NULL in {model.__name__}") + # Loop through each flowspec record and update org_id based on the user's organization + updated = 0 + for rule in rules: + orgs = rule.user.organization.all() + if len(orgs) == 1: + user_org = orgs[0] + if user_org: + rule.org_id = user_org.id + updated += 1 + else: + print(f"User {rule.user.email} has multiple organizations") + user_with_multiple_orgs[rule.user.email] = [org.name for org in orgs] + # Commit the changes + db.session.commit() + + return render_template("pages/user_list.html", users=user_with_multiple_orgs, updated=updated) diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index 3ae96fb5..92e5f668 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -13,6 +13,7 @@ ApiKey, MachineApiKey, Community, + check_rule_limit, get_user_nets, get_user_actions, get_ipv4_model_if_exists, @@ -86,9 +87,9 @@ def authorize(user_key): "id": model.user.id, "readonly": model.readonly, "roles": [role.name for role in model.user.role.all()], - "org": [org.name for org in model.user.organization.all()], + "org": model.org.name, + "org_id": model.org.id, "role_ids": [role.id for role in model.user.role.all()], - "org_ids": [org.id for org in model.user.organization.all()], }, "exp": datetime.now() + timedelta(minutes=30), } @@ -187,6 +188,10 @@ def all_communities(current_user): return jsonify({"message": "no actions for this user?"}), 404 +def limit_reached(rule_type): + return jsonify({"message": f"rule limit reached for {rule_type}"}), 403 + + def create_ipv4(current_user): """ Api method for new IPv4 rule @@ -194,6 +199,9 @@ def create_ipv4(current_user): :param current_user: data from jwt token :return: json response """ + if check_rule_limit(current_user["org_id"], RuleTypes.IPv4): + return limit_reached(rule_type=RuleTypes.IPv4) + net_ranges = get_user_nets(current_user["id"]) json_request_data = request.get_json() form = IPv4Form(data=json_request_data, meta={"csrf": False}) @@ -228,6 +236,7 @@ def create_ipv4(current_user): comment=quote_to_ent(form.comment.data), action_id=form.action.data, user_id=current_user["id"], + org_id=current_user["org_id"], rstate_id=get_state_by_time(form.expires.data), ) flash_message = "IPv4 Rule saved" @@ -252,9 +261,9 @@ def create_ipv4(current_user): RuleTypes.IPv4, f"{current_user['uuid']} / {current_user['org']}", ) - pref_format = output_date_format(json_request_data, form.expires.pref_format) - return jsonify({"message": flash_message, "rule": model.to_dict(pref_format)}), 201 + response = {"message": flash_message, "rule": model.to_dict(pref_format)} + return jsonify(response), 201 def create_ipv6(current_user): @@ -264,6 +273,9 @@ def create_ipv6(current_user): :param current_user: data from jwt token :return: """ + if check_rule_limit(current_user["org_id"], RuleTypes.IPv6): + return limit_reached(rule_type=RuleTypes.IPv6) + net_ranges = get_user_nets(current_user["id"]) json_request_data = request.get_json() form = IPv6Form(data=json_request_data, meta={"csrf": False}) @@ -295,6 +307,7 @@ def create_ipv6(current_user): comment=quote_to_ent(form.comment.data), action_id=form.action.data, user_id=current_user["id"], + org_id=current_user["org_id"], rstate_id=get_state_by_time(form.expires.data), ) flash_message = "IPv6 Rule saved" @@ -325,6 +338,13 @@ def create_ipv6(current_user): def create_rtbh(current_user): + """ + Create new RTBH rule + """ + # check limit + if check_rule_limit(current_user["org_id"], RuleTypes.RTBH): + return limit_reached(rule_type=RuleTypes.RTBH) + all_com = db.session.query(Community).all() if not all_com: insert_initial_communities() @@ -357,6 +377,7 @@ def create_rtbh(current_user): expires=form.expires.data, comment=quote_to_ent(form.comment.data), user_id=current_user["id"], + org_id=current_user["org_id"], rstate_id=get_state_by_time(form.expires.data), ) db.session.add(model) diff --git a/flowapp/views/api_keys.py b/flowapp/views/api_keys.py index b7b511ab..c88a9e25 100644 --- a/flowapp/views/api_keys.py +++ b/flowapp/views/api_keys.py @@ -63,6 +63,7 @@ def add(): readonly=form.readonly.data, comment=form.comment.data, user_id=session["user_id"], + org_id=session["user_org_id"], ) db.session.add(model) diff --git a/flowapp/views/dashboard.py b/flowapp/views/dashboard.py index dff36d98..8ed262f0 100644 --- a/flowapp/views/dashboard.py +++ b/flowapp/views/dashboard.py @@ -59,8 +59,10 @@ def index(rtype=None, rstate="active"): # params sanitization if rtype not in current_app.config["DASHBOARD"].keys(): + print("DEBUG rtype not in dashboard keys config") return abort(404) if rstate not in COMP_FUNCS.keys(): + print("DEBUG rstate not in dashboard keys config") return abort(404) if sum(session["user_role_ids"]) == 1: rstate = "active" diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 249b6620..38dfdaad 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -20,16 +20,14 @@ Community, Flowspec4, Flowspec6, - User, + Organization, + check_rule_limit, get_ipv4_model_if_exists, get_ipv6_model_if_exists, get_rtbh_model_if_exists, get_user_actions, get_user_communities, get_user_nets, - get_user_orgs_choices, - get_user_orgs_limits, - increment_rule_count, insert_initial_communities, ) from flowapp.output import ROUTE_MODELS, announce_route, log_route, log_withdraw, RouteSources, Route @@ -87,6 +85,11 @@ def reactivate_rule(rule_type, rule_id): # do not need to validate - all is readonly if request.method == "POST": + # check if rule will be reactivated + state = get_state_by_time(form.expires.data) + if state == 1 and check_rule_limit(session["user_org_id"], rule_type=rule_type): + return redirect(url_for("rules.limit_reached", rule_type=rule_type)) + # set new expiration date model.expires = round_to_ten_minutes(form.expires.data) # set again the active state @@ -101,7 +104,7 @@ def reactivate_rule(rule_type, rule_id): # announce route command = route_model(model, constants.ANNOUNCE) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -111,13 +114,13 @@ def reactivate_rule(rule_type, rule_id): session["user_id"], model, rule_type, - f"{session['user_email']} / {session['user_orgs']}", + f"{session['user_email']} / {session['user_org']}", ) else: # withdraw route command = route_model(model, constants.WITHDRAW) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -128,7 +131,7 @@ def reactivate_rule(rule_type, rule_id): route.command, rule_type, model.id, - "{} / {}".format(session["user_email"], session["user_orgs"]), + f"{session['user_email']} / {session['user_org']}", ) return redirect( @@ -180,7 +183,7 @@ def delete_rule(rule_type, rule_id): # withdraw route command = route_model(model, constants.WITHDRAW) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -191,7 +194,7 @@ def delete_rule(rule_type, rule_id): route.command, rule_type, model.id, - "{} / {}".format(session["user_email"], session["user_orgs"]), + f"{session['user_email']} / {session['user_org']}", ) # delete from db @@ -259,7 +262,7 @@ def group_delete(): model = db.session.query(model_name).get(rule_id) command = route_model(model, constants.WITHDRAW) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -270,15 +273,15 @@ def group_delete(): route.command, rule_type_int, model.id, - "{} / {}".format(session["user_email"], session["user_orgs"]), + f"{session['user_email']} / {session['user_org']}", ) db.session.query(model_name).filter(model_name.id.in_(to_delete)).delete(synchronize_session=False) db.session.commit() - flash("Rules {} deleted".format(to_delete), "alert-success") + flash(f"Rules {to_delete} deleted", "alert-success") else: - flash("You can not delete rules {}".format(to_delete), "alert-warning") + flash(f"You can not delete rules {to_delete}", "alert-warning") return redirect( url_for( @@ -376,18 +379,23 @@ def group_update_save(rule_type): route_model = ROUTE_MODELS[rule_type] for rule_id in to_update: + # check if rule will be reactivated + check = check_rule_limit(session["user_org_id"], rule_type=rule_type) + if rstate_id == 1 and check: + return redirect(url_for("rules.limit_reached", rule_type=rule_type)) + # update record model = db.session.query(model_name).get(rule_id) model.expires = expires model.rstate_id = rstate_id - model.comment = "{} {}".format(model.comment, comment) + model.comment = f"{model.comment} {comment}" db.session.commit() if model.rstate_id == 1: # announce route command = route_model(model, constants.ANNOUNCE) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -397,11 +405,16 @@ def group_update_save(rule_type): session["user_id"], model, rule_type, - f"{session['user_email']} / {session['user_orgs']}", + f"{session['user_email']} / {session['user_org']}", ) else: # withdraw route - route = route_model(model, constants.WITHDRAW) + command = route_model(model, constants.WITHDRAW) + route = Route( + author=f"{session['user_email']} / {session['user_org']}", + source=RouteSources.UI, + command=command, + ) announce_route(route) # log changes log_withdraw( @@ -409,10 +422,10 @@ def group_update_save(rule_type): route.command, rule_type, model.id, - "{} / {}".format(session["user_email"], session["user_orgs"]), + f"{session['user_email']} / {session['user_org']}", ) - flash("Rules {} successfully updated".format(to_update), "alert-success") + flash(f"Rules {to_update} successfully updated", "alert-success") return redirect( url_for( @@ -430,6 +443,9 @@ def group_update_save(rule_type): @auth_required @user_or_admin_required def ipv4_rule(): + if check_rule_limit(session["user_org_id"], RuleTypes.IPv4): + return redirect(url_for("rules.limit_reached", rule_type=RuleTypes.IPv4)) + net_ranges = get_user_nets(session["user_id"]) form = IPv4Form(request.form) @@ -443,20 +459,13 @@ def ipv4_rule(): form.action.default = 0 form.net_ranges = net_ranges - user = db.session.query(User).get(session["user_id"]) - for org in user.organization: - count = org.count_rules(RuleTypes.IPv4) - current_app.logger.debug(f"Org: {org.name}, Count: {count}") - if request.method == "POST" and form.validate(): model = get_ipv4_model_if_exists(form.data, 1) if model: - increment = False model.expires = round_to_ten_minutes(form.expires.data) flash_message = "Existing IPv4 Rule found. Expiration time was updated to new value." else: - increment = True model = Flowspec4( source=form.source.data, source_mask=form.source_mask.data, @@ -472,6 +481,7 @@ def ipv4_rule(): comment=quote_to_ent(form.comment.data), action_id=form.action.data, user_id=session["user_id"], + org_id=session["user_org_id"], rstate_id=get_state_by_time(form.expires.data), ) flash_message = "IPv4 Rule saved" @@ -480,15 +490,11 @@ def ipv4_rule(): db.session.commit() flash(flash_message, "alert-success") - # increment counter if new rule was added - if increment: - increment_rule_count(RuleTypes.IPv4, form.organization.data) - # announce route if model is in active state if model.rstate_id == 1: command = messages.create_ipv4(model, constants.ANNOUNCE) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -499,7 +505,7 @@ def ipv4_rule(): session["user_id"], model, RuleTypes.IPv4, - f"{session['user_email']} / {session['user_orgs']}", + f"{session['user_email']} / {session['user_org']}", ) return redirect(url_for("index")) @@ -518,6 +524,9 @@ def ipv4_rule(): @auth_required @user_or_admin_required def ipv6_rule(): + if check_rule_limit(session["user_org_id"], RuleTypes.IPv6): + return redirect(url_for("rules.limit_reached", rule_type=RuleTypes.IPv6)) + net_ranges = get_user_nets(session["user_id"]) form = IPv6Form(request.form) @@ -551,6 +560,7 @@ def ipv6_rule(): comment=quote_to_ent(form.comment.data), action_id=form.action.data, user_id=session["user_id"], + org_id=session["user_org_id"], rstate_id=get_state_by_time(form.expires.data), ) flash_message = "IPv6 Rule saved" @@ -563,7 +573,7 @@ def ipv6_rule(): if model.rstate_id == 1: command = messages.create_ipv6(model, constants.ANNOUNCE) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -574,7 +584,7 @@ def ipv6_rule(): session["user_id"], model, RuleTypes.IPv6, - f"{session['user_email']} / {session['user_orgs']}", + f"{session['user_email']} / {session['user_org']}", ) return redirect(url_for("index")) @@ -593,6 +603,9 @@ def ipv6_rule(): @auth_required @user_or_admin_required def rtbh_rule(): + if check_rule_limit(session["user_org_id"], RuleTypes.RTBH): + return redirect(url_for("rules.limit_reached", rule_type=RuleTypes.RTBH)) + all_com = db.session.query(Community).all() if not all_com: insert_initial_communities() @@ -623,6 +636,7 @@ def rtbh_rule(): expires=round_to_ten_minutes(form.expires.data), comment=quote_to_ent(form.comment.data), user_id=session["user_id"], + org_id=session["user_org_id"], rstate_id=get_state_by_time(form.expires.data), ) db.session.add(model) @@ -634,7 +648,7 @@ def rtbh_rule(): if model.rstate_id == 1: command = messages.create_rtbh(model, constants.ANNOUNCE) route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"{session['user_email']} / {session['user_org']}", source=RouteSources.UI, command=command, ) @@ -644,7 +658,7 @@ def rtbh_rule(): session["user_id"], model, RuleTypes.RTBH, - f"{session['user_email']} / {session['user_orgs']}", + f"{session['user_email']} / {session['user_org']}", ) return redirect(url_for("index")) @@ -659,6 +673,24 @@ def rtbh_rule(): return render_template("forms/rtbh_rule.html", form=form, action_url=url_for("rules.rtbh_rule")) +@rules.route("/limit_reached/") +@auth_required +def limit_reached(rule_type): + rule_type = constants.RULE_NAMES_DICT[int(rule_type)] + count_4 = db.session.query(Flowspec4).filter_by(rstate_id=1, org_id=session["user_org_id"]).count() + count_6 = db.session.query(Flowspec6).filter_by(rstate_id=1, org_id=session["user_org_id"]).count() + count_rtbh = db.session.query(RTBH).filter_by(rstate_id=1, org_id=session["user_org_id"]).count() + org = db.session.query(Organization).get(session["user_org_id"]) + return render_template( + "pages/limit_reached.html", + rule_type=rule_type, + count_4=count_4, + count_6=count_6, + count_rtbh=count_rtbh, + org=org, + ) + + @rules.route("/export") @auth_required @admin_required @@ -738,9 +770,11 @@ def announce_all_routes(action=constants.ANNOUNCE): messages_all.extend(messages_v6) messages_all.extend(messages_rtbh) + author_action = "announce all" if action == constants.ANNOUNCE else "withdraw all expired" + for command in messages_all: route = Route( - author=f"{session['user_email']} / {session['user_orgs']}", + author=f"System call / {author_action} rules", source=RouteSources.UI, command=command, ) @@ -749,14 +783,6 @@ def announce_all_routes(action=constants.ANNOUNCE): if action == constants.WITHDRAW: for ruleset in [rules4, rules6, rules_rtbh]: for rule in ruleset: - set_withdraw_state(rule) - + rule.rstate_id = 2 -def set_withdraw_state(rule): - """ - set rule state to withdrawed in db - :param rule: rule to update, can be any of rule types - :return: none - """ - rule.rstate_id = 2 - db.session.commit() + db.session.commit() diff --git a/migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py b/migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py new file mode 100644 index 00000000..7a84878d --- /dev/null +++ b/migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py @@ -0,0 +1,42 @@ +"""organization in apikey and machineapikey models + +Revision ID: ae99a66ed9f8 +Revises: e25b385bae2c +Create Date: 2024-10-18 11:35:18.402633 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ae99a66ed9f8' +down_revision = 'e25b385bae2c' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('api_key', schema=None) as batch_op: + batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) + batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) + + with op.batch_alter_table('machine_api_key', schema=None) as batch_op: + batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) + batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('machine_api_key', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.drop_column('org_id') + + with op.batch_alter_table('api_key', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.drop_column('org_id') + + # ### end Alembic commands ### From 806455b414aec2acd20f15585c79eacc88ed0159 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 21 Oct 2024 19:43:46 +0200 Subject: [PATCH 29/54] global limit check as a separate function, viewpoint and template --- flowapp/models.py | 12 ++++- flowapp/templates/pages/limit_reached.html | 4 +- flowapp/tests/test_api_v3.py | 40 +++++++++++++++- flowapp/views/api_common.py | 56 +++++++++++++++++++--- flowapp/views/rules.py | 49 +++++++++++++++++++ 5 files changed, 151 insertions(+), 10 deletions(-) diff --git a/flowapp/models.py b/flowapp/models.py index 5ec94700..a5f2b312 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -749,7 +749,7 @@ def insert_initial_rulestates(table, conn, *args, **kwargs): # Misc functions -def check_rule_limit(org_id: int, rule_type: RuleTypes): +def check_rule_limit(org_id: int, rule_type: RuleTypes) -> bool: """ Check if the organization has reached the rule limit :param org_id: integer organization id @@ -773,7 +773,17 @@ def check_rule_limit(org_id: int, rule_type: RuleTypes): if rule_type == RuleTypes.RTBH and org.limit_rtbh > 0: count = db.session.query(RTBH).filter_by(org_id=org_id, rstate_id=1).count() return count >= org.limit_rtbh or rtbh >= rtbh_limit + + +def check_global_rule_limit(rule_type: RuleTypes) -> bool: + flowspec_limit = current_app.config.get("FLOWSPEC_MAX_RULES", 9000) + rtbh_limit = current_app.config.get("RTBH_MAX_RULES", 100000) + fs4 = db.session.query(Flowspec4).filter_by(rstate_id=1).count() + fs6 = db.session.query(Flowspec6).filter_by(rstate_id=1).count() + rtbh = db.session.query(RTBH).filter_by(rstate_id=1).count() + # check the global limits if the organization limits are not set + if rule_type == RuleTypes.IPv4: return fs4 >= flowspec_limit if rule_type == RuleTypes.IPv6: diff --git a/flowapp/templates/pages/limit_reached.html b/flowapp/templates/pages/limit_reached.html index 4129912c..2837e65f 100644 --- a/flowapp/templates/pages/limit_reached.html +++ b/flowapp/templates/pages/limit_reached.html @@ -4,10 +4,10 @@

You can't add new / reactivate {{ rule_type }} rule.

-
Rule limit of you organization has been reached.
+
{{ message }}
- + diff --git a/flowapp/tests/test_api_v3.py b/flowapp/tests/test_api_v3.py index d2b53998..9520b5d8 100644 --- a/flowapp/tests/test_api_v3.py +++ b/flowapp/tests/test_api_v3.py @@ -543,4 +543,42 @@ def test_update_existing_v4rule_with_timestamp_limit(client, db, app, jwt_token) assert req.status_code == 403 data = json.loads(req.data) assert data["message"] - assert data["message"].startswith("rule limit reached for") + assert data["message"].startswith("Rule limit") + + +def test_overall_limit(client, db, app, jwt_token): + """ + test that update with different data passes + """ + app.config.update({"FLOWSPEC_MAX_RULES": 5, "RTBH_MAX_RULES": 5}) + + with app.app_context(): + # count + + org = db.session.query(Organization).filter_by(id=1).first() + org.limit_flowspec4 = 20 + db.session.commit() + + sources = ["147.230.42.1", "147.230.42.2", "147.230.42.3", "147.230.42.4"] + codes = [201, 201, 201, 403] + + for source, code in zip(sources, codes): + data = { + "action": 1, + "protocol": "tcp", + "source": source, + "source_mask": 32, + "source_port": "", + "expires": "10/15/2050 14:46", + } + req = client.post( + f"{V_PREFIX}/rules/ipv4", + headers={"x-access-token": jwt_token}, + json=data, + ) + print(source) + assert req.status_code == code + + data = json.loads(req.data) + assert data["message"] + assert data["message"].startswith("System limit") diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index 92e5f668..e7ad21ae 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -5,7 +5,7 @@ from functools import wraps from datetime import datetime, timedelta -from flowapp.constants import WITHDRAW, ANNOUNCE, TIME_FORMAT_ARG, RuleTypes +from flowapp.constants import RULE_NAMES_DICT, WITHDRAW, ANNOUNCE, TIME_FORMAT_ARG, RuleTypes from flowapp.models import ( RTBH, Flowspec4, @@ -13,6 +13,8 @@ ApiKey, MachineApiKey, Community, + Organization, + check_global_rule_limit, check_rule_limit, get_user_nets, get_user_actions, @@ -188,8 +190,35 @@ def all_communities(current_user): return jsonify({"message": "no actions for this user?"}), 404 -def limit_reached(rule_type): - return jsonify({"message": f"rule limit reached for {rule_type}"}), 403 +def limit_reached(count, rule_type, org_id): + rule_name = RULE_NAMES_DICT[int(rule_type)] + org = db.session.query(Organization).get(org_id) + if rule_type == RuleTypes.IPv4: + limit = org.limit_flowspec4 + elif rule_type == RuleTypes.IPv6: + limit = org.limit_flowspec6 + elif rule_type == RuleTypes.RTBH: + limit = org.rtbh + + return ( + jsonify({"message": f"Rule limit {limit} reached for {rule_name}, currently you have {count} active rules."}), + 403, + ) + + +def global_limit_reached(count, rule_type): + rule_name = RULE_NAMES_DICT[int(rule_type)] + if rule_type == RuleTypes.IPv4 or rule_type == RuleTypes.IPv6: + limit = current_app.config.get("FLOWSPEC_MAX_RULES") + elif rule_type == RuleTypes.RTBH: + limit = current_app.config.get("RTBH_MAX_RULES") + + return ( + jsonify( + {"message": f"System limit {limit} reached for {rule_name}. Currently there are {count} active rules."} + ), + 403, + ) def create_ipv4(current_user): @@ -199,8 +228,13 @@ def create_ipv4(current_user): :param current_user: data from jwt token :return: json response """ + if check_global_rule_limit(RuleTypes.IPv4): + count = db.session.query(Flowspec4).filter_by(rstate_id=1).count() + return global_limit_reached(count=count, rule_type=RuleTypes.IPv4) + if check_rule_limit(current_user["org_id"], RuleTypes.IPv4): - return limit_reached(rule_type=RuleTypes.IPv4) + count = db.session.query(Flowspec4).filter_by(rstate_id=1, org_id=current_user["org_id"]).count() + return limit_reached(count=count, rule_type=RuleTypes.IPv4, org_id=current_user["org_id"]) net_ranges = get_user_nets(current_user["id"]) json_request_data = request.get_json() @@ -273,8 +307,13 @@ def create_ipv6(current_user): :param current_user: data from jwt token :return: """ + if check_global_rule_limit(RuleTypes.IPv6): + count = db.session.query(Flowspec6).filter_by(rstate_id=1).count() + return global_limit_reached(count=count, rule_type=RuleTypes.IPv6) + if check_rule_limit(current_user["org_id"], RuleTypes.IPv6): - return limit_reached(rule_type=RuleTypes.IPv6) + count = db.session.query(Flowspec6).filter_by(rstate_id=1, org_id=current_user["org_id"]).count() + return limit_reached(count=count, rule_type=RuleTypes.IPv6, org_id=current_user["org_id"]) net_ranges = get_user_nets(current_user["id"]) json_request_data = request.get_json() @@ -341,9 +380,14 @@ def create_rtbh(current_user): """ Create new RTBH rule """ + if check_global_rule_limit(RuleTypes.RTBH): + count = db.session.query(RTBH).filter_by(rstate_id=1).count() + return global_limit_reached(count=count, rule_type=RuleTypes.RTBH) + # check limit if check_rule_limit(current_user["org_id"], RuleTypes.RTBH): - return limit_reached(rule_type=RuleTypes.RTBH) + count = db.session.query(RTBH).filter_by(rstate_id=1, org_id=current_user["org_id"]).count() + return limit_reached(count=count, rule_type=RuleTypes.RTBH, org_id=current_user["org_id"]) all_com = db.session.query(Community).all() if not all_com: diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 38dfdaad..637c840f 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -1,6 +1,7 @@ # flowapp/views/admin.py from datetime import datetime, timedelta from operator import ge, lt +from collections import namedtuple from flask import Blueprint, current_app, flash, redirect, render_template, request, session, url_for @@ -21,6 +22,7 @@ Flowspec4, Flowspec6, Organization, + check_global_rule_limit, check_rule_limit, get_ipv4_model_if_exists, get_ipv6_model_if_exists, @@ -87,6 +89,12 @@ def reactivate_rule(rule_type, rule_id): if request.method == "POST": # check if rule will be reactivated state = get_state_by_time(form.expires.data) + + # check global limit + check_gl = check_global_rule_limit(rule_type) + if state == 1 and check_gl: + return redirect(url_for("rules.global_limit_reached", rule_type=rule_type)) + # check org limit if state == 1 and check_rule_limit(session["user_org_id"], rule_type=rule_type): return redirect(url_for("rules.limit_reached", rule_type=rule_type)) @@ -379,6 +387,11 @@ def group_update_save(rule_type): route_model = ROUTE_MODELS[rule_type] for rule_id in to_update: + # check global limit + check_gl = check_global_rule_limit(rule_type) + if rstate_id == 1 and check_gl: + return redirect(url_for("rules.global_limit_reached", rule_type=rule_type)) + # check if rule will be reactivated check = check_rule_limit(session["user_org_id"], rule_type=rule_type) if rstate_id == 1 and check: @@ -443,6 +456,9 @@ def group_update_save(rule_type): @auth_required @user_or_admin_required def ipv4_rule(): + if check_global_rule_limit(RuleTypes.IPv4): + return redirect(url_for("rules.global_limit_reached", rule_type=RuleTypes.IPv4)) + if check_rule_limit(session["user_org_id"], RuleTypes.IPv4): return redirect(url_for("rules.limit_reached", rule_type=RuleTypes.IPv4)) @@ -524,6 +540,9 @@ def ipv4_rule(): @auth_required @user_or_admin_required def ipv6_rule(): + if check_global_rule_limit(RuleTypes.IPv6): + return redirect(url_for("rules.global_limit_reached", rule_type=RuleTypes.IPv6)) + if check_rule_limit(session["user_org_id"], RuleTypes.IPv6): return redirect(url_for("rules.limit_reached", rule_type=RuleTypes.IPv6)) @@ -603,6 +622,9 @@ def ipv6_rule(): @auth_required @user_or_admin_required def rtbh_rule(): + if check_global_rule_limit(RuleTypes.RTBH): + return redirect(url_for("rules.global_limit_reached", rule_type=RuleTypes.RTBH)) + if check_rule_limit(session["user_org_id"], RuleTypes.RTBH): return redirect(url_for("rules.limit_reached", rule_type=RuleTypes.RTBH)) @@ -683,6 +705,7 @@ def limit_reached(rule_type): org = db.session.query(Organization).get(session["user_org_id"]) return render_template( "pages/limit_reached.html", + message="Your organization limit has been reached.", rule_type=rule_type, count_4=count_4, count_6=count_6, @@ -691,6 +714,32 @@ def limit_reached(rule_type): ) +@rules.route("/global_limit_reached/") +@auth_required +def global_limit_reached(rule_type): + rule_type = constants.RULE_NAMES_DICT[int(rule_type)] + count_4 = db.session.query(Flowspec4).filter_by(rstate_id=1).count() + count_6 = db.session.query(Flowspec6).filter_by(rstate_id=1).count() + count_rtbh = db.session.query(RTBH).filter_by(rstate_id=1).count() + + Limit = namedtuple("Limit", ["limit_flowspec4", "limit_flowspec6", "limit_rtbh"]) + limit = Limit( + limit_flowspec4=current_app.config["FLOWSPEC_MAX_RULES"], + limit_flowspec6=current_app.config["FLOWSPEC_MAX_RULES"], + limit_rtbh=current_app.config["RTBH_MAX_RULES"], + ) + + return render_template( + "pages/limit_reached.html", + message="Global system limit has been reached. Please contact your administrator.", + rule_type=rule_type, + count_4=count_4, + count_6=count_6, + count_rtbh=count_rtbh, + org=limit, + ) + + @rules.route("/export") @auth_required @admin_required From 4c2cff4cba138e858ef9e59cc924a676c02d9010 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 22 Oct 2024 09:15:47 +0200 Subject: [PATCH 30/54] debug update auth --- flowapp/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flowapp/auth.py b/flowapp/auth.py index ee2f8851..932526de 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -17,10 +17,10 @@ def decorated(*args, **kwargs): if not user: if current_app.config.get("SSO_AUTH"): current_app.logger.warning("SSO AUTH SET BUT FAILS") - return redirect(url_for("login")) + return redirect("/login") if current_app.config.get("HEADER_AUTH", False): - return redirect(url_for("ext_login")) + return redirect("/ext_login") if current_app.config.get("LOCAL_AUTH"): return redirect(url_for("local_login")) From 9b5e7bd196f395eb35b9302b212ec81540c878d2 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 22 Oct 2024 13:22:34 +0200 Subject: [PATCH 31/54] user bulk import --- flowapp/auth.py | 2 +- flowapp/forms.py | 55 +++++++++++++++++ flowapp/instance_config.py | 1 + flowapp/models.py | 12 +++- flowapp/templates/forms/bulk_user_form.html | 34 +++++++++++ flowapp/views/admin.py | 66 ++++++++++++++++++++- 6 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 flowapp/templates/forms/bulk_user_form.html diff --git a/flowapp/auth.py b/flowapp/auth.py index 932526de..7828abcc 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -16,7 +16,7 @@ def decorated(*args, **kwargs): user = get_user() if not user: if current_app.config.get("SSO_AUTH"): - current_app.logger.warning("SSO AUTH SET BUT FAILS") + current_app.logger.warning("SSO AUTH SET") return redirect("/login") if current_app.config.get("HEADER_AUTH", False): diff --git a/flowapp/forms.py b/flowapp/forms.py index 8a1396b2..4b914dce 100644 --- a/flowapp/forms.py +++ b/flowapp/forms.py @@ -1,3 +1,7 @@ +import csv +from io import StringIO + + from flask_wtf import FlaskForm from wtforms import widgets from wtforms import ( @@ -11,6 +15,7 @@ TextAreaField, ) from wtforms.validators import ( + ValidationError, DataRequired, Email, InputRequired, @@ -109,6 +114,56 @@ class UserForm(FlaskForm): ) +class BulkUserForm(FlaskForm): + """ + Bulk User Form object + used in Admin + """ + + users = TextAreaField("Users in CSV - see example below", validators=[DataRequired()]) + + def __init__(self, *args, **kwargs): + super(BulkUserForm, self).__init__(*args, **kwargs) + self.roles = None + self.organizations = None + self.uuids = None + + # Custom validator for CSV data + def validate_users(self, field): + csv_data = field.data + + # Parse CSV data + csv_reader = csv.DictReader(StringIO(csv_data), delimiter=",") + + # List to keep track of failed validation rows + errors = 0 + for row_num, row in enumerate(csv_reader, start=1): + try: + # check if the user not already exists + if row["uuid-eppn"] in self.uuids: + field.errors.append(f"Row {row_num}: User with UUID {row['uuid-eppn']} already exists.") + errors += 1 + + # Check if role exists in the database + role_id = int(row["role"]) # Convert role field to integer + if role_id not in self.roles: + field.errors.append(f"Row {row_num}: Role ID {role_id} does not exist.") + errors += 1 + + # Check if organization exists in the database + org_id = int(row["organizace"]) # Convert organization field to integer + if org_id not in self.organizations: + field.errors.append(f"Row {row_num}: Organization ID {org_id} does not exist.") + errors += 1 + + except (KeyError, ValueError) as e: + field.errors.append(f"Row {row_num}: Invalid data / key - {str(e)}. Check CSV head row.") + + if errors > 0: + # Raise validation error if any invalid rows found + raise ValidationError("Invalid CSV Data - check the errors above.") + + class ApiKeyForm(FlaskForm): """ ApiKey for User diff --git a/flowapp/instance_config.py b/flowapp/instance_config.py index 9d5a1bfa..1ffcb1f8 100644 --- a/flowapp/instance_config.py +++ b/flowapp/instance_config.py @@ -85,6 +85,7 @@ class InstanceConfig: "divide_before": True, }, {"name": "Add User", "url": "admin.user"}, + {"name": "Add Multiple Users", "url": "admin.bulk_import_users"}, {"name": "Organizations", "url": "admin.organizations"}, {"name": "Add Org.", "url": "admin.organization"}, { diff --git a/flowapp/models.py b/flowapp/models.py index a5f2b312..fe8cd67d 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -887,7 +887,15 @@ def insert_users(users): db.session.commit() -def insert_user(uuid, role_ids, org_ids, name=None, phone=None, email=None, comment=None): +def insert_user( + uuid: str, + role_ids: list, + org_ids: list, + name: str = None, + phone: str = None, + email: str = None, + comment: str = None, +): """ insert new user with multiple roles and organizations :param uuid: string unique user id (eppn or similar) @@ -900,7 +908,7 @@ def insert_user(uuid, role_ids, org_ids, name=None, phone=None, email=None, comm :return: None """ u = User(uuid=uuid, name=name, phone=phone, comment=comment, email=email) - + print(u) for role_id in role_ids: r = Role.query.filter_by(id=role_id).first() u.role.append(r) diff --git a/flowapp/templates/forms/bulk_user_form.html b/flowapp/templates/forms/bulk_user_form.html new file mode 100644 index 00000000..b5362cbe --- /dev/null +++ b/flowapp/templates/forms/bulk_user_form.html @@ -0,0 +1,34 @@ +{% extends 'layouts/default.html' %} +{% from 'forms/macros.html' import render_field, render_checkbox_field %} +{% block title %}Add New Machine with ApiKey{% endblock %} +{% block content %} +

Create multiple users.

+
+ {{ form.hidden_tag() if form.hidden_tag }} +
+
+ {{ render_field(form.users,) }} +
+
+ +
+
+ +
+
+ +
+
+

Example CSV data

+ 
+            
+uuid-eppn,name,telefon,email,role,organizace
+test@example.com,Test 1,123,test@example.com,1,1
+test-bad@example.com,Test 1,123,test-bad@example.com,4,4
+test2@example.com,Test 1,123456,test2@example.com,2,1
+            
+
+
+
+ +{% endblock %} \ No newline at end of file diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index 9791f2b6..fd1add18 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -1,12 +1,14 @@ -# flowapp/views/admin.py +import csv +from io import StringIO from datetime import datetime, timedelta import secrets from sqlalchemy import func from flask import Blueprint, render_template, redirect, flash, request, session, url_for, current_app +import sqlalchemy from sqlalchemy.exc import IntegrityError -from ..forms import ASPathForm, MachineApiKeyForm, UserForm, ActionForm, OrganizationForm, CommunityForm +from ..forms import ASPathForm, BulkUserForm, MachineApiKeyForm, UserForm, ActionForm, OrganizationForm, CommunityForm from ..models import ( ASPath, MachineApiKey, @@ -199,6 +201,66 @@ def users(): return render_template("pages/users.html", users=users) +@admin.route("/bulk-import-users", methods=["GET"]) +@auth_required +@admin_required +def bulk_import_users(): + form = BulkUserForm(request.form) + return render_template("forms/bulk_user_form.html", form=form) + + +@admin.route("/bulk-import-users", methods=["POST"]) +@auth_required +@admin_required +def bulk_import_users_save(): + form = BulkUserForm(request.form) + roles = [role.id for role in db.session.query(Role).all()] + orgs = [org.id for org in db.session.query(Organization).all()] + uuids = [user.uuid for user in db.session.query(User).all()] + form.roles = roles + form.organizations = orgs + form.uuids = uuids + + if request.method == "POST" and form.validate(): + # Get CSV data from textarea + csv_data = form.users.data + + # Parse CSV data + csv_reader = csv.DictReader(StringIO(csv_data), delimiter=",") + errored = False + for row in csv_reader: + try: + # Extract and prepare data + uuid = row["uuid/eppn"] + name = row["name"] + phone = row["telefon"] + email = row["email"] + + # Convert role and organization fields to lists of integers + role_ids = [int(row["role"])] # role_id should be a list + org_ids = [int(row["organizace"])] # org_id should be a list + + # Insert user + insert_user(uuid=uuid, role_ids=role_ids, org_ids=org_ids, name=name, phone=phone, email=email) + except KeyError as e: + errored = True + # Handle missing fields or other errors in the CSV + flash(f"Missing field in CSV: {e}", "alert-danger") + except ValueError as e: + errored = True + # Handle conversion issues (like invalid int for role/org) + flash(f"Data conversion error: {e}", "alert-danger") + except sqlalchemy.exc.IntegrityError as e: + errored = True + db.session.rollback() + flash(f"SQL Integrity error: {e}", "alert-danger") + + if not errored: + return redirect(url_for("admin.users")) + + return render_template("forms/bulk_user_form.html", form=form) + + @admin.route("/organizations") @auth_required @admin_required From 1aa5abfee9b1d6280c1c7ca94ccab07e1bcdea72 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Wed, 23 Oct 2024 12:43:19 +0200 Subject: [PATCH 32/54] flowspec4 and 6 have now separate limit for each. Updated admin to add org_id also to the ApiKey and MachineApiKey. Before getting rid of session.query (deprecated) --- config.example.py | 38 ++++++++++++++++-------------------- flowapp/models.py | 9 +++++---- flowapp/tests/test_api_v3.py | 2 +- flowapp/views/admin.py | 3 ++- flowapp/views/api_common.py | 2 +- flowapp/views/rules.py | 4 ++-- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/config.example.py b/config.example.py index 1583cfa2..2d4d4a29 100644 --- a/config.example.py +++ b/config.example.py @@ -3,16 +3,25 @@ class Config: Default config options """ + # Limits + FLOWSPEC4_MAX_RULES = 9000 + FLOWSPEC6_MAX_RULES = 9000 + RTBH_MAX_RULES = 100000 + # Flask debugging DEBUG = True # Flask testing TESTING = False - # SSO auth enabled - SSO_AUTH = True + # Choose your authentication method and set it to True here or + # the production / development config + # SSO auth enabled + SSO_AUTH = False # Authentication is done outside the app, use HTTP header to get the user uuid. # If SSO_AUTH is set to True, this option is ignored and SSO auth is used. HEADER_AUTH = False + # Local authentication - used when SSO_AUTH and HEADER_AUTH are set to False + LOCAL_AUTH = False # Name of HTTP header containing the UUID of authenticated user. # Only used when HEADER_AUTH is set to True @@ -24,7 +33,7 @@ class Config: # ExaApi configuration # possible values HTTP, RABBIT - EXA_API = "HTTP" + EXA_API = "RABBIT" # for HTTP EXA_API_URL must be specified EXA_API_URL = "http://localhost:5000/" # for RABBITMQ EXA_API_RABBIT_* must be specified @@ -39,24 +48,6 @@ class Config: JWT_SECRET = "GenerateSomeLongRandomSequence" SECRET_KEY = "GenerateSomeLongRandomSequence" - # LOCAL user parameters - when the app is used without SSO_AUTH - # Defined in User model - LOCAL_USER_UUID = "admin@example.com" - # Defined in User model - LOCAL_USER_ID = 1 - # Defined in Role model / default 1 - view, 2 - normal user, 3 - admin - LOCAL_USER_ROLES = ["admin"] - # Defined in Organization model - # List of organizations for the local user. There can be many of them. - # Define the name and the adress range. The range is then used for first data insert - # after the tables are created with db-init.py script. - LOCAL_USER_ORGS = [ - {"name": "Example Org.", "arange": "192.168.0.0/16\n2121:414:1a0b::/48"}, - ] - # Defined in Role model / default 1 - view, 2 - normal user, 3 - admin - LOCAL_USER_ROLE_IDS = [3] - # Defined in Organization model - LOCAL_USER_ORG_IDS = [1] # APP Name - display in main toolbar APP_NAME = "ExaFS" # Route Distinguisher for VRF @@ -105,6 +96,11 @@ class DevelopmentConfig(Config): DEBUG = True DEVEL = True + # LOCAL user parameters - when the app is used without SSO_AUTH + # Local User must be in the database + LOCAL_USER_UUID = "admin@example.com" + LOCAL_AUTH = True + class TestingConfig(Config): TESTING = True diff --git a/flowapp/models.py b/flowapp/models.py index fe8cd67d..4e1291f3 100644 --- a/flowapp/models.py +++ b/flowapp/models.py @@ -776,7 +776,8 @@ def check_rule_limit(org_id: int, rule_type: RuleTypes) -> bool: def check_global_rule_limit(rule_type: RuleTypes) -> bool: - flowspec_limit = current_app.config.get("FLOWSPEC_MAX_RULES", 9000) + flowspec4_limit = current_app.config.get("FLOWSPEC4_MAX_RULES", 9000) + flowspec6_limit = current_app.config.get("FLOWSPEC6_MAX_RULES", 9000) rtbh_limit = current_app.config.get("RTBH_MAX_RULES", 100000) fs4 = db.session.query(Flowspec4).filter_by(rstate_id=1).count() fs6 = db.session.query(Flowspec6).filter_by(rstate_id=1).count() @@ -785,9 +786,9 @@ def check_global_rule_limit(rule_type: RuleTypes) -> bool: # check the global limits if the organization limits are not set if rule_type == RuleTypes.IPv4: - return fs4 >= flowspec_limit + return fs4 >= flowspec4_limit if rule_type == RuleTypes.IPv6: - return fs6 >= flowspec_limit + return fs6 >= flowspec6_limit if rule_type == RuleTypes.RTBH: return rtbh >= rtbh_limit @@ -908,7 +909,7 @@ def insert_user( :return: None """ u = User(uuid=uuid, name=name, phone=phone, comment=comment, email=email) - print(u) + for role_id in role_ids: r = Role.query.filter_by(id=role_id).first() u.role.append(r) diff --git a/flowapp/tests/test_api_v3.py b/flowapp/tests/test_api_v3.py index 9520b5d8..96b24786 100644 --- a/flowapp/tests/test_api_v3.py +++ b/flowapp/tests/test_api_v3.py @@ -550,7 +550,7 @@ def test_overall_limit(client, db, app, jwt_token): """ test that update with different data passes """ - app.config.update({"FLOWSPEC_MAX_RULES": 5, "RTBH_MAX_RULES": 5}) + app.config.update({"FLOWSPEC4_MAX_RULES": 5, "FLOWSPEC6_MAX_RULES": 5, "RTBH_MAX_RULES": 5}) with app.app_context(): # count diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index fd1add18..da7be48c 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -11,6 +11,7 @@ from ..forms import ASPathForm, BulkUserForm, MachineApiKeyForm, UserForm, ActionForm, OrganizationForm, CommunityForm from ..models import ( ASPath, + ApiKey, MachineApiKey, User, Action, @@ -633,7 +634,7 @@ def delete_community(community_id): def update_rules_set_org(): # Get all flowspec records where org_id is NULL (if this is needed) - models = [Flowspec4, Flowspec6, RTBH] + models = [Flowspec4, Flowspec6, RTBH, ApiKey, MachineApiKey] user_with_multiple_orgs = {} for model in models: rules = model.query.filter(model.org_id == 0).all() diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index e7ad21ae..61159773 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -192,7 +192,7 @@ def all_communities(current_user): def limit_reached(count, rule_type, org_id): rule_name = RULE_NAMES_DICT[int(rule_type)] - org = db.session.query(Organization).get(org_id) + org = db.session.get(Organization, org_id) if rule_type == RuleTypes.IPv4: limit = org.limit_flowspec4 elif rule_type == RuleTypes.IPv6: diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 637c840f..60fd3019 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -724,8 +724,8 @@ def global_limit_reached(rule_type): Limit = namedtuple("Limit", ["limit_flowspec4", "limit_flowspec6", "limit_rtbh"]) limit = Limit( - limit_flowspec4=current_app.config["FLOWSPEC_MAX_RULES"], - limit_flowspec6=current_app.config["FLOWSPEC_MAX_RULES"], + limit_flowspec4=current_app.config["FLOWSPEC4_MAX_RULES"], + limit_flowspec6=current_app.config["FLOWSPEC6_MAX_RULES"], limit_rtbh=current_app.config["RTBH_MAX_RULES"], ) From b3412333b152a1881877e6d865d9affc9ab33874 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Wed, 23 Oct 2024 13:39:08 +0200 Subject: [PATCH 33/54] converted all session.query().get() to session.get() --- flowapp/views/admin.py | 22 +++++++++++----------- flowapp/views/api_common.py | 4 ++-- flowapp/views/api_keys.py | 2 +- flowapp/views/rules.py | 10 +++++----- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index da7be48c..53a6b86c 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -105,7 +105,7 @@ def delete_machine_key(key_id): Delete api_key and machine :param key_id: integer """ - model = db.session.query(MachineApiKey).get(key_id) + model = db.session.get(MachineApiKey, key_id) # delete from db db.session.delete(model) db.session.commit() @@ -153,7 +153,7 @@ def user(): @auth_required @admin_required def edit_user(user_id): - user = db.session.query(User).get(user_id) + user = db.session.get(User, user_id) form = UserForm(request.form, obj=user) form.role_ids.choices = [(g.id, g.name) for g in db.session.query(Role).order_by("name")] form.org_ids.choices = [(g.id, g.name) for g in db.session.query(Organization).order_by("name")] @@ -178,7 +178,7 @@ def edit_user(user_id): @auth_required @admin_required def delete_user(user_id): - user = db.session.query(User).get(user_id) + user = db.session.get(User, user_id) username = user.email db.session.delete(user) @@ -350,7 +350,7 @@ def organization(): @auth_required @admin_required def edit_organization(org_id): - org = db.session.query(Organization).get(org_id) + org = db.session.get(Organization, org_id) form = OrganizationForm(request.form, obj=org) if request.method == "POST" and form.validate(): @@ -372,7 +372,7 @@ def edit_organization(org_id): @auth_required @admin_required def delete_organization(org_id): - org = db.session.query(Organization).get(org_id) + org = db.session.get(Organization, org_id) aname = org.name db.session.delete(org) message = "Organization {} deleted".format(aname) @@ -428,7 +428,7 @@ def as_path(): @auth_required @admin_required def edit_as_path(path_id): - pth = db.session.query(ASPath).get(path_id) + pth = db.session.get(ASPath, path_id) form = ASPathForm(request.form, obj=pth) if request.method == "POST" and form.validate(): @@ -450,7 +450,7 @@ def edit_as_path(path_id): @auth_required @admin_required def delete_as_path(path_id): - pth = db.session.query(ASPath).get(path_id) + pth = db.session.get(ASPath, path_id) db.session.delete(pth) message = f"AS path {pth.prefix} : {pth.as_path} deleted" alert_type = "alert-success" @@ -508,7 +508,7 @@ def action(): @auth_required @admin_required def edit_action(action_id): - action = db.session.query(Action).get(action_id) + action = db.session.get(Action, action_id) form = ActionForm(request.form, obj=action) if request.method == "POST" and form.validate(): form.populate_obj(action) @@ -529,7 +529,7 @@ def edit_action(action_id): @auth_required @admin_required def delete_action(action_id): - action = db.session.query(Action).get(action_id) + action = db.session.get(Action, action_id) aname = action.name db.session.delete(action) @@ -592,7 +592,7 @@ def community(): @auth_required @admin_required def edit_community(community_id): - community = db.session.query(Community).get(community_id) + community = db.session.get(Community, community_id) form = CommunityForm(request.form, obj=community) if request.method == "POST" and form.validate(): form.populate_obj(community) @@ -613,7 +613,7 @@ def edit_community(community_id): @auth_required @admin_required def delete_community(community_id): - community = db.session.query(Community).get(community_id) + community = db.session.get(Community, community_id) aname = community.name db.session.delete(community) message = "Community {} deleted".format(aname) diff --git a/flowapp/views/api_common.py b/flowapp/views/api_common.py index 61159773..6ce24bf6 100644 --- a/flowapp/views/api_common.py +++ b/flowapp/views/api_common.py @@ -456,7 +456,7 @@ def ipv4_rule_get(current_user, rule_id): :param rule_id: :return: """ - model = db.session.query(Flowspec4).get(rule_id) + model = db.session.get(Flowspec4, rule_id) return get_rule(current_user, model, rule_id) @@ -467,7 +467,7 @@ def ipv6_rule_get(current_user, rule_id): :param rule_id: :return: """ - model = db.session.query(Flowspec6).get(rule_id) + model = db.session.get(Flowspec6, rule_id) return get_rule(current_user, model, rule_id) diff --git a/flowapp/views/api_keys.py b/flowapp/views/api_keys.py index c88a9e25..7eae8c3e 100644 --- a/flowapp/views/api_keys.py +++ b/flowapp/views/api_keys.py @@ -89,7 +89,7 @@ def delete(key_id): key_list = request.cookies.get(COOKIE_KEY) key_list = jwt.decode(key_list, current_app.config.get("JWT_SECRET"), algorithms=["HS256"]) - model = db.session.query(ApiKey).get(key_id) + model = db.session.get(ApiKey, key_id) if model.id not in key_list["keys"]: flash("You can't delete this key!", "alert-danger") elif model.user_id == session["user_id"] or 3 in session["user_role_ids"]: diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 60fd3019..41d68a4a 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -67,7 +67,7 @@ def reactivate_rule(rule_type, rule_id): model_name = DATA_MODELS[rule_type] form_name = DATA_FORMS[rule_type] - model = db.session.query(model_name).get(rule_id) + model = db.session.get(model_name, rule_id) form = form_name(request.form, obj=model) form.net_ranges = get_user_nets(session["user_id"]) @@ -186,7 +186,7 @@ def delete_rule(rule_type, rule_id): model_name = DATA_MODELS[rule_type] route_model = ROUTE_MODELS[rule_type] - model = db.session.query(model_name).get(rule_id) + model = db.session.get(model_name, rule_id) if model.id in session[constants.RULES_KEY]: # withdraw route command = route_model(model, constants.WITHDRAW) @@ -267,7 +267,7 @@ def group_delete(): if set(to_delete).issubset(set(rules)) or is_admin(session["user_roles"]): for rule_id in to_delete: # withdraw route - model = db.session.query(model_name).get(rule_id) + model = db.session.get(model_name, rule_id) command = route_model(model, constants.WITHDRAW) route = Route( author=f"{session['user_email']} / {session['user_org']}", @@ -398,7 +398,7 @@ def group_update_save(rule_type): return redirect(url_for("rules.limit_reached", rule_type=rule_type)) # update record - model = db.session.query(model_name).get(rule_id) + model = db.session.get(model_name, rule_id) model.expires = expires model.rstate_id = rstate_id model.comment = f"{model.comment} {comment}" @@ -702,7 +702,7 @@ def limit_reached(rule_type): count_4 = db.session.query(Flowspec4).filter_by(rstate_id=1, org_id=session["user_org_id"]).count() count_6 = db.session.query(Flowspec6).filter_by(rstate_id=1, org_id=session["user_org_id"]).count() count_rtbh = db.session.query(RTBH).filter_by(rstate_id=1, org_id=session["user_org_id"]).count() - org = db.session.query(Organization).get(session["user_org_id"]) + org = db.session.get(Organization, session["user_org_id"]) return render_template( "pages/limit_reached.html", message="Your organization limit has been reached.", From ad69ed538b8bc6e1d5b9c92601cc3af479c78233 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Wed, 23 Oct 2024 14:29:39 +0200 Subject: [PATCH 34/54] added org with id to multiple user editor --- flowapp/__about__.py | 2 +- flowapp/templates/forms/bulk_user_form.html | 20 +++++++++++++++++++- flowapp/templates/pages/user_list.html | 2 +- flowapp/views/admin.py | 7 ++++--- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/flowapp/__about__.py b/flowapp/__about__.py index 1c11a6ec..ca651cc2 100755 --- a/flowapp/__about__.py +++ b/flowapp/__about__.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "1.0.1" diff --git a/flowapp/templates/forms/bulk_user_form.html b/flowapp/templates/forms/bulk_user_form.html index b5362cbe..34f8e2e6 100644 --- a/flowapp/templates/forms/bulk_user_form.html +++ b/flowapp/templates/forms/bulk_user_form.html @@ -7,7 +7,25 @@

Create multiple users.

{{ form.hidden_tag() if form.hidden_tag }}
- {{ render_field(form.users,) }} + {{ render_field(form.users, rows=12) }} +
+
+
Rule typeCurrent countYour org LimitRule typeCurrent countLimit
IPv4 (Flowspec4){{ count_4 }}{{ org.limit_flowspec4|unlimited }}
+ + + + + + + + {% for org in orgs %} + + + + + {% endfor %} + +
Organization nameID
{{ org.name }}{{ org.id }}
diff --git a/flowapp/templates/pages/user_list.html b/flowapp/templates/pages/user_list.html index ba08dc3d..a26c8577 100644 --- a/flowapp/templates/pages/user_list.html +++ b/flowapp/templates/pages/user_list.html @@ -1,7 +1,7 @@ {% extends 'layouts/default.html' %} {% block title %}Flowspec Actions{% endblock %} {% block content %} -

Updated {{ updated}} rules.

+

Updated {{ updated}} records.

Users with multiple organizations

Records of users with multilple orgs could not be updated.

{% for user, orgs in users.items() %} diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index 53a6b86c..081c151b 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -207,7 +207,8 @@ def users(): @admin_required def bulk_import_users(): form = BulkUserForm(request.form) - return render_template("forms/bulk_user_form.html", form=form) + orgs = db.session.execute(db.select(Organization).order_by(Organization.name)).scalars() + return render_template("forms/bulk_user_form.html", form=form, orgs=orgs) @admin.route("/bulk-import-users", methods=["POST"]) @@ -628,10 +629,10 @@ def delete_community(community_id): return redirect(url_for("admin.communities")) -@admin.route("/set-org-for-rules", methods=["GET"]) +@admin.route("/set-org-if-zero", methods=["GET"]) @auth_required @admin_required -def update_rules_set_org(): +def update_set_org(): # Get all flowspec records where org_id is NULL (if this is needed) models = [Flowspec4, Flowspec6, RTBH, ApiKey, MachineApiKey] From 546267c432e5a06798d9eee3dffc59cabddbc7e3 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 5 Nov 2024 12:25:25 +0100 Subject: [PATCH 35/54] flowspec4 and 6 default expiration only 1 hour, RTBH kept 7 days --- flowapp/views/rules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flowapp/views/rules.py b/flowapp/views/rules.py index 41d68a4a..5618789f 100644 --- a/flowapp/views/rules.py +++ b/flowapp/views/rules.py @@ -530,7 +530,8 @@ def ipv4_rule(): for error in errors: current_app.logger.debug("Error in the %s field - %s" % (getattr(form, field).label.text, error)) - default_expires = datetime.now() + timedelta(days=7) + print("NOW", datetime.now()) + default_expires = datetime.now() + timedelta(hours=1) form.expires.data = default_expires return render_template("forms/ipv4_rule.html", form=form, action_url=url_for("rules.ipv4_rule")) @@ -612,7 +613,7 @@ def ipv6_rule(): for error in errors: current_app.logger.debug("Error in the %s field - %s" % (getattr(form, field).label.text, error)) - default_expires = datetime.now() + timedelta(days=7) + default_expires = datetime.now() + timedelta(hours=1) form.expires.data = default_expires return render_template("forms/ipv6_rule.html", form=form, action_url=url_for("rules.ipv6_rule")) From 78dfa5be55432520a1a8abc6e7afe9e34621631f Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 5 Nov 2024 13:09:59 +0100 Subject: [PATCH 36/54] fix operationalerror in admin.delete_user --- flowapp/views/admin.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index 081c151b..b45c92e6 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -6,7 +6,7 @@ from sqlalchemy import func from flask import Blueprint, render_template, redirect, flash, request, session, url_for, current_app import sqlalchemy -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, OperationalError from ..forms import ASPathForm, BulkUserForm, MachineApiKeyForm, UserForm, ActionForm, OrganizationForm, CommunityForm from ..models import ( @@ -179,15 +179,25 @@ def edit_user(user_id): @admin_required def delete_user(user_id): user = db.session.get(User, user_id) + if not user: + flash("User not found.", "alert-danger") + return redirect(url_for("admin.users")) + username = user.email db.session.delete(user) - message = "User {} deleted".format(username) + message = f"User {username} deleted" alert_type = "alert-success" + try: db.session.commit() except IntegrityError: - message = f"User {username} owns some rules, can not be deleted!" + db.session.rollback() # Rollback on IntegrityError + message = f"User {username} owns some rules, cannot be deleted! Delete rules first." + alert_type = "alert-danger" + except OperationalError: + db.session.rollback() # Rollback on OperationalError + message = f"User {username} owns some rules, cannot be deleted! Delete rules first." alert_type = "alert-danger" flash(message, alert_type) From 5f9b3a403645bd819da7b44e764e2f84c404303b Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Wed, 6 Nov 2024 09:50:27 +0100 Subject: [PATCH 37/54] no migrations in git --- .gitignore | 1 + migrations/README | 1 - migrations/alembic.ini | 50 -------- migrations/env.py | 113 ------------------ migrations/script.py.mako | 24 ---- ...ed7f6_add_rule_counters_to_organization.py | 36 ------ ...224c9f1f_add_rule_limit_to_organization.py | 32 ----- ...e99a66ed9f8_organization_in_apikey_and_.py | 42 ------- ...imit_for_each_type_of_rule_org_id_as_a_.py | 74 ------------ 9 files changed, 1 insertion(+), 372 deletions(-) delete mode 100644 migrations/README delete mode 100644 migrations/alembic.ini delete mode 100644 migrations/env.py delete mode 100644 migrations/script.py.mako delete mode 100644 migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py delete mode 100644 migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py delete mode 100644 migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py delete mode 100644 migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py diff --git a/.gitignore b/.gitignore index 436de6f0..b5bca2aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ config.py run.py +migrations/ # PyCharm .idea/ diff --git a/migrations/README b/migrations/README deleted file mode 100644 index 0e048441..00000000 --- a/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Single-database configuration for Flask. diff --git a/migrations/alembic.ini b/migrations/alembic.ini deleted file mode 100644 index ec9d45c2..00000000 --- a/migrations/alembic.ini +++ /dev/null @@ -1,50 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic,flask_migrate - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[logger_flask_migrate] -level = INFO -handlers = -qualname = flask_migrate - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py deleted file mode 100644 index 4c970927..00000000 --- a/migrations/env.py +++ /dev/null @@ -1,113 +0,0 @@ -import logging -from logging.config import fileConfig - -from flask import current_app - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -fileConfig(config.config_file_name) -logger = logging.getLogger('alembic.env') - - -def get_engine(): - try: - # this works with Flask-SQLAlchemy<3 and Alchemical - return current_app.extensions['migrate'].db.get_engine() - except (TypeError, AttributeError): - # this works with Flask-SQLAlchemy>=3 - return current_app.extensions['migrate'].db.engine - - -def get_engine_url(): - try: - return get_engine().url.render_as_string(hide_password=False).replace( - '%', '%%') - except AttributeError: - return str(get_engine().url).replace('%', '%%') - - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -config.set_main_option('sqlalchemy.url', get_engine_url()) -target_db = current_app.extensions['migrate'].db - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def get_metadata(): - if hasattr(target_db, 'metadatas'): - return target_db.metadatas[None] - return target_db.metadata - - -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, target_metadata=get_metadata(), literal_binds=True - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - - # this callback is used to prevent an auto-migration from being generated - # when there are no changes to the schema - # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html - def process_revision_directives(context, revision, directives): - if getattr(config.cmd_opts, 'autogenerate', False): - script = directives[0] - if script.upgrade_ops.is_empty(): - directives[:] = [] - logger.info('No changes in schema detected.') - - conf_args = current_app.extensions['migrate'].configure_args - if conf_args.get("process_revision_directives") is None: - conf_args["process_revision_directives"] = process_revision_directives - - connectable = get_engine() - - with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=get_metadata(), - **conf_args - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako deleted file mode 100644 index 2c015630..00000000 --- a/migrations/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py b/migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py deleted file mode 100644 index f6b6e107..00000000 --- a/migrations/versions/58ac38ced7f6_add_rule_counters_to_organization.py +++ /dev/null @@ -1,36 +0,0 @@ -"""add rule counters to organization - -Revision ID: 58ac38ced7f6 -Revises: 7719224c9f1f -Create Date: 2024-10-15 16:53:48.222167 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '58ac38ced7f6' -down_revision = '7719224c9f1f' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.add_column(sa.Column('ipv4count', sa.Integer(), nullable=True)) - batch_op.add_column(sa.Column('ipv6count', sa.Integer(), nullable=True)) - batch_op.add_column(sa.Column('rtbhcount', sa.Integer(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.drop_column('rtbhcount') - batch_op.drop_column('ipv6count') - batch_op.drop_column('ipv4count') - - # ### end Alembic commands ### diff --git a/migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py b/migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py deleted file mode 100644 index 97b623de..00000000 --- a/migrations/versions/7719224c9f1f_add_rule_limit_to_organization.py +++ /dev/null @@ -1,32 +0,0 @@ -"""add rule_limit to organization - -Revision ID: 7719224c9f1f -Revises: -Create Date: 2024-10-15 15:44:56.937747 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '7719224c9f1f' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.add_column(sa.Column('rule_limit', sa.Integer(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.drop_column('rule_limit') - - # ### end Alembic commands ### diff --git a/migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py b/migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py deleted file mode 100644 index 7a84878d..00000000 --- a/migrations/versions/ae99a66ed9f8_organization_in_apikey_and_.py +++ /dev/null @@ -1,42 +0,0 @@ -"""organization in apikey and machineapikey models - -Revision ID: ae99a66ed9f8 -Revises: e25b385bae2c -Create Date: 2024-10-18 11:35:18.402633 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'ae99a66ed9f8' -down_revision = 'e25b385bae2c' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('api_key', schema=None) as batch_op: - batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) - batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) - - with op.batch_alter_table('machine_api_key', schema=None) as batch_op: - batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) - batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('machine_api_key', schema=None) as batch_op: - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.drop_column('org_id') - - with op.batch_alter_table('api_key', schema=None) as batch_op: - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.drop_column('org_id') - - # ### end Alembic commands ### diff --git a/migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py b/migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py deleted file mode 100644 index 6e463aef..00000000 --- a/migrations/versions/e25b385bae2c_limit_for_each_type_of_rule_org_id_as_a_.py +++ /dev/null @@ -1,74 +0,0 @@ -"""limit for each type of rule, org id as a key for each rule - -Revision ID: e25b385bae2c -Revises: 58ac38ced7f6 -Create Date: 2024-10-17 16:41:01.116727 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision = 'e25b385bae2c' -down_revision = '58ac38ced7f6' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('RTBH', schema=None) as batch_op: - batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) - batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) - - with op.batch_alter_table('flowspec4', schema=None) as batch_op: - batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) - batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) - - with op.batch_alter_table('flowspec6', schema=None) as batch_op: - batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=False)) - batch_op.create_foreign_key(None, 'organization', ['org_id'], ['id']) - - with op.batch_alter_table('log', schema=None) as batch_op: - batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=True)) - - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.add_column(sa.Column('limit_flowspec4', sa.Integer(), nullable=True)) - batch_op.add_column(sa.Column('limit_flowspec6', sa.Integer(), nullable=True)) - batch_op.add_column(sa.Column('limit_rtbh', sa.Integer(), nullable=True)) - batch_op.drop_column('ipv6count') - batch_op.drop_column('rule_limit') - batch_op.drop_column('rtbhcount') - batch_op.drop_column('ipv4count') - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.add_column(sa.Column('ipv4count', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) - batch_op.add_column(sa.Column('rtbhcount', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) - batch_op.add_column(sa.Column('rule_limit', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) - batch_op.add_column(sa.Column('ipv6count', mysql.INTEGER(display_width=11), autoincrement=False, nullable=True)) - batch_op.drop_column('limit_rtbh') - batch_op.drop_column('limit_flowspec6') - batch_op.drop_column('limit_flowspec4') - - with op.batch_alter_table('log', schema=None) as batch_op: - batch_op.drop_column('org_id') - - with op.batch_alter_table('flowspec6', schema=None) as batch_op: - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.drop_column('org_id') - - with op.batch_alter_table('flowspec4', schema=None) as batch_op: - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.drop_column('org_id') - - with op.batch_alter_table('RTBH', schema=None) as batch_op: - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.drop_column('org_id') - - # ### end Alembic commands ### From ab9b83346ebf48a2bec050db4c2414feed1ed1a1 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Wed, 6 Nov 2024 13:38:30 +0100 Subject: [PATCH 38/54] update endpoit for org id setup --- flowapp/views/admin.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index b45c92e6..982f86c6 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -643,26 +643,43 @@ def delete_community(community_id): @auth_required @admin_required def update_set_org(): + # Define the raw SQL update statement + update_statement = """ + UPDATE organization + SET limit_flowspec4 = 0, limit_flowspec6 = 0, limit_rtbh = 0 + WHERE limit_flowspec4 IS NULL OR limit_flowspec6 IS NULL OR limit_rtbh IS NULL; + """ + try: + # Execute the update query + db.session.execute(update_statement) + db.session.commit() + except Exception as e: + db.session.rollback() + flash(f"Error updating organizations: {e}", "alert-danger") # Get all flowspec records where org_id is NULL (if this is needed) models = [Flowspec4, Flowspec6, RTBH, ApiKey, MachineApiKey] user_with_multiple_orgs = {} for model in models: - rules = model.query.filter(model.org_id == 0).all() - print(f"Found {len(rules)} records with org_id NULL in {model.__name__}") + data_records = model.query.filter(model.org_id == 0).all() + print(f"Found {len(data_records)} records with org_id NULL in {model.__name__}") # Loop through each flowspec record and update org_id based on the user's organization updated = 0 - for rule in rules: - orgs = rule.user.organization.all() + for row in data_records: + orgs = row.user.organization.all() if len(orgs) == 1: user_org = orgs[0] if user_org: - rule.org_id = user_org.id + row.org_id = user_org.id updated += 1 else: - print(f"User {rule.user.email} has multiple organizations") - user_with_multiple_orgs[rule.user.email] = [org.name for org in orgs] + print(f"User {row.user.email} has multiple organizations") + user_with_multiple_orgs[row.user.email] = [org.name for org in orgs] # Commit the changes - db.session.commit() + try: + db.session.commit() + except Exception as e: + db.session.rollback() + flash(f"Error updating {model.__name__}: {e}", "alert-danger") return render_template("pages/user_list.html", users=user_with_multiple_orgs, updated=updated) From 9f86337a9874b6d26426f19ba3c671606360ec3f Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Wed, 6 Nov 2024 13:49:41 +0100 Subject: [PATCH 39/54] update endpoit for org id setup --- flowapp/views/admin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index 982f86c6..b8a899cb 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -3,7 +3,7 @@ from datetime import datetime, timedelta import secrets -from sqlalchemy import func +from sqlalchemy import func, text from flask import Blueprint, render_template, redirect, flash, request, session, url_for, current_app import sqlalchemy from sqlalchemy.exc import IntegrityError, OperationalError @@ -644,11 +644,13 @@ def delete_community(community_id): @admin_required def update_set_org(): # Define the raw SQL update statement - update_statement = """ + update_statement = update_statement = text( + """ UPDATE organization SET limit_flowspec4 = 0, limit_flowspec6 = 0, limit_rtbh = 0 WHERE limit_flowspec4 IS NULL OR limit_flowspec6 IS NULL OR limit_rtbh IS NULL; """ + ) try: # Execute the update query db.session.execute(update_statement) From c2285e0c1ae4ca7f4e435cbaa1508ce1869eadd5 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 7 Nov 2024 08:49:20 +0100 Subject: [PATCH 40/54] fix show version --- flowapp/auth.py | 1 + flowapp/templates/layouts/default.html | 2 ++ 2 files changed, 3 insertions(+) diff --git a/flowapp/auth.py b/flowapp/auth.py index 7828abcc..553eb529 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -14,6 +14,7 @@ def auth_required(f): @wraps(f) def decorated(*args, **kwargs): user = get_user() + session["app_version"] = __version__ if not user: if current_app.config.get("SSO_AUTH"): current_app.logger.warning("SSO AUTH SET") diff --git a/flowapp/templates/layouts/default.html b/flowapp/templates/layouts/default.html index dd0dd4c9..47b049c4 100644 --- a/flowapp/templates/layouts/default.html +++ b/flowapp/templates/layouts/default.html @@ -57,6 +57,8 @@ {% endif %}
  • {{ item.name }}
    • {% endfor %} +
    • +
  • ExaFS version {{ session['app_version'] }}
    • {% endif %} From 1e68bcbac3430732ad11f4ae3cf3052b176df8ef Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 7 Nov 2024 08:50:33 +0100 Subject: [PATCH 41/54] deprecated check_auth function removed from auth.py --- flowapp/auth.py | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/flowapp/auth.py b/flowapp/auth.py index 553eb529..2f12cee8 100644 --- a/flowapp/auth.py +++ b/flowapp/auth.py @@ -1,8 +1,7 @@ from functools import wraps from flask import current_app, redirect, request, url_for, session, abort -from flowapp import db, __version__ -from .models import User +from flowapp import __version__ # auth atd. @@ -77,38 +76,6 @@ def decorated(*args, **kwargs): return decorated -def check_auth(uuid): - """ - This function is every time when someone accessing the endpoint - - Default behaviour is that uuid from SSO AUTH is used. If SSO AUTH is not used - default local user and roles are taken from database. In that case there is no user auth check - and it needs to be done outside the app - for example in Apache. - """ - - session["app_version"] = __version__ - - if current_app.config.get("HEADER_AUTH", False): - # External auth (for example apache) - header_name = current_app.config.get("AUTH_HEADER_NAME", "X-Authenticated-User") - if header_name not in request.headers or not session.get("user_uuid"): - return False - return db.session.query(User).filter_by(uuid=request.headers.get(header_name)) - - if current_app.config.get("SSO_AUTH"): - current_app.logger.warning("CHECK AUTH, SSO AUTH SET UUID : {uuid}") - elif current_app.config.get("LOCAL_AUTH"): - uuid = current_app.config.get("LOCAL_USER_UUID", False) - current_app.logger.warning(f"CHECK AUTH, LOCAL AUTH SET UUID: {uuid}") - - exist = False - if uuid: - exist = db.session.query(User).filter_by(uuid=uuid).first() - - current_app.logger.debug(f"CHECK AUTH RETURN, UUID: {uuid}, EXIST: {exist}") - return exist - - def check_access_rights(current_user, model_id): """ Check if the current user has right to edit/delete certain model data From c3cf7d345a24a824834f1948f4e256acaa775865 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Fri, 29 Nov 2024 09:15:21 +0100 Subject: [PATCH 42/54] minor docs update --- docs/INSTALL.md | 7 ++++--- .../supervisor/exafs.supervisord.conf | 0 .../supervisor/supervisord.example.service | 0 setup.py | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) rename exafs.supervisord.conf => docs/supervisor/exafs.supervisord.conf (100%) rename supervisord.example.service => docs/supervisor/supervisord.example.service (100%) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 2589709b..c8067a6b 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -100,7 +100,8 @@ systemctl start httpd #### Supervisord - install as root -Supervisord is used to run and manage application. +Supervisord is used to run and manage applications, but it is not mandatory for deployment. +You can skip this section if you are using a different deployment method, such as Docker. 1. install: `pip install supervisor` @@ -112,9 +113,9 @@ Supervisord is used to run and manage application. 3. setup as service: - `cp supervisord.example.service /usr/lib/systemd/system/supervisord.service` + `cp docs/supervisor/supervisord.example.service /usr/lib/systemd/system/supervisord.service` 4. copy exafs.supervisord.conf to /etc/supervisord/ - `cp exafs.supervisord.conf /etc/supervisord/conf.d/` + `cp docs/supervisor/exafs.supervisord.conf /etc/supervisord/conf.d/` 5. start service `systemctl start supervisord` 6. view service status: diff --git a/exafs.supervisord.conf b/docs/supervisor/exafs.supervisord.conf similarity index 100% rename from exafs.supervisord.conf rename to docs/supervisor/exafs.supervisord.conf diff --git a/supervisord.example.service b/docs/supervisor/supervisord.example.service similarity index 100% rename from supervisord.example.service rename to docs/supervisor/supervisord.example.service diff --git a/setup.py b/setup.py index b10fb667..c28a06d8 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ """ Author(s): Jiri Vrany +Petr Adamec Jakub Man Setuptools configuration From 8598c4cb1dda77db917596bf952cee18928f62fe Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 3 Dec 2024 09:19:29 +0100 Subject: [PATCH 43/54] api description in the docs --- docs/api/apiario.md | 493 ++++++++++++++++++++++++++++++++++++++++++ docs/api/swagger,yaml | 422 ++++++++++++++++++++++++++++++++++++ 2 files changed, 915 insertions(+) create mode 100644 docs/api/apiario.md create mode 100644 docs/api/swagger,yaml diff --git a/docs/api/apiario.md b/docs/api/apiario.md new file mode 100644 index 00000000..d0f49dc5 --- /dev/null +++ b/docs/api/apiario.md @@ -0,0 +1,493 @@ +FORMAT: 1A +HOST: http://localhost/api/v3/ + +# ExaFS API v3 + +ExaFS API allows authorized machines to send commands directly in JSON, without the web forms. +The commands are validated in the same way as normal rules. + +## Authorization [/auth] + ++ Cookies + + x-api-key (string) - API authorization key, generated for machine identified by IPv4 address + +### Get JWT token [GET] + +Machine must get JWT token from the API first, using it's API key. Then the JWT token is used as the x-access-token for authorization of all operations. + ++ Request + + + Headers + + x-api-key: your_api_key + ++ Response 200 (application/json) + + { + "token": "jwt_token_used_for_all_message_auth" + } + + +## Rules Collection [/rules] + +### List all rules [GET] + + ++ Request + + + Headers + + x-access-token: jwt_auth_token + + ++ Response 200 (application/json) + + { + "flowspec_ipv4_rw": [ + { + "action": "QoS 1 Mbps", + "comment": "", + "created": "06/06/2018 13:40", + "dest": "", + "dest_mask": null, + "dest_port": "", + "expires": "06/06/2018 15:40", + "flags": "", + "id": 83, + "packet_len": "", + "protocol": "tcp", + "rstate": "active rule", + "source": "192.168.1.2", + "source_mask": 32, + "source_port": "", + "user": "root@example.com" + } + { + "action": "Accept", + "comment": "", + "created": "06/06/2018 13:40", + "dest": "", + "dest_mask": null, + "dest_port": "", + "expires": "06/06/2018 15:40", + "flags": "PSH", + "id": 78, + "packet_len": "", + "protocol": "tcp", + "rstate": "active rule", + "source": "192.168.1.2", + "source_mask": 32, + "source_port": "", + "user": "root@example.com" + } + ], + "flowspec_ipv6_rw": [], + "rtbh_any_rw": [ + { + "comment": "", + "community": "2852:666", + "created": "06/06/2018 13:40", + "expires": "06/06/2018 15:40", + "id": 5, + "ipv4": "192.168.0.1", + "ipv4_mask": 32, + "ipv6": "", + "ipv6_mask": null, + "rstate": "active rule", + "user": "root@example.com" + } + ], + "flowspec_ipv4_ro": [], + "flowspec_ipv6_ro": [], + "rtbh_any_ro": [] + } + +# IPv4 rules [/rules/ipv4] + +## Create new rule [POST] + +Create new IPv4 rule. +Valid IPv4 address and mask must be provided either for source or for the destination. +The address must be from the addres range of authorized user = machine owner. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + + Body + + { + "action": 2, + "protocol": "tcp", + "source": "192.168.1.2", + "source_mask": 32, + "source_port": "", + "expires": "06/06/2018 15:40", + } + ++ Response 201 (application/json) + + + Body + + { + "message": "IPv4 Rule saved", + "rule": { + "action": "QoS 1 Mbps", + "comment": "", + "created": "2018/06/06 11:40", + "dest": "", + "dest_mask": null, + "dest_port": "", + "expires": "2018/06/06 15:40", + "flags": "", + "id": 86, + "packet_len": "", + "protocol": "tcp", + "rstate": "active rule", + "source": "192.168.1.2", + "source_mask": 32, + "source_port": "", + "user": "root@example.com" + } + } + +## IPv4 rule [/rules/ipv4/{rule_id}] + ++ Parameters + + rule_id (int) - Rule ID + +### Get rule details [GET] + +Get single IPv4 rule. Machine owner must have access rights to selected rule. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + ++ Response 200 (application/json) + + { + "action": "QoS 1 Mbps", + "comment": "", + "created": "2018/06/06 11:40", + "dest": "", + "dest_mask": null, + "dest_port": "", + "expires": "2018/06/06 15:40", + "flags": "", + "id": 86, + "packet_len": "", + "protocol": "tcp", + "rstate": "active rule", + "source": "192.168.1.1", + "source_mask": 32, + "source_port": "", + "user": "root@example.com" + } + + +### Delete rule [DELETE] + +Delete rule. Must be the owner of the record or admin. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + ++ Response 201 (application/json) + + { + "message": "rule deleted" + } + + +## IPv6 rules [/rules/ipv6] + +### Create new rule [POST] + +Create new IPv6 rule. +Valid IPv6 address and mask must be provided either for source or for the destination. +The address must be from the addres range of authorized user = machine owner. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + + Body + + { + "action": 32, + "next_header": "tcp", + "source": "2011:78:1C01:1111::", + "source_mask": 64, + "source_port": "", + "expires": "06/06/2018 15:40" + } + ++ Response 201 (application/json) + + + Body + + { + "message": "IPv6 Rule saved", + "rule": { + "action": "QoS 1 Mbps", + "comment": "", + "created": "2018/06/06 11:40", + "dest": "", + "dest_mask": null, + "dest_port": "", + "expires": "2018/06/06 15:40", + "flags": "", + "id": 86, + "packet_len": "", + "protocol": "tcp", + "rstate": "active rule", + "source": "192.168.1.1", + "source_mask": 32, + "source_port": "", + "user": "root@example.com" + } + } + +## IPv6 rule [/rules/ipv6/{rule_id}] + ++ Parameters + + rule_id (int) - Rule ID + +### Get rule details [GET] + +Get single IPv6 rule. Machine owner must have access rights to selected rule. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + ++ Response 200 (application/json) + + { + "action": "QoS 1 Mbps", + "comment": "", + "created": "2018/06/06 11:40", + "dest": "", + "dest_mask": null, + "dest_port": "", + "expires": "2018/06/06 15:40", + "flags": "", + "id": 86, + "packet_len": "", + "protocol": "tcp", + "rstate": "active rule", + "source": "192.168.1.1", + "source_mask": 32, + "source_port": "", + "user": "root@example.com" + } + + +### Delete rule [DELETE] + +Delete rule. Must be the owner of the record or admin. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + ++ Response 201 (application/json) + + { + "message": "rule deleted" + } + + + +## RTBH rules [/rules/rtbh] + +### Create new rule [POST] + +Create new RTBH rule. +Valid IPv6 or IPv4 address and mask must be provided as the source. +The address must be from the addres range of authorized user = machine owner. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + + Body + + { + "community": 2, + "ipv4": "192.168.2.1", + "ipv4_mask": 32, + "expires": "06/06/2018 15:40" + } + ++ Response 201 (application/json) + + + Body + + { + "message": "RTBH Rule saved", + "rule": { + "comment": "", + "community": "RTBH example", + "created": "2018/06/06 11:40", + "expires": "2018/06/06 15:40", + "id": 4, + "ipv4": "192.168.2.1", + "ipv4_mask": 32, + "ipv6": "", + "ipv6_mask": null, + "rstate": "active rule", + "user": "root@example.org" + } + } + +## RTBH rule [/rules/rtbh/{rule_id}] + ++ Parameters + + rule_id (int) - Rule ID + +### Get rule details [GET] + +Get single RTBH rule. Machine owner must have access rights to selected rule. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + ++ Response 200 (application/json) + + { + "comment": "", + "community": "RTBH example", + "created": "2018/06/06 11:40", + "expires": "2018/06/06 15:40", + "id": 4, + "ipv4": "192.168.2.1", + "ipv4_mask": 32, + "ipv6": "", + "ipv6_mask": null, + "rstate": "active rule", + "user": "root@example.org" + } + + +### Delete rule [DELETE] + +Delete rule. Must be the owner of the record or admin. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + + ++ Response 201 (application/json) + + { + "message": "rule deleted" + } + + + + +## Actions collection [/actions] + +### Get All Actions [GET] + +List all actions for the user / machine owner. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + ++ Response 200 (application/json) + + + Body + + [ + [ + 1, + "QoS 0.1 Mbps" + ], + [ + 2, + "QoS 1 Mbps" + ], + [ + 3, + "QoS 10 Mbps" + ], + [ + 5, + "QoS 100 Mbps" + ], + [ + 6, + "QoS 500 Mbps" + ], + [ + 7, + "Discard" + ], + [ + 8, + "Accept" + ] + ] + +## Communities collection [/communities] + +### Get All Communities [GET] + +List all RTBH communites for the user / machine owner. + ++ Request (application/json) + + + Headers + + x-access-token: jwt_auth_token + ++ Response 200 (application/json) + + + Body + + [ + [ + 4, + "RTBH NIX" + ], + [ + 5, + "RTBH CESNET only" + ], + [ + 8, + "RTBH CESNET + external sites" + ] + ] \ No newline at end of file diff --git a/docs/api/swagger,yaml b/docs/api/swagger,yaml new file mode 100644 index 00000000..716ac188 --- /dev/null +++ b/docs/api/swagger,yaml @@ -0,0 +1,422 @@ +openapi: 3.0.3 +info: + title: ExaFS API v3 + description: ExaFS API allows authorized machines to send commands directly in JSON, without the web forms. + version: 3.0.0 +servers: + - url: http://localhost/api/v3/ + +paths: + /auth: + get: + summary: Get JWT token + description: Obtain a JWT token using the machine's API key. + responses: + '200': + description: Successful token retrieval + content: + application/json: + schema: + type: object + properties: + token: + type: string + description: JWT token used for authorization. + + /rules: + get: + summary: List all rules + description: Retrieve all active rules. + security: + - bearerAuth: [] + responses: + '200': + description: List of rules + content: + application/json: + schema: + type: object + properties: + flowspec_ipv4_rw: + type: array + items: + $ref: '#/components/schemas/Rule' + flowspec_ipv6_rw: + type: array + items: + $ref: '#/components/schemas/Rule' + rtbh_any_rw: + type: array + items: + $ref: '#/components/schemas/RTBHRule' + flowspec_ipv4_ro: + type: array + items: + $ref: '#/components/schemas/Rule' + flowspec_ipv6_ro: + type: array + items: + $ref: '#/components/schemas/Rule' + rtbh_any_ro: + type: array + items: + $ref: '#/components/schemas/RTBHRule' + + /rules/ipv4: + post: + summary: Create new IPv4 rule + description: Add a new IPv4 rule. A valid IPv4 address and mask must be provided for the source or destination. + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/IPv4RuleRequest' + responses: + '201': + description: Rule successfully created + content: + application/json: + schema: + $ref: '#/components/schemas/IPv4RuleResponse' + + /rules/ipv4/{rule_id}: + get: + summary: Get IPv4 rule details + description: Retrieve details of a single IPv4 rule. + parameters: + - name: rule_id + in: path + required: true + schema: + type: integer + security: + - bearerAuth: [] + responses: + '200': + description: Rule details retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/Rule' + + delete: + summary: Delete IPv4 rule + description: Delete a specific IPv4 rule. Must be the owner or admin. + parameters: + - name: rule_id + in: path + required: true + schema: + type: integer + security: + - bearerAuth: [] + responses: + '201': + description: Rule deleted + content: + application/json: + schema: + type: object + properties: + message: + type: string + + /rules/ipv6: + post: + summary: Create new IPv6 rule + description: Add a new IPv6 rule. A valid IPv6 address and mask must be provided for the source or destination. + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/IPv6RuleRequest' + responses: + '201': + description: Rule successfully created + content: + application/json: + schema: + $ref: '#/components/schemas/IPv6RuleResponse' + + /rules/ipv6/{rule_id}: + get: + summary: Get IPv6 rule details + description: Retrieve details of a single IPv6 rule. + parameters: + - name: rule_id + in: path + required: true + schema: + type: integer + security: + - bearerAuth: [] + responses: + '200': + description: Rule details retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/Rule' + + delete: + summary: Delete IPv6 rule + description: Delete a specific IPv6 rule. Must be the owner or admin. + parameters: + - name: rule_id + in: path + required: true + schema: + type: integer + security: + - bearerAuth: [] + responses: + '201': + description: Rule deleted + content: + application/json: + schema: + type: object + properties: + message: + type: string + + /rules/rtbh: + post: + summary: Create new RTBH rule + description: Add a new RTBH rule. A valid IPv4 or IPv6 address and mask must be provided as the source. + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/RTBHRuleRequest' + responses: + '201': + description: Rule successfully created + content: + application/json: + schema: + $ref: '#/components/schemas/RTBHRuleResponse' + + /rules/rtbh/{rule_id}: + get: + summary: Get RTBH rule details + description: Retrieve details of a single RTBH rule. + parameters: + - name: rule_id + in: path + required: true + schema: + type: integer + security: + - bearerAuth: [] + responses: + '200': + description: Rule details retrieved + content: + application/json: + schema: + $ref: '#/components/schemas/RTBHRule' + + delete: + summary: Delete RTBH rule + description: Delete a specific RTBH rule. Must be the owner or admin. + parameters: + - name: rule_id + in: path + required: true + schema: + type: integer + security: + - bearerAuth: [] + responses: + '201': + description: Rule deleted + content: + application/json: + schema: + type: object + properties: + message: + type: string + + /actions: + get: + summary: Get all actions + description: Retrieve a list of all available actions for the user/machine owner. + security: + - bearerAuth: [] + responses: + '200': + description: List of actions + content: + application/json: + schema: + type: array + items: + type: array + items: + type: string + + /communities: + get: + summary: Get all communities + description: Retrieve a list of all RTBH communities for the user/machine owner. + security: + - bearerAuth: [] + responses: + '200': + description: List of communities + content: + application/json: + schema: + type: array + items: + type: array + items: + type: string + +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + + schemas: + Rule: + type: object + properties: + action: + type: string + comment: + type: string + created: + type: string + format: date-time + dest: + type: string + dest_mask: + type: integer + nullable: true + dest_port: + type: string + expires: + type: string + format: date-time + flags: + type: string + id: + type: integer + packet_len: + type: string + protocol: + type: string + rstate: + type: string + source: + type: string + source_mask: + type: integer + source_port: + type: string + user: + type: string + + IPv4RuleRequest: + type: object + properties: + action: + type: integer + protocol: + type: string + source: + type: string + source_mask: + type: integer + source_port: + type: string + expires: + type: string + format: date-time + + IPv4RuleResponse: + type: object + properties: + message: + type: string + rule: + $ref: '#/components/schemas/Rule' + + IPv6RuleRequest: + type: object + properties: + action: + type: integer + next_header: + type: string + source: + type: string + source_mask: + type: integer + source_port: + type: string + expires: + type: string + format: date-time + + IPv6RuleResponse: + type: object + properties: + message: + type: string + rule: + $ref: '#/components/schemas/Rule' + + RTBHRuleRequest: + type: object + properties: + community: + type: integer + ipv4: + type: string + ipv4_mask: + type: integer + expires: + type: string + format: date-time + + RTBHRuleResponse: + type: object + properties: + message: + type: string + rule: + $ref: '#/components/schemas/RTBHRule' + + RTBHRule: + type: object + properties: + created: + type: string + format: date-time + expires: + type: string + format: date-time + ipv4: + type: string + ipv4_mask: + type: integer + ipv6: + type: string + ipv6_mask: + type: integer + rstate: + type: string From e304ed2d7f66cd22e583871fc669112bfd967494 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 3 Dec 2024 11:04:28 +0100 Subject: [PATCH 44/54] add Swagger api docs to app, updated requirements add flasgger --- docs/API.md | 7 +- docs/AUTH.md | 3 +- docs/api/swagger,yaml | 422 -------------------------------- docs/api/swagger.yaml | 481 +++++++++++++++++++++++++++++++++++++ flowapp/__init__.py | 5 + flowapp/static/swagger.yml | 481 +++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 7 files changed, 973 insertions(+), 427 deletions(-) delete mode 100644 docs/api/swagger,yaml create mode 100644 docs/api/swagger.yaml create mode 100644 flowapp/static/swagger.yml diff --git a/docs/API.md b/docs/API.md index 3549dae7..9dbb44f7 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,4 +1,5 @@ -# ExaFS tool -## API +# ExaFS API -ExaFS API documentation can be [found on Apiary.io](https://exafs.docs.apiary.io/#). \ No newline at end of file +Local Swagger API docs is available on https://your.server.name/apidocs + +ExaFS API documentation can be also [found on Apiary.io](https://exafs.docs.apiary.io/#). \ No newline at end of file diff --git a/docs/AUTH.md b/docs/AUTH.md index fbea38c4..e5c6c93c 100644 --- a/docs/AUTH.md +++ b/docs/AUTH.md @@ -1,5 +1,4 @@ -# ExaFS tool -## Auth mechanism +# ExaFS Auth mechanism Since version 0.7.3, the application supports three different forms of user authorization. diff --git a/docs/api/swagger,yaml b/docs/api/swagger,yaml deleted file mode 100644 index 716ac188..00000000 --- a/docs/api/swagger,yaml +++ /dev/null @@ -1,422 +0,0 @@ -openapi: 3.0.3 -info: - title: ExaFS API v3 - description: ExaFS API allows authorized machines to send commands directly in JSON, without the web forms. - version: 3.0.0 -servers: - - url: http://localhost/api/v3/ - -paths: - /auth: - get: - summary: Get JWT token - description: Obtain a JWT token using the machine's API key. - responses: - '200': - description: Successful token retrieval - content: - application/json: - schema: - type: object - properties: - token: - type: string - description: JWT token used for authorization. - - /rules: - get: - summary: List all rules - description: Retrieve all active rules. - security: - - bearerAuth: [] - responses: - '200': - description: List of rules - content: - application/json: - schema: - type: object - properties: - flowspec_ipv4_rw: - type: array - items: - $ref: '#/components/schemas/Rule' - flowspec_ipv6_rw: - type: array - items: - $ref: '#/components/schemas/Rule' - rtbh_any_rw: - type: array - items: - $ref: '#/components/schemas/RTBHRule' - flowspec_ipv4_ro: - type: array - items: - $ref: '#/components/schemas/Rule' - flowspec_ipv6_ro: - type: array - items: - $ref: '#/components/schemas/Rule' - rtbh_any_ro: - type: array - items: - $ref: '#/components/schemas/RTBHRule' - - /rules/ipv4: - post: - summary: Create new IPv4 rule - description: Add a new IPv4 rule. A valid IPv4 address and mask must be provided for the source or destination. - security: - - bearerAuth: [] - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/IPv4RuleRequest' - responses: - '201': - description: Rule successfully created - content: - application/json: - schema: - $ref: '#/components/schemas/IPv4RuleResponse' - - /rules/ipv4/{rule_id}: - get: - summary: Get IPv4 rule details - description: Retrieve details of a single IPv4 rule. - parameters: - - name: rule_id - in: path - required: true - schema: - type: integer - security: - - bearerAuth: [] - responses: - '200': - description: Rule details retrieved - content: - application/json: - schema: - $ref: '#/components/schemas/Rule' - - delete: - summary: Delete IPv4 rule - description: Delete a specific IPv4 rule. Must be the owner or admin. - parameters: - - name: rule_id - in: path - required: true - schema: - type: integer - security: - - bearerAuth: [] - responses: - '201': - description: Rule deleted - content: - application/json: - schema: - type: object - properties: - message: - type: string - - /rules/ipv6: - post: - summary: Create new IPv6 rule - description: Add a new IPv6 rule. A valid IPv6 address and mask must be provided for the source or destination. - security: - - bearerAuth: [] - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/IPv6RuleRequest' - responses: - '201': - description: Rule successfully created - content: - application/json: - schema: - $ref: '#/components/schemas/IPv6RuleResponse' - - /rules/ipv6/{rule_id}: - get: - summary: Get IPv6 rule details - description: Retrieve details of a single IPv6 rule. - parameters: - - name: rule_id - in: path - required: true - schema: - type: integer - security: - - bearerAuth: [] - responses: - '200': - description: Rule details retrieved - content: - application/json: - schema: - $ref: '#/components/schemas/Rule' - - delete: - summary: Delete IPv6 rule - description: Delete a specific IPv6 rule. Must be the owner or admin. - parameters: - - name: rule_id - in: path - required: true - schema: - type: integer - security: - - bearerAuth: [] - responses: - '201': - description: Rule deleted - content: - application/json: - schema: - type: object - properties: - message: - type: string - - /rules/rtbh: - post: - summary: Create new RTBH rule - description: Add a new RTBH rule. A valid IPv4 or IPv6 address and mask must be provided as the source. - security: - - bearerAuth: [] - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/RTBHRuleRequest' - responses: - '201': - description: Rule successfully created - content: - application/json: - schema: - $ref: '#/components/schemas/RTBHRuleResponse' - - /rules/rtbh/{rule_id}: - get: - summary: Get RTBH rule details - description: Retrieve details of a single RTBH rule. - parameters: - - name: rule_id - in: path - required: true - schema: - type: integer - security: - - bearerAuth: [] - responses: - '200': - description: Rule details retrieved - content: - application/json: - schema: - $ref: '#/components/schemas/RTBHRule' - - delete: - summary: Delete RTBH rule - description: Delete a specific RTBH rule. Must be the owner or admin. - parameters: - - name: rule_id - in: path - required: true - schema: - type: integer - security: - - bearerAuth: [] - responses: - '201': - description: Rule deleted - content: - application/json: - schema: - type: object - properties: - message: - type: string - - /actions: - get: - summary: Get all actions - description: Retrieve a list of all available actions for the user/machine owner. - security: - - bearerAuth: [] - responses: - '200': - description: List of actions - content: - application/json: - schema: - type: array - items: - type: array - items: - type: string - - /communities: - get: - summary: Get all communities - description: Retrieve a list of all RTBH communities for the user/machine owner. - security: - - bearerAuth: [] - responses: - '200': - description: List of communities - content: - application/json: - schema: - type: array - items: - type: array - items: - type: string - -components: - securitySchemes: - bearerAuth: - type: http - scheme: bearer - - schemas: - Rule: - type: object - properties: - action: - type: string - comment: - type: string - created: - type: string - format: date-time - dest: - type: string - dest_mask: - type: integer - nullable: true - dest_port: - type: string - expires: - type: string - format: date-time - flags: - type: string - id: - type: integer - packet_len: - type: string - protocol: - type: string - rstate: - type: string - source: - type: string - source_mask: - type: integer - source_port: - type: string - user: - type: string - - IPv4RuleRequest: - type: object - properties: - action: - type: integer - protocol: - type: string - source: - type: string - source_mask: - type: integer - source_port: - type: string - expires: - type: string - format: date-time - - IPv4RuleResponse: - type: object - properties: - message: - type: string - rule: - $ref: '#/components/schemas/Rule' - - IPv6RuleRequest: - type: object - properties: - action: - type: integer - next_header: - type: string - source: - type: string - source_mask: - type: integer - source_port: - type: string - expires: - type: string - format: date-time - - IPv6RuleResponse: - type: object - properties: - message: - type: string - rule: - $ref: '#/components/schemas/Rule' - - RTBHRuleRequest: - type: object - properties: - community: - type: integer - ipv4: - type: string - ipv4_mask: - type: integer - expires: - type: string - format: date-time - - RTBHRuleResponse: - type: object - properties: - message: - type: string - rule: - $ref: '#/components/schemas/RTBHRule' - - RTBHRule: - type: object - properties: - created: - type: string - format: date-time - expires: - type: string - format: date-time - ipv4: - type: string - ipv4_mask: - type: integer - ipv6: - type: string - ipv6_mask: - type: integer - rstate: - type: string diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml new file mode 100644 index 00000000..b0d945a4 --- /dev/null +++ b/docs/api/swagger.yaml @@ -0,0 +1,481 @@ +swagger: '2.0' +info: + title: ExaFS API + version: '3.0' + description: ExaFS API allows authorized machines to send commands directly in JSON, without the web forms. The commands are validated in the same way as normal rules. +securityDefinitions: + ApiKeyAuth: + type: apiKey + in: header + name: x-api-key + description: API key for initial authentication + TokenAuth: + type: apiKey + in: header + name: x-access-token + description: auth token received from /auth endpoint +security: + - TokenAuth: [] + +tags: + - name: Authorization + description: Endpoints for obtaining and managing API tokens. + - name: Rules + description: Endpoints for managing IPv4, IPv6, and RTBH rules. + - name: Choices + description: Choices for rule actions and communities. + + +paths: + /auth: + get: + tags: + - Authorization + security: + - ApiKeyAuth: [] + summary: Authenticate and get JWT token + description: Generate API Key for the logged user using PyJWT + responses: + '200': + description: Successfully authenticated + schema: + type: object + properties: + token: + type: string + description: JWT token to be used in subsequent requests + '401': + description: Authentication failed - token expired + '403': + description: Authentication failed - token invalid + + /rules: + get: + tags: + - Rules + summary: Get all rules + description: Returns all flow rules accessible to the authenticated user + parameters: + - name: time_format + in: query + type: string + required: false + description: Preferred time format for dates in response + responses: + '200': + description: List of all rules + schema: + type: object + properties: + flowspec_ipv4_rw: + type: array + items: + $ref: '#/definitions/IPv4Rule' + flowspec_ipv6_rw: + type: array + items: + $ref: '#/definitions/IPv6Rule' + rtbh_any_rw: + type: array + items: + $ref: '#/definitions/RTBHRule' + flowspec_ipv4_ro: + type: array + items: + $ref: '#/definitions/IPv4Rule' + flowspec_ipv6_ro: + type: array + items: + $ref: '#/definitions/IPv6Rule' + + + /rules/ipv4: + post: + tags: + - Rules + summary: Create IPv4 rule + description: Create a new IPv4 flow rule + parameters: + - name: rule + in: body + required: true + schema: + $ref: '#/definitions/IPv4RuleInput' + responses: + '201': + description: Rule created successfully + schema: + type: object + properties: + message: + type: string + rule: + $ref: '#/definitions/IPv4Rule' + '400': + description: Invalid input data + '403': + description: Rule limit reached or read-only token + + /rules/ipv6: + post: + tags: + - Rules + summary: Create IPv6 rule + description: Create a new IPv6 flow rule + parameters: + - name: rule + in: body + required: true + schema: + $ref: '#/definitions/IPv6RuleInput' + responses: + '201': + description: Rule created successfully + schema: + type: object + properties: + message: + type: string + rule: + $ref: '#/definitions/IPv6Rule' + '400': + description: Invalid input data + '403': + description: Rule limit reached or read-only token + + /rules/rtbh: + post: + tags: + - Rules + summary: Create RTBH rule + description: Create a new RTBH rule + parameters: + - name: rule + in: body + required: true + schema: + $ref: '#/definitions/RTBHRuleInput' + responses: + '201': + description: Rule created successfully + schema: + type: object + properties: + message: + type: string + rule: + $ref: '#/definitions/RTBHRule' + '400': + description: Invalid input data + '403': + description: Rule limit reached or read-only token + + /rules/ipv4/{rule_id}: + parameters: + - name: rule_id + in: path + required: true + type: integer + description: ID of the IPv4 rule + get: + tags: + - Rules + summary: Get IPv4 rule + description: Get details of a specific IPv4 rule + responses: + '200': + description: Rule details + schema: + $ref: '#/definitions/IPv4Rule' + '401': + description: Not allowed to view this rule + '404': + description: Rule not found + delete: + tags: + - Rules + summary: Delete IPv4 rule + description: Delete a specific IPv4 rule + responses: + '201': + description: Rule deleted successfully + '401': + description: Not allowed to delete this rule + '403': + description: Read-only token + '404': + description: Rule not found + + /rules/ipv6/{rule_id}: + parameters: + - name: rule_id + in: path + required: true + type: integer + description: ID of the IPv6 rule + get: + tags: + - Rules + summary: Get IPv6 rule + description: Get details of a specific IPv6 rule + responses: + '200': + description: Rule details + schema: + $ref: '#/definitions/IPv6Rule' + '401': + description: Not allowed to view this rule + '404': + description: Rule not found + delete: + tags: + - Rules + summary: Delete IPv6 rule + description: Delete a specific IPv6 rule + responses: + '201': + description: Rule deleted successfully + '401': + description: Not allowed to delete this rule + '403': + description: Read-only token + '404': + description: Rule not found + + /rules/rtbh/{rule_id}: + parameters: + - name: rule_id + in: path + required: true + type: integer + description: ID of the RTBH rule + get: + tags: + - Rules + summary: Get RTBH rule + description: Get details of a specific RTBH rule + responses: + '200': + description: Rule details + schema: + $ref: '#/definitions/RTBHRule' + '401': + description: Not allowed to view this rule + '404': + description: Rule not found + delete: + tags: + - Rules + summary: Delete RTBH rule + description: Delete a specific RTBH rule + responses: + '201': + description: Rule deleted successfully + '401': + description: Not allowed to delete this rule + '403': + description: Read-only token + '404': + description: Rule not found + + /actions: + get: + tags: + - Choices + summary: Get available actions + description: Returns actions allowed for current user + responses: + '200': + description: List of available actions + '404': + description: No actions found for user + + /communities: + get: + tags: + - Choices + summary: Get available communities + description: Returns RTBH communities allowed for current user + responses: + '200': + description: List of available communities + '404': + description: No communities found for user + + +definitions: + IPv4RuleInput: + type: object + required: + - source + - source_mask + - dest + - dest_mask + - expires + - action + properties: + source: + type: string + description: Source IP address + source_mask: + type: integer + description: Source network mask + source_port: + type: string + description: Source port(s) + dest: + type: string + description: Destination IP address + dest_mask: + type: integer + description: Destination network mask + destination_port: + type: string + description: Destination port(s) + protocol: + type: string + description: Protocol + flags: + type: array + items: + type: string + description: TCP flags + packet_len: + type: string + description: Packet length + fragment: + type: array + items: + type: string + description: Fragment types + expires: + type: string + format: date-time + description: Rule expiration time + comment: + type: string + description: Rule comment + action: + type: integer + description: Action ID + + IPv4Rule: + allOf: + - $ref: '#/definitions/IPv4RuleInput' + - type: object + properties: + id: + type: integer + rstate_id: + type: integer + user_id: + type: integer + org_id: + type: integer + + IPv6RuleInput: + type: object + required: + - source + - source_mask + - dest + - dest_mask + - expires + - action + properties: + source: + type: string + description: Source IPv6 address + source_mask: + type: integer + description: Source network mask + source_port: + type: string + description: Source port(s) + dest: + type: string + description: Destination IPv6 address + dest_mask: + type: integer + description: Destination network mask + destination_port: + type: string + description: Destination port(s) + next_header: + type: string + description: Next header + flags: + type: array + items: + type: string + description: TCP flags + packet_len: + type: string + description: Packet length + expires: + type: string + format: date-time + description: Rule expiration time + comment: + type: string + description: Rule comment + action: + type: integer + description: Action ID + + IPv6Rule: + allOf: + - $ref: '#/definitions/IPv6RuleInput' + - type: object + properties: + id: + type: integer + rstate_id: + type: integer + user_id: + type: integer + org_id: + type: integer + + RTBHRuleInput: + type: object + required: + - expires + - community + properties: + ipv4: + type: string + description: IPv4 address + ipv4_mask: + type: integer + description: IPv4 network mask + ipv6: + type: string + description: IPv6 address + ipv6_mask: + type: integer + description: IPv6 network mask + community: + type: integer + description: Community ID + expires: + type: string + format: date-time + description: Rule expiration time + comment: + type: string + description: Rule comment + + RTBHRule: + allOf: + - $ref: '#/definitions/RTBHRuleInput' + - type: object + properties: + id: + type: integer + rstate_id: + type: integer + user_id: + type: integer + org_id: + type: integer \ No newline at end of file diff --git a/flowapp/__init__.py b/flowapp/__init__.py index 411fe71c..8dd3d8b1 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -10,6 +10,7 @@ from flask_wtf.csrf import CSRFProtect from flask_migrate import Migrate from flask_session import Session +from flasgger import Swagger from .__about__ import __version__ from .instance_config import InstanceConfig @@ -20,6 +21,7 @@ csrf = CSRFProtect() ext = SSO() sess = Session() +swagger = Swagger(template_file="static/swagger.yml") class InterceptHandler(logging.Handler): @@ -53,6 +55,9 @@ def create_app(config_object=None): # Init SSO ext.init_app(app) + # Init swagger + swagger.init_app(app) + from flowapp import models, constants, validators from .views.admin import admin from .views.rules import rules diff --git a/flowapp/static/swagger.yml b/flowapp/static/swagger.yml new file mode 100644 index 00000000..b0d945a4 --- /dev/null +++ b/flowapp/static/swagger.yml @@ -0,0 +1,481 @@ +swagger: '2.0' +info: + title: ExaFS API + version: '3.0' + description: ExaFS API allows authorized machines to send commands directly in JSON, without the web forms. The commands are validated in the same way as normal rules. +securityDefinitions: + ApiKeyAuth: + type: apiKey + in: header + name: x-api-key + description: API key for initial authentication + TokenAuth: + type: apiKey + in: header + name: x-access-token + description: auth token received from /auth endpoint +security: + - TokenAuth: [] + +tags: + - name: Authorization + description: Endpoints for obtaining and managing API tokens. + - name: Rules + description: Endpoints for managing IPv4, IPv6, and RTBH rules. + - name: Choices + description: Choices for rule actions and communities. + + +paths: + /auth: + get: + tags: + - Authorization + security: + - ApiKeyAuth: [] + summary: Authenticate and get JWT token + description: Generate API Key for the logged user using PyJWT + responses: + '200': + description: Successfully authenticated + schema: + type: object + properties: + token: + type: string + description: JWT token to be used in subsequent requests + '401': + description: Authentication failed - token expired + '403': + description: Authentication failed - token invalid + + /rules: + get: + tags: + - Rules + summary: Get all rules + description: Returns all flow rules accessible to the authenticated user + parameters: + - name: time_format + in: query + type: string + required: false + description: Preferred time format for dates in response + responses: + '200': + description: List of all rules + schema: + type: object + properties: + flowspec_ipv4_rw: + type: array + items: + $ref: '#/definitions/IPv4Rule' + flowspec_ipv6_rw: + type: array + items: + $ref: '#/definitions/IPv6Rule' + rtbh_any_rw: + type: array + items: + $ref: '#/definitions/RTBHRule' + flowspec_ipv4_ro: + type: array + items: + $ref: '#/definitions/IPv4Rule' + flowspec_ipv6_ro: + type: array + items: + $ref: '#/definitions/IPv6Rule' + + + /rules/ipv4: + post: + tags: + - Rules + summary: Create IPv4 rule + description: Create a new IPv4 flow rule + parameters: + - name: rule + in: body + required: true + schema: + $ref: '#/definitions/IPv4RuleInput' + responses: + '201': + description: Rule created successfully + schema: + type: object + properties: + message: + type: string + rule: + $ref: '#/definitions/IPv4Rule' + '400': + description: Invalid input data + '403': + description: Rule limit reached or read-only token + + /rules/ipv6: + post: + tags: + - Rules + summary: Create IPv6 rule + description: Create a new IPv6 flow rule + parameters: + - name: rule + in: body + required: true + schema: + $ref: '#/definitions/IPv6RuleInput' + responses: + '201': + description: Rule created successfully + schema: + type: object + properties: + message: + type: string + rule: + $ref: '#/definitions/IPv6Rule' + '400': + description: Invalid input data + '403': + description: Rule limit reached or read-only token + + /rules/rtbh: + post: + tags: + - Rules + summary: Create RTBH rule + description: Create a new RTBH rule + parameters: + - name: rule + in: body + required: true + schema: + $ref: '#/definitions/RTBHRuleInput' + responses: + '201': + description: Rule created successfully + schema: + type: object + properties: + message: + type: string + rule: + $ref: '#/definitions/RTBHRule' + '400': + description: Invalid input data + '403': + description: Rule limit reached or read-only token + + /rules/ipv4/{rule_id}: + parameters: + - name: rule_id + in: path + required: true + type: integer + description: ID of the IPv4 rule + get: + tags: + - Rules + summary: Get IPv4 rule + description: Get details of a specific IPv4 rule + responses: + '200': + description: Rule details + schema: + $ref: '#/definitions/IPv4Rule' + '401': + description: Not allowed to view this rule + '404': + description: Rule not found + delete: + tags: + - Rules + summary: Delete IPv4 rule + description: Delete a specific IPv4 rule + responses: + '201': + description: Rule deleted successfully + '401': + description: Not allowed to delete this rule + '403': + description: Read-only token + '404': + description: Rule not found + + /rules/ipv6/{rule_id}: + parameters: + - name: rule_id + in: path + required: true + type: integer + description: ID of the IPv6 rule + get: + tags: + - Rules + summary: Get IPv6 rule + description: Get details of a specific IPv6 rule + responses: + '200': + description: Rule details + schema: + $ref: '#/definitions/IPv6Rule' + '401': + description: Not allowed to view this rule + '404': + description: Rule not found + delete: + tags: + - Rules + summary: Delete IPv6 rule + description: Delete a specific IPv6 rule + responses: + '201': + description: Rule deleted successfully + '401': + description: Not allowed to delete this rule + '403': + description: Read-only token + '404': + description: Rule not found + + /rules/rtbh/{rule_id}: + parameters: + - name: rule_id + in: path + required: true + type: integer + description: ID of the RTBH rule + get: + tags: + - Rules + summary: Get RTBH rule + description: Get details of a specific RTBH rule + responses: + '200': + description: Rule details + schema: + $ref: '#/definitions/RTBHRule' + '401': + description: Not allowed to view this rule + '404': + description: Rule not found + delete: + tags: + - Rules + summary: Delete RTBH rule + description: Delete a specific RTBH rule + responses: + '201': + description: Rule deleted successfully + '401': + description: Not allowed to delete this rule + '403': + description: Read-only token + '404': + description: Rule not found + + /actions: + get: + tags: + - Choices + summary: Get available actions + description: Returns actions allowed for current user + responses: + '200': + description: List of available actions + '404': + description: No actions found for user + + /communities: + get: + tags: + - Choices + summary: Get available communities + description: Returns RTBH communities allowed for current user + responses: + '200': + description: List of available communities + '404': + description: No communities found for user + + +definitions: + IPv4RuleInput: + type: object + required: + - source + - source_mask + - dest + - dest_mask + - expires + - action + properties: + source: + type: string + description: Source IP address + source_mask: + type: integer + description: Source network mask + source_port: + type: string + description: Source port(s) + dest: + type: string + description: Destination IP address + dest_mask: + type: integer + description: Destination network mask + destination_port: + type: string + description: Destination port(s) + protocol: + type: string + description: Protocol + flags: + type: array + items: + type: string + description: TCP flags + packet_len: + type: string + description: Packet length + fragment: + type: array + items: + type: string + description: Fragment types + expires: + type: string + format: date-time + description: Rule expiration time + comment: + type: string + description: Rule comment + action: + type: integer + description: Action ID + + IPv4Rule: + allOf: + - $ref: '#/definitions/IPv4RuleInput' + - type: object + properties: + id: + type: integer + rstate_id: + type: integer + user_id: + type: integer + org_id: + type: integer + + IPv6RuleInput: + type: object + required: + - source + - source_mask + - dest + - dest_mask + - expires + - action + properties: + source: + type: string + description: Source IPv6 address + source_mask: + type: integer + description: Source network mask + source_port: + type: string + description: Source port(s) + dest: + type: string + description: Destination IPv6 address + dest_mask: + type: integer + description: Destination network mask + destination_port: + type: string + description: Destination port(s) + next_header: + type: string + description: Next header + flags: + type: array + items: + type: string + description: TCP flags + packet_len: + type: string + description: Packet length + expires: + type: string + format: date-time + description: Rule expiration time + comment: + type: string + description: Rule comment + action: + type: integer + description: Action ID + + IPv6Rule: + allOf: + - $ref: '#/definitions/IPv6RuleInput' + - type: object + properties: + id: + type: integer + rstate_id: + type: integer + user_id: + type: integer + org_id: + type: integer + + RTBHRuleInput: + type: object + required: + - expires + - community + properties: + ipv4: + type: string + description: IPv4 address + ipv4_mask: + type: integer + description: IPv4 network mask + ipv6: + type: string + description: IPv6 address + ipv6_mask: + type: integer + description: IPv6 network mask + community: + type: integer + description: Community ID + expires: + type: string + format: date-time + description: Rule expiration time + comment: + type: string + description: Rule comment + + RTBHRule: + allOf: + - $ref: '#/definitions/RTBHRuleInput' + - type: object + properties: + id: + type: integer + rstate_id: + type: integer + user_id: + type: integer + org_id: + type: integer \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e26e9e9a..b3ccf950 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ email_validator>=1.1 pika>=1.3.0 mysqlclient>=2.0.0 loguru +flasgger From 583870a800861977b1ff192d3875f49b14f7bc86 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 16 Dec 2024 13:45:43 +0100 Subject: [PATCH 45/54] typo in admin bulk insert users --- flowapp/views/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index b8a899cb..6ea27b1b 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -236,14 +236,14 @@ def bulk_import_users_save(): if request.method == "POST" and form.validate(): # Get CSV data from textarea csv_data = form.users.data - # Parse CSV data csv_reader = csv.DictReader(StringIO(csv_data), delimiter=",") errored = False for row in csv_reader: + print(row) try: # Extract and prepare data - uuid = row["uuid/eppn"] + uuid = row["uuid-eppn"] name = row["name"] phone = row["telefon"] email = row["email"] From f963b48c0472b21b09462f22b64e9bfe8c8c01a1 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 16 Dec 2024 14:24:19 +0100 Subject: [PATCH 46/54] minor update bulk user form help text --- flowapp/templates/forms/bulk_user_form.html | 15 +++++++++++---- flowapp/views/admin.py | 6 ++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/flowapp/templates/forms/bulk_user_form.html b/flowapp/templates/forms/bulk_user_form.html index 34f8e2e6..c41389ea 100644 --- a/flowapp/templates/forms/bulk_user_form.html +++ b/flowapp/templates/forms/bulk_user_form.html @@ -38,14 +38,21 @@

    Create multiple users.

    Example CSV data

    +

    CSV data must contain header row when posting

                 
-uuid-eppn,name,telefon,email,role,organizace
-test@example.com,Test 1,123,test@example.com,1,1
-test-bad@example.com,Test 1,123,test-bad@example.com,4,4
-test2@example.com,Test 1,123456,test2@example.com,2,1
+uuid-eppn,name,telefon,email,role,organizace,poznamka
+view@example.com,Test View,123,view@example.com,1,1,View
+user@example.com,Test User,123456,user@example.com,2,1,User
+admin@example.com,Test Admin,+420 111 111 111,admin@example.com,3,1,Admin
    +

    Role

    +
      +
    • 1 - View
      • +
    • 2 - User
      • +
    • 3 - Admin
      • +
    diff --git a/flowapp/views/admin.py b/flowapp/views/admin.py index 6ea27b1b..d014c87c 100644 --- a/flowapp/views/admin.py +++ b/flowapp/views/admin.py @@ -240,7 +240,6 @@ def bulk_import_users_save(): csv_reader = csv.DictReader(StringIO(csv_data), delimiter=",") errored = False for row in csv_reader: - print(row) try: # Extract and prepare data uuid = row["uuid-eppn"] @@ -251,9 +250,12 @@ def bulk_import_users_save(): # Convert role and organization fields to lists of integers role_ids = [int(row["role"])] # role_id should be a list org_ids = [int(row["organizace"])] # org_id should be a list + notice = row["poznamka"] # Insert user - insert_user(uuid=uuid, role_ids=role_ids, org_ids=org_ids, name=name, phone=phone, email=email) + insert_user( + uuid=uuid, role_ids=role_ids, org_ids=org_ids, name=name, phone=phone, email=email, comment=notice + ) except KeyError as e: errored = True # Handle missing fields or other errors in the CSV From 5be0ca725d81eec1af06c810c3a4c67d3740b829 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 9 Jan 2025 12:38:52 +0100 Subject: [PATCH 47/54] readme update for version 1.0.1 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b54b8c1c..6ae3e37c 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ You may also need to monitor the ExaBGP and renew the commands after restart / s * [Local database instalation notes](./docs/DB_LOCAL.md) ## Change Log -- 1.0.0 . DRAFT - ExaAPI and Guarda modules moved outside of the project to their own repositories. ExaAPI is now available also as a [pip package exabgp-process](https://pypi.org/project/exabgp-process/). New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. +- 1.0.1 . minor bug fixes +- 1.0.0 . ExaAPI and Guarda modules moved outside of the project to their own repositories. ExaAPI is now available also as a [pip package exabgp-process](https://pypi.org/project/exabgp-process/). New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. There are now limits for rules for organization and overall limit for the instalation. Database changed / migration is required. - 0.8.1 application is using Flask-Session stored in DB using SQL Alchemy driver. This can be configured for other drivers, however server side session is required for the application proper function. - 0.8.0 - API keys update. **Run migration scripts to update your DB**. Keys can now have expiration date and readonly flag. Admin can create special keys for certain machinnes. - 0.7.3 - New possibility of external auth proxy. From 1e058466889915bb8a10509be9cc70619a807076 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Fri, 10 Jan 2025 09:41:50 +0100 Subject: [PATCH 48/54] update requirements - app is now Flask3 compliant --- requirements.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index b3ccf950..ea233782 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Flask<3 +Flask>=2.0.2 Flask-SQLAlchemy>=2.2 Flask-SSO>=0.4.0 Flask-WTF>=1.0.0 @@ -10,8 +10,9 @@ PyMySQL>=1.0.0 pytest>=7.0.0 requests>=2.20.0 babel>=2.7.0 +mysqlclient>=2.0.0 email_validator>=1.1 pika>=1.3.0 -mysqlclient>=2.0.0 +gunicorn>=22.0.0 loguru -flasgger +flasgger \ No newline at end of file From cc522d88fa03a650fc3d551731a00760e0bfed85 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Fri, 31 Jan 2025 14:02:05 +0100 Subject: [PATCH 49/54] add python-dotenv to requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ea233782..d517bf6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,5 @@ email_validator>=1.1 pika>=1.3.0 gunicorn>=22.0.0 loguru -flasgger \ No newline at end of file +flasgger +python-dotenv \ No newline at end of file From 60f716b36cea72f588a910ca806a4bd88829afca Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Thu, 6 Feb 2025 12:48:56 +0100 Subject: [PATCH 50/54] update create_app and config if the app is behind reverse proxy --- config.example.py | 3 +++ flowapp/__init__.py | 6 ++++++ requirements.txt | 2 -- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config.example.py b/config.example.py index 2d4d4a29..32b3efd4 100644 --- a/config.example.py +++ b/config.example.py @@ -79,6 +79,9 @@ class ProductionConfig(Config): DEBUG = False DEVEL = False + # is production behind a reverse proxy? + BEHIND_PROXY = True + # Set cookie behavior SESSION_COOKIE_SECURE = (True,) SESSION_COOKIE_HTTPONLY = (True,) diff --git a/flowapp/__init__.py b/flowapp/__init__.py index 8dd3d8b1..f029c37d 100644 --- a/flowapp/__init__.py +++ b/flowapp/__init__.py @@ -11,6 +11,8 @@ from flask_migrate import Migrate from flask_session import Session from flasgger import Swagger +from werkzeug.middleware.proxy_fix import ProxyFix + from .__about__ import __version__ from .instance_config import InstanceConfig @@ -58,6 +60,10 @@ def create_app(config_object=None): # Init swagger swagger.init_app(app) + # handle proxy fix + if app.config.get("BEHIND_PROXY", False): + app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1) + from flowapp import models, constants, validators from .views.admin import admin from .views.rules import rules diff --git a/requirements.txt b/requirements.txt index d517bf6a..fb33ad36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,10 +10,8 @@ PyMySQL>=1.0.0 pytest>=7.0.0 requests>=2.20.0 babel>=2.7.0 -mysqlclient>=2.0.0 email_validator>=1.1 pika>=1.3.0 -gunicorn>=22.0.0 loguru flasgger python-dotenv \ No newline at end of file From ec59f035106b89a7cb3231ef1e5e92f42e6e33bd Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 24 Feb 2025 15:04:05 +0100 Subject: [PATCH 51/54] bugfix for IPv6 messages - empty fragment should be empty string not None --- flowapp/__about__.py | 2 +- flowapp/messages.py | 56 +++++++++++--------------------------------- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/flowapp/__about__.py b/flowapp/__about__.py index ca651cc2..d10af327 100755 --- a/flowapp/__about__.py +++ b/flowapp/__about__.py @@ -1 +1 @@ -__version__ = "1.0.1" +__version__ = "1.0.2" diff --git a/flowapp/messages.py b/flowapp/messages.py index 9609eaf4..a708a8f2 100644 --- a/flowapp/messages.py +++ b/flowapp/messages.py @@ -25,11 +25,7 @@ def create_ipv4(rule, message_type=ANNOUNCE): flagstring = rule.flags.replace(";", " ") if rule.flags else "" - flags = ( - "tcp-flags {};".format(flagstring) - if rule.flags and rule.protocol == "tcp" - else "" - ) + flags = "tcp-flags {};".format(flagstring) if rule.flags and rule.protocol == "tcp" else "" fragment_string = rule.fragment.replace(";", " ") if rule.fragment else "" fragment = "fragment [ {} ];".format(fragment_string) if rule.fragment else "" @@ -55,11 +51,7 @@ def create_ipv6(rule, message_type=ANNOUNCE): if rule.next_header and rule.next_header != "all": protocol = "next-header ={};".format(IPV6_NEXT_HEADER[rule.next_header]) flagstring = rule.flags.replace(";", " ") - flags = ( - "tcp-flags {};".format(flagstring) - if rule.flags and rule.next_header == "tcp" - else "" - ) + flags = "tcp-flags {};".format(flagstring) if rule.flags and rule.next_header == "tcp" else "" spec = {"protocol": protocol, "mask": IPV6_DEFMASK, "flags": flags} @@ -103,25 +95,17 @@ def create_rtbh(rule, message_type=ANNOUNCE): targets = current_app.config["MULTI_NEIGHBOR"].get(rule.community.comm) else: targets = current_app.config["MULTI_NEIGHBOR"].get("primary") - + neighbor = prepare_multi_neighbor(targets) else: neighbor = "" except KeyError: neighbor = "" - community_string = ( - "community [{}]".format(rule.community.comm) if rule.community.comm else "" - ) - large_community_string = ( - "large-community [{}]".format(rule.community.larcomm) - if rule.community.larcomm - else "" - ) + community_string = "community [{}]".format(rule.community.comm) if rule.community.comm else "" + large_community_string = "large-community [{}]".format(rule.community.larcomm) if rule.community.larcomm else "" extended_community_string = ( - "extended-community [{}]".format(rule.community.extcomm) - if rule.community.extcomm - else "" + "extended-community [{}]".format(rule.community.extcomm) if rule.community.extcomm else "" ) as_path_string = "" @@ -165,27 +149,19 @@ def create_message(rule, ipv_specific, message_type=ANNOUNCE): source = "source {}".format(rule.source) if rule.source else "" source += "/{};".format(smask) if rule.source else "" - source_port = ( - "source-port {};".format(trps(rule.source_port)) if rule.source_port else "" - ) + source_port = "source-port {};".format(trps(rule.source_port)) if rule.source_port else "" dmask = sanitize_mask(rule.dest_mask, ipv_specific["mask"]) dest = " destination {}".format(rule.dest) if rule.dest else "" dest += "/{};".format(dmask) if rule.dest else "" - dest_port = ( - "destination-port {};".format(trps(rule.dest_port)) if rule.dest_port else "" - ) + dest_port = "destination-port {};".format(trps(rule.dest_port)) if rule.dest_port else "" - protocol = ipv_specific["protocol"] - flags = ipv_specific["flags"] - fragment = ipv_specific.get("fragment", None) + protocol = ipv_specific.get("protocol", "") + flags = ipv_specific.get("flags", "") + fragment = ipv_specific.get("fragment", "") - packet_len = ( - "packet-length {};".format(trps(rule.packet_len, MAX_PACKET)) - if rule.packet_len - else "" - ) + packet_len = "packet-length {};".format(trps(rule.packet_len, MAX_PACKET)) if rule.packet_len else "" match_body = "{source} {source_port} {dest} {dest_port} {protocol} {fragment} {flags} {packet_len}".format( source=source, @@ -202,12 +178,8 @@ def create_message(rule, ipv_specific, message_type=ANNOUNCE): try: if current_app.config["USE_RD"]: - rd_string = "route-distinguisher {rd};".format( - rd=current_app.config["RD_STRING"] - ) - rt_string = "extended-community target:{rt};".format( - rt=current_app.config["RT_STRING"] - ) + rd_string = "route-distinguisher {rd};".format(rd=current_app.config["RD_STRING"]) + rt_string = "extended-community target:{rt};".format(rt=current_app.config["RT_STRING"]) else: rd_string = "" rt_string = "" From 2c5304927dfb8ab220715263c35f9f7c3cc70052 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Mon, 24 Feb 2025 15:28:19 +0100 Subject: [PATCH 52/54] match_body composed by list comprehension instead of format, to avoid double spacing --- flowapp/messages.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/flowapp/messages.py b/flowapp/messages.py index a708a8f2..45ab4f3c 100644 --- a/flowapp/messages.py +++ b/flowapp/messages.py @@ -163,16 +163,8 @@ def create_message(rule, ipv_specific, message_type=ANNOUNCE): packet_len = "packet-length {};".format(trps(rule.packet_len, MAX_PACKET)) if rule.packet_len else "" - match_body = "{source} {source_port} {dest} {dest_port} {protocol} {fragment} {flags} {packet_len}".format( - source=source, - source_port=source_port, - dest=dest, - dest_port=dest_port, - protocol=protocol, - fragment=fragment, - flags=flags, - packet_len=packet_len, - ) + values = [source, source_port, dest, dest_port, protocol, fragment, flags, packet_len] + match_body = " ".join(v for v in values if v) command = "{};".format(rule.action.command) From 681cf4f3e914e4118d184c4ddc7299d2124921e9 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Tue, 25 Feb 2025 15:36:03 +0100 Subject: [PATCH 53/54] Bugfix/http api (#48) * fixed json output for route if exabgp process http api is used --------- Co-authored-by: Jiri Vrany --- flowapp/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flowapp/output.py b/flowapp/output.py index e2868277..3dde8221 100644 --- a/flowapp/output.py +++ b/flowapp/output.py @@ -54,7 +54,7 @@ def announce_to_http(route): """ if not current_app.config["TESTING"]: try: - resp = requests.post(current_app.config["EXA_API_URL"], data={"command": route}) + resp = requests.post(current_app.config["EXA_API_URL"], data={"command": json.dumps(route)}) resp.raise_for_status() except requests.exceptions.HTTPError as err: current_app.logger.error("ExaAPI HTTP Error: ", err) From 3ca92db849ff436aa1dc9953ee66d4d51b10bb45 Mon Sep 17 00:00:00 2001 From: Jiri Vrany Date: Wed, 19 Mar 2025 09:51:08 +0100 Subject: [PATCH 54/54] updated docs --- README.md | 11 ++++++++++- docs/DB_MIGRATIONS.md | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 docs/DB_MIGRATIONS.md diff --git a/README.md b/README.md index 6ae3e37c..f0bc9742 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,17 @@ You may also need to monitor the ExaBGP and renew the commands after restart / s * [Local database instalation notes](./docs/DB_LOCAL.md) ## Change Log +- 1.0.2 - fixed bug in IPv6 Flowspec messages - 1.0.1 . minor bug fixes -- 1.0.0 . ExaAPI and Guarda modules moved outside of the project to their own repositories. ExaAPI is now available also as a [pip package exabgp-process](https://pypi.org/project/exabgp-process/). New format of message for ExaAPI - now sends information about user (author of rule) for logging purposes. There are now limits for rules for organization and overall limit for the instalation. Database changed / migration is required. +- 1.0.0 . Major changes + - Limits for nuber of rules in the system introduced. There are now limits for rules for organization and overall limit for the instalation. Database changed / migration is required. Migrating the database to version 1.0.x is a bit more complicated, you need to link existing rules to organizations. [A more detailed description is in a separate document](./docs/DB_MIGRATIONS.md). + - Rules are now tied to organization. If the user belongs to more than one organization, the organization for the session must be selected after login. + - Bulk import for users enabled for admin. + - Introduced Swagger docs for API on the local system. Just open /apidocs url. + - New format of message for ExaAPI - now sends information about author of rule (user) for logging purposes. + - ExaAPI and Guarda modules moved outside of the project. + - ExaAPI is now available as a [pip package exabgp-process](https://pypi.org/project/exabgp-process/), with own [github repostiory](https://github.com/CESNET/exabgp-process). + - Watch of exabgp restart can be still done by guarda service - see docs. Or it can be done by override of the exabgp service settings. - 0.8.1 application is using Flask-Session stored in DB using SQL Alchemy driver. This can be configured for other drivers, however server side session is required for the application proper function. - 0.8.0 - API keys update. **Run migration scripts to update your DB**. Keys can now have expiration date and readonly flag. Admin can create special keys for certain machinnes. - 0.7.3 - New possibility of external auth proxy. diff --git a/docs/DB_MIGRATIONS.md b/docs/DB_MIGRATIONS.md new file mode 100644 index 00000000..93b8de3d --- /dev/null +++ b/docs/DB_MIGRATIONS.md @@ -0,0 +1,35 @@ +# How to Upgrade the Database + +## General Guidelines +Migrations can be inconsistent. To avoid issues, we removed migrations from git repostory. To start the migration on your server, it is recomended reset the migration state on the server and run the migration based on the updated database models when switching application versions via Git. + +```bash +rm -rf migrations/ +``` + +```SQL +DROP TABLE alembic_version; +``` + +```bash +flask db init +flask db migrate -m "Initial migration based on current DB state" +flask db upgrade +``` + +## Steps for Upgrading to v1.0.x +Limits for number of rules were introduced. Some database engines (Mariadb 10.x for example) have issue to set Non Null foreigin key to 0 and automatic migrations fail. The solution may be in diferent version (Mariadb 11.x works fine), or to set limits in db manually later. + +To set the limit to 0 for existing organizations run + +```SQL +UPDATE organization +SET limit_flowspec4 = 0, limit_flowspec6 = 0, limit_rtbh = 0 +WHERE limit_flowspec4 IS NULL OR limit_flowspec6 IS NULL OR limit_rtbh IS NULL; +``` + +In all cases we need later assign rules to organizations. There's an admin endpoint for this: + +`https://yourexafs.url/admin/set-org-if-zero` + +Or you can start with clean database and manually migrate data by SQL dump later. Feel free to contact jiri.vrany@cesnet.cz if you need help with the DB migration to 1.0.x.