From d2c2156a41d6cc10713aa569dd68f6e9be68b644 Mon Sep 17 00:00:00 2001
From: Sasha Romijn <github@mxsasha.eu>
Date: Mon, 8 Apr 2024 16:41:10 +0200
Subject: [PATCH] Use RPSLDatabaseQuery.default_suppression more widely
---
irrd/mirroring/mirror_runners_export.py | 8 +------
.../tests/test_mirror_runners_export.py | 11 ++-------
irrd/scripts/irr_rpsl_submit.py | 2 +-
irrd/server/http/event_stream.py | 14 +++--------
irrd/server/http/tests/test_event_stream.py | 7 +-----
irrd/storage/preload.py | 15 +++---------
irrd/storage/tests/test_database.py | 1 +
irrd/storage/tests/test_preload.py | 23 ++++---------------
pyproject.toml | 2 +-
9 files changed, 18 insertions(+), 65 deletions(-)
diff --git a/irrd/mirroring/mirror_runners_export.py b/irrd/mirroring/mirror_runners_export.py
index bb707cf..8200888 100644
--- a/irrd/mirroring/mirror_runners_export.py
+++ b/irrd/mirroring/mirror_runners_export.py
@@ -6,9 +6,6 @@
from tempfile import NamedTemporaryFile
from irrd.conf import get_setting
-from irrd.routepref.status import RoutePreferenceStatus
-from irrd.rpki.status import RPKIStatus
-from irrd.scopefilter.status import ScopeFilterStatus
from irrd.storage.database_handler import DatabaseHandler
from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseQuery
from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func
@@ -73,10 +70,7 @@ def _export(self, export_destination, remove_auth_hashes=True):
serial = None
with gzip.open(export_tmpfile.name, "wb") as fh:
- query = RPSLDatabaseQuery().sources([self.source])
- query = query.rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
- query = query.scopefilter_status([ScopeFilterStatus.in_scope])
- query = query.route_preference_status([RoutePreferenceStatus.visible])
+ query = RPSLDatabaseQuery().sources([self.source]).default_suppression()
for obj in self.database_handler.execute_query(query):
object_text = obj["object_text"]
if remove_auth_hashes:
diff --git a/irrd/mirroring/tests/test_mirror_runners_export.py b/irrd/mirroring/tests/test_mirror_runners_export.py
index 34dd250..548dfb2 100644
--- a/irrd/mirroring/tests/test_mirror_runners_export.py
+++ b/irrd/mirroring/tests/test_mirror_runners_export.py
@@ -4,9 +4,6 @@
from pathlib import Path
from unittest.mock import Mock
-from irrd.routepref.status import RoutePreferenceStatus
-from irrd.rpki.status import RPKIStatus
-from irrd.scopefilter.status import ScopeFilterStatus
from irrd.utils.test_utils import flatten_mock_calls
from ..mirror_runners_export import EXPORT_PERMISSIONS, SourceExportRunner
@@ -71,13 +68,9 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog):
]
assert flatten_mock_calls(mock_dq) == [
["sources", (["TEST"],), {}],
- ["rpki_status", ([RPKIStatus.not_found, RPKIStatus.valid],), {}],
- ["scopefilter_status", ([ScopeFilterStatus.in_scope],), {}],
- ["route_preference_status", ([RoutePreferenceStatus.visible],), {}],
+ ["default_suppression", (), {}],
["sources", (["TEST"],), {}],
- ["rpki_status", ([RPKIStatus.not_found, RPKIStatus.valid],), {}],
- ["scopefilter_status", ([ScopeFilterStatus.in_scope],), {}],
- ["route_preference_status", ([RoutePreferenceStatus.visible],), {}],
+ ["default_suppression", (), {}],
]
assert "Starting a source export for TEST" in caplog.text
assert "Export for TEST complete" in caplog.text
diff --git a/irrd/scripts/irr_rpsl_submit.py b/irrd/scripts/irr_rpsl_submit.py
index 01879c8..7fde798 100755
--- a/irrd/scripts/irr_rpsl_submit.py
+++ b/irrd/scripts/irr_rpsl_submit.py
@@ -760,7 +760,7 @@ def send_request(requests_text, args):
raise XHTTPConnectionFailed(args.url, http_request) from error # pragma: no cover
if reason == "Not Found":
raise XHTTPNotFound(args.url, http_request) from error
- raise error
+ raise error # pragma: no cover: CI glitch workaround
except Exception as error:
raise error
diff --git a/irrd/server/http/event_stream.py b/irrd/server/http/event_stream.py
index bd38d62..c61b409 100644
--- a/irrd/server/http/event_stream.py
+++ b/irrd/server/http/event_stream.py
@@ -16,10 +16,7 @@
from starlette.websockets import WebSocket
from irrd.conf import get_setting
-from irrd.routepref.status import RoutePreferenceStatus
-from irrd.rpki.status import RPKIStatus
from irrd.rpsl.rpsl_objects import rpsl_object_from_text
-from irrd.scopefilter.status import ScopeFilterStatus
from irrd.server.access_check import STARLETTE_TEST_CLIENT_HOST, is_client_permitted
from irrd.storage.database_handler import DatabaseHandler
from irrd.storage.event_stream import (
@@ -110,14 +107,9 @@ async def generate_rows(self):
}
async def generate_sql_query(self):
- query = (
- RPSLDatabaseQuery(
- column_names=["rpsl_pk", "object_class", "object_text", "source", "updated", "parsed_data"]
- )
- .rpki_status([RPKIStatus.not_found.name, RPKIStatus.valid.name])
- .scopefilter_status([ScopeFilterStatus.in_scope.name])
- .route_preference_status([RoutePreferenceStatus.visible.name])
- )
+ query = RPSLDatabaseQuery(
+ column_names=["rpsl_pk", "object_class", "object_text", "source", "updated", "parsed_data"]
+ ).default_suppression()
if self.sources:
query = query.sources(self.sources)
if self.object_classes:
diff --git a/irrd/server/http/tests/test_event_stream.py b/irrd/server/http/tests/test_event_stream.py
index 7d7c4e3..95725cf 100644
--- a/irrd/server/http/tests/test_event_stream.py
+++ b/irrd/server/http/tests/test_event_stream.py
@@ -12,10 +12,7 @@
from starlette.testclient import TestClient
from starlette.websockets import WebSocketDisconnect
-from irrd.routepref.status import RoutePreferenceStatus
-from irrd.rpki.status import RPKIStatus
from irrd.rpsl.rpsl_objects import rpsl_object_from_text
-from irrd.scopefilter.status import ScopeFilterStatus
from irrd.storage.event_stream import OPERATION_JOURNAL_EXTENDED
from irrd.storage.queries import (
RPSLDatabaseJournalQuery,
@@ -103,9 +100,7 @@ def mock_copy_to_side_effect(source, dest, engine_or_conn, format):
RPSLDatabaseQuery(
column_names=["rpsl_pk", "object_class", "object_text", "source", "updated", "parsed_data"]
)
- .rpki_status([RPKIStatus.not_found.name, RPKIStatus.valid.name])
- .scopefilter_status([ScopeFilterStatus.in_scope.name])
- .route_preference_status([RoutePreferenceStatus.visible.name])
+ .default_suppression()
.sources(["TEST"])
.object_classes(["mntner"])
.finalise_statement()
diff --git a/irrd/storage/preload.py b/irrd/storage/preload.py
index edf4d0f..829a634 100644
--- a/irrd/storage/preload.py
+++ b/irrd/storage/preload.py
@@ -11,9 +11,6 @@
from setproctitle import setproctitle
from irrd.conf import get_setting
-from irrd.routepref.status import RoutePreferenceStatus
-from irrd.rpki.status import RPKIStatus
-from irrd.scopefilter.status import ScopeFilterStatus
from irrd.utils.process_support import ExceptionLoggingProcess
from .queries import RPSLDatabaseQuery
@@ -509,9 +506,7 @@ def _update_routes(self, dh):
column_names=["ip_version", "ip_first", "prefix_length", "asn_first", "source"],
enable_ordering=False,
)
- q = q.object_classes(["route", "route6"]).rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
- q = q.scopefilter_status([ScopeFilterStatus.in_scope])
- q = q.route_preference_status([RoutePreferenceStatus.visible])
+ q = q.object_classes(["route", "route6"]).default_suppression()
for result in dh.execute_query(q):
prefix = result["ip_first"]
key = result["source"] + REDIS_KEY_PK_SOURCE_SEPARATOR + "AS" + str(result["asn_first"])
@@ -542,9 +537,7 @@ def _update_set(self, dh, set_class, member_classes):
enable_ordering=False,
)
.object_classes([set_class])
- .rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
- .scopefilter_status([ScopeFilterStatus.in_scope])
- .route_preference_status([RoutePreferenceStatus.visible])
+ .default_suppression()
)
member_store: Dict[str, set] = {}
@@ -568,9 +561,7 @@ def _update_set(self, dh, set_class, member_classes):
)
.lookup_attr("member-of", True)
.object_classes(member_classes)
- .rpki_status([RPKIStatus.not_found, RPKIStatus.valid])
- .scopefilter_status([ScopeFilterStatus.in_scope])
- .route_preference_status([RoutePreferenceStatus.visible])
+ .default_suppression()
)
for row in dh.execute_query(q):
for member_of in row["parsed_data"].get("member-of", []):
diff --git a/irrd/storage/tests/test_database.py b/irrd/storage/tests/test_database.py
index 3973802..39c8f14 100644
--- a/irrd/storage/tests/test_database.py
+++ b/irrd/storage/tests/test_database.py
@@ -1153,6 +1153,7 @@ def test_non_matching_filters(self, irrd_db_mock_preload, database_handler_with_
self._assert_no_match(RPSLDatabaseQuery().rpki_status([RPKIStatus.valid]))
self._assert_no_match(RPSLDatabaseQuery().scopefilter_status([ScopeFilterStatus.out_scope_as]))
self._assert_no_match(RPSLDatabaseQuery().route_preference_status([RoutePreferenceStatus.suppressed]))
+ self._assert_no_match(RPSLDatabaseQuery().default_suppression())
def test_ordering_sources(self, irrd_db_mock_preload, database_handler_with_route):
self.dh = database_handler_with_route
diff --git a/irrd/storage/tests/test_preload.py b/irrd/storage/tests/test_preload.py
index 5c4b343..1bc4662 100644
--- a/irrd/storage/tests/test_preload.py
+++ b/irrd/storage/tests/test_preload.py
@@ -4,9 +4,6 @@
import pytest
-from irrd.routepref.status import RoutePreferenceStatus
-from irrd.rpki.status import RPKIStatus
-from irrd.scopefilter.status import ScopeFilterStatus
from irrd.utils.test_utils import flatten_mock_calls
from ..database_handler import DatabaseHandler
@@ -334,27 +331,17 @@ def test_preload_updater(self, monkeypatch):
assert flatten_mock_calls(mock_reload_lock) == [["acquire", (), {}], ["release", (), {}]]
assert flatten_mock_calls(mock_database_query) == [
["object_classes", (["route", "route6"],), {}],
- ["rpki_status", ([RPKIStatus.not_found, RPKIStatus.valid],), {}],
- ["scopefilter_status", ([ScopeFilterStatus.in_scope],), {}],
- ["route_preference_status", ([RoutePreferenceStatus.visible],), {}],
+ ["default_suppression", (), {}],
["object_classes", (["as-set"],), {}],
- ["rpki_status", ([RPKIStatus.not_found, RPKIStatus.valid],), {}],
- ["scopefilter_status", ([ScopeFilterStatus.in_scope],), {}],
- ["route_preference_status", ([RoutePreferenceStatus.visible],), {}],
+ ["default_suppression", (), {}],
["lookup_attr", ("member-of", True), {}],
["object_classes", (["aut-num"],), {}],
- ["rpki_status", ([RPKIStatus.not_found, RPKIStatus.valid],), {}],
- ["scopefilter_status", ([ScopeFilterStatus.in_scope],), {}],
- ["route_preference_status", ([RoutePreferenceStatus.visible],), {}],
+ ["default_suppression", (), {}],
["object_classes", (["route-set"],), {}],
- ["rpki_status", ([RPKIStatus.not_found, RPKIStatus.valid],), {}],
- ["scopefilter_status", ([ScopeFilterStatus.in_scope],), {}],
- ["route_preference_status", ([RoutePreferenceStatus.visible],), {}],
+ ["default_suppression", (), {}],
["lookup_attr", ("member-of", True), {}],
["object_classes", (["route", "route6"],), {}],
- ["rpki_status", ([RPKIStatus.not_found, RPKIStatus.valid],), {}],
- ["scopefilter_status", ([ScopeFilterStatus.in_scope],), {}],
- ["route_preference_status", ([RoutePreferenceStatus.visible],), {}],
+ ["default_suppression", (), {}],
]
assert flatten_mock_calls(mock_preload_obj) == [
diff --git a/pyproject.toml b/pyproject.toml
index 497d9ec..6a5d7ee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -138,7 +138,7 @@ ruff = "ruff irrd --fix"
isort = "isort irrd"
mypy = "mypy irrd"
lint = ["black", "ruff", "isort", "mypy"]
-test = "pytest --cov-report term-missing:skip-covered --cov=irrd --basetemp=.tmpdirs/ irrd"
+test = "pytest --cov-report term-missing:skip-covered --cov=irrd --basetemp=.tmpdirs/ "
[tool.coverage.report]
exclude_lines = [