Skip to content

Skip integration dashboards in dashboard lists #683

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
riyazsh wants to merge 1 commit into
base: main
Choose a base branch
from
+145 −22
Open

Skip integration dashboards in dashboard lists #683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 68 additions & 13 deletions datadog_sync/model/dashboard_lists.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,15 @@ class DashboardLists(BaseResource):
resource_config = ResourceConfig(
resource_connections={"dashboards": ["dashboards.id"]},
base_path="/api/v1/dashboard/lists/manual",
excluded_attributes=["id", "type", "author", "created", "modified", "is_favorite", "dashboard_count"],
excluded_attributes=[
"id",
"type",
"author",
"created",
"modified",
"is_favorite",
"dashboard_count",
],
skip_resource_mapping=True,
)
# Additional Dashboards specific attributes
Expand All @@ -30,11 +38,15 @@ async def get_resources(self, client: CustomClient) -> List[Dict]:

return resp["dashboard_lists"]

async def import_resource(self, _id: Optional[str] = None, resource: Optional[Dict] = None) -> Tuple[str, Dict]:
async def import_resource(
self, _id: Optional[str] = None, resource: Optional[Dict] = None
) -> Tuple[str, Dict]:
source_client = self.config.source_client

if _id:
resource = await source_client.get(self.resource_config.base_path + f"/{_id}")
resource = await source_client.get(
self.resource_config.base_path + f"/{_id}"
)

resource = cast(dict, resource)
_id = str(resource["id"])
Expand All @@ -57,7 +69,7 @@ async def import_resource(self, _id: Optional[str] = None, resource: Optional[Di
return _id, resource

async def pre_resource_action_hook(self, _id, resource: Dict) -> None:
pass
self._drop_integration_dashboards(_id, resource)

async def pre_apply_hook(self) -> None:
pass
Expand All @@ -82,7 +94,8 @@ async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
resource.pop("dashboards")

resp = await destination_client.put(
self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}",
self.resource_config.base_path
+ f"/{self.config.state.destination[self.resource_type][_id]['id']}",
resource,
)

Expand All @@ -101,25 +114,67 @@ async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
async def delete_resource(self, _id: str) -> None:
destination_client = self.config.destination_client
await destination_client.delete(
self.resource_config.base_path + f"/{self.config.state.destination[self.resource_type][_id]['id']}"
self.resource_config.base_path
+ f"/{self.config.state.destination[self.resource_type][_id]['id']}"
)

def connect_id(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
if resource_to_connect == "dashboards" and self._is_integration_dashboard(r_obj):
def connect_id(
self, key: str, r_obj: Dict, resource_to_connect: str
) -> Optional[List[str]]:
if resource_to_connect == "dashboards" and self._is_integration_dashboard(
r_obj
):
return None
return super(DashboardLists, self).connect_id(key, r_obj, resource_to_connect)

def extract_source_ids(self, key: str, r_obj: Dict, resource_to_connect: str) -> Optional[List[str]]:
if resource_to_connect == "dashboards" and self._is_integration_dashboard(r_obj):
def extract_source_ids(
self, key: str, r_obj: Dict, resource_to_connect: str
) -> Optional[List[str]]:
if resource_to_connect == "dashboards" and self._is_integration_dashboard(
r_obj
):
return None
return super().extract_source_ids(key, r_obj, resource_to_connect)

@staticmethod
def _is_integration_dashboard(r_obj: Dict) -> bool:
return str(r_obj.get("type", "")).startswith("integration_")

async def update_dash_list_items(self, _id: str, dashboards: Dict, dashboard_list: dict):
payload = {"dashboards": dashboards}
def _drop_integration_dashboards(self, _id: str, resource: Dict) -> None:
dashboards = resource.get("dashboards")
if not isinstance(dashboards, list):
return

portable_dashboards = [
dash for dash in dashboards if not self._is_integration_dashboard(dash)
]
if len(portable_dashboards) == len(dashboards):
return

dropped = sorted(
str(dash.get("id", ""))
for dash in dashboards
if self._is_integration_dashboard(dash)
)
self.config.logger.info(
"dropping integration dashboards from dashboard list before sync; "
"integration dashboard IDs are not portable across orgs",
resource_type=self.resource_type,
_id=_id,
dropped_dashboard_ids=",".join(dropped),
)
resource["dashboards"] = portable_dashboards

async def update_dash_list_items(
self, _id: str, dashboards: List[Dict], dashboard_list: dict
):
payload = {
"dashboards": [
dash for dash in dashboards if not self._is_integration_dashboard(dash)
]
}
destination_client = self.config.destination_client
dashboards = await destination_client.put(self.dash_list_items_path.format(_id), payload)
dashboards = await destination_client.put(
self.dash_list_items_path.format(_id), payload
)
dashboard_list.update(dashboards)
86 changes: 77 additions & 9 deletions tests/unit/test_dashboard_lists.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@
import pytest

from datadog_sync.model.dashboard_lists import DashboardLists
from datadog_sync.utils.resource_utils import CustomClientHTTPError, ResourceConnectionError
from datadog_sync.utils.resource_utils import (
CustomClientHTTPError,
ResourceConnectionError,
)


def _make_dashboard_lists() -> DashboardLists:
Expand All @@ -23,22 +26,46 @@ def _make_dashboard_lists() -> DashboardLists:
return DashboardLists(config)


def test_connect_resources_maps_custom_dashboards_and_keeps_integration_dashboards():
def test_pre_resource_action_hook_drops_integration_dashboards_before_apply():
dashboard_lists = _make_dashboard_lists()
dashboard_lists.config.state.destination["dashboards"]["dash-src"] = {"id": "dash-dst"}
dashboard_lists.config.state.destination["dashboards"]["dash-src"] = {
"id": "dash-dst"
}
resource = {
"id": 510887,
"dashboards": [
{"id": "dash-src", "type": "custom_timeboard"},
{"id": "62", "type": "integration_timeboard"},
{"id": "30516", "type": "integration_timeboard"},
],
}

asyncio.run(dashboard_lists.pre_resource_action_hook("510887", resource))
dashboard_lists.connect_resources("510887", resource)

assert resource["dashboards"] == [
{"id": "dash-dst", "type": "custom_timeboard"},
{"id": "62", "type": "integration_timeboard"},
]


def test_connect_resources_ignores_unmapped_integration_dashboards():
dashboard_lists = _make_dashboard_lists()
dashboard_lists.config.state.destination["dashboards"]["dash-src"] = {
"id": "dash-dst"
}
resource = {
"id": 510887,
"dashboards": [
{"id": "dash-src", "type": "custom_timeboard"},
{"id": "30516", "type": "integration_timeboard"},
],
}

dashboard_lists.connect_resources("510887", resource)

assert resource["dashboards"] == [
{"id": "dash-dst", "type": "custom_timeboard"},
{"id": "30516", "type": "integration_timeboard"},
]


Expand Down Expand Up @@ -76,6 +103,33 @@ def test_connect_resources_still_fails_missing_custom_dashboards():
dashboard_lists.connect_resources("510887", resource)


def test_update_dash_list_items_drops_integration_dashboards_from_payload():
dashboard_lists = _make_dashboard_lists()
dashboard_lists.config.destination_client.put = AsyncMock(
return_value={"dashboards": [{"id": "dash-dst", "type": "custom_timeboard"}]}
)
dashboard_list = {}

asyncio.run(
dashboard_lists.update_dash_list_items(
"dst-list",
[
{"id": "dash-dst", "type": "custom_timeboard"},
{"id": "30516", "type": "integration_timeboard"},
],
dashboard_list,
)
)

dashboard_lists.config.destination_client.put.assert_awaited_once_with(
dashboard_lists.dash_list_items_path.format("dst-list"),
{"dashboards": [{"id": "dash-dst", "type": "custom_timeboard"}]},
)
assert dashboard_list == {
"dashboards": [{"id": "dash-dst", "type": "custom_timeboard"}]
}


def _http_error(status: int) -> CustomClientHTTPError:
resp = MagicMock()
resp.status = status
Expand All @@ -102,18 +156,30 @@ def test_500_on_items_fetch_propagates(self):
classifies it as http_5xx (transient) — counted as failure, logged
at WARNING, no exit-code poisoning, no incomplete state written."""
dashboard_lists = _make_dashboard_lists()
dashboard_lists.config.source_client.get = AsyncMock(side_effect=_http_error(500))
dashboard_lists.config.source_client.get = AsyncMock(
side_effect=_http_error(500)
)

with pytest.raises(CustomClientHTTPError):
asyncio.run(dashboard_lists.import_resource(resource={"id": "42", "name": "my-list"}))
asyncio.run(
dashboard_lists.import_resource(
resource={"id": "42", "name": "my-list"}
)
)

def test_503_on_items_fetch_propagates(self):
"""Any 5xx propagates — same treatment as 500."""
dashboard_lists = _make_dashboard_lists()
dashboard_lists.config.source_client.get = AsyncMock(side_effect=_http_error(503))
dashboard_lists.config.source_client.get = AsyncMock(
side_effect=_http_error(503)
)

with pytest.raises(CustomClientHTTPError):
asyncio.run(dashboard_lists.import_resource(resource={"id": "42", "name": "my-list"}))
asyncio.run(
dashboard_lists.import_resource(
resource={"id": "42", "name": "my-list"}
)
)

def test_items_fetch_success_populates_dashboards(self):
"""Happy path: items endpoint returns dashboard IDs that are
Expand All @@ -127,6 +193,8 @@ async def fake_get(path, **kwargs):

dashboard_lists.config.source_client.get = AsyncMock(side_effect=fake_get)

_id, resource = asyncio.run(dashboard_lists.import_resource(resource={"id": "42", "name": "my-list"}))
_id, resource = asyncio.run(
dashboard_lists.import_resource(resource={"id": "42", "name": "my-list"})
)

assert resource["dashboards"] == [{"id": "dash-1", "type": "custom_timeboard"}]
Loading