Repair metric tag config metadata type conflicts

datadog_sync/commands/shared/options.py

@@ -588,6 +588,20 @@ def click_config_file_provider(ctx: Context, opts: CustomOptionClass, value: Non
         "role/user activation is not ready yet; filtered monitors are not created or updated.",
         cls=CustomOptionClass,
     ),
+    option(
+        "--repair-metric-tag-configuration-metadata-type-conflicts",
+        required=False,
+        is_flag=True,
+        default=False,
+        show_default=True,
+        help=(
+            "When creating or updating metric_tag_configurations, repair a destination metrics_metadata "
+            "type mismatch by setting the destination metric type to the source tag configuration's "
+            "metric_type, then retry once. This mutates destination metric metadata and should only be "
+            "used after confirming the source tag configuration type is the desired source of truth."
+        ),
+        cls=CustomOptionClass,
+    ),
     option(
         "--create-global-downtime",
         required=False,

datadog_sync/model/metric_tag_configurations.py

@@ -35,6 +35,16 @@ def _is_missing_metadata_type_error(error: CustomClientHTTPError) -> bool:
     return error.status_code == 400 and "metadata type must be set prior to configuring tags" in body
 
 
+def _is_metadata_type_conflict_error(error: CustomClientHTTPError) -> bool:
+    body = _error_body(error)
+    return (
+        error.status_code == 400
+        and "cannot configure tags for" in body
+        and "with a metric_type of" in body
+        and "metadata set to type" in body
+    )
+
+
 def _metric_type_from_tag_configuration(resource: Dict) -> Optional[str]:
     metric_type = resource.get("attributes", {}).get("metric_type")
     if not isinstance(metric_type, str):
@@ -81,6 +91,14 @@ async def _set_destination_metric_metadata_type(self, _id: str, resource: Dict)
         await self.config.destination_client.put(f"/api/v1/metrics/{_id}", {"type": metric_type})
         return True
 
+    def _should_repair_metadata_type_error(self, error: CustomClientHTTPError) -> bool:
+        if _is_missing_metadata_type_error(error):
+            return True
+        return (
+            getattr(self.config, "repair_metric_tag_configuration_metadata_type_conflicts", False) is True
+            and _is_metadata_type_conflict_error(error)
+        )
+
     async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
         if _id in self._existing_resources_map:
             self.config.state.destination[self.resource_type][_id] = self._existing_resources_map[_id]
@@ -101,7 +119,9 @@ async def create_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
                     reason=FAILURE_CLASS_DESTINATION_METRIC_MISSING,
                     outcome_details={"metric_name": _id, "operation": "tag_configuration_create"},
                 )
-            if _is_missing_metadata_type_error(e) and await self._set_destination_metric_metadata_type(_id, resource):
+            if self._should_repair_metadata_type_error(e) and await self._set_destination_metric_metadata_type(
+                _id, resource
+            ):
                 try:
                     resp = await destination_client.post(path, payload)
                 except CustomClientHTTPError as retry_e:
@@ -140,7 +160,9 @@ async def update_resource(self, _id: str, resource: Dict) -> Tuple[str, Dict]:
                     reason=FAILURE_CLASS_DESTINATION_METRIC_MISSING,
                     outcome_details={"metric_name": _id, "operation": "tag_configuration_update"},
                 )
-            if _is_missing_metadata_type_error(e) and await self._set_destination_metric_metadata_type(_id, resource):
+            if self._should_repair_metadata_type_error(e) and await self._set_destination_metric_metadata_type(
+                _id, resource
+            ):
                 resp = await destination_client.patch(path, payload)
                 return _id, resp["data"]
             raise

datadog_sync/utils/configuration.py

@@ -127,6 +127,7 @@ class Configuration(object):
     prune_dry_run: bool = False
     destination_logs_intake_url: Optional[str] = None
     skip_monitors_with_restricted_roles: bool = False
+    repair_metric_tag_configuration_metadata_type_conflicts: bool = False
 
     async def init_async(self, cmd: Command):
         await self.source_client._init_session()
@@ -512,6 +513,9 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
     drop_unresolvable_principals = kwargs.get("drop_unresolvable_principals") or False
     refresh_destination_state_before_apply = kwargs.get("refresh_destination_state_before_apply") or False
     skip_monitors_with_restricted_roles = kwargs.get("skip_monitors_with_restricted_roles") or False
+    repair_metric_tag_configuration_metadata_type_conflicts = (
+        kwargs.get("repair_metric_tag_configuration_metadata_type_conflicts") or False
+    )
     max_workers = kwargs.get("max_workers")
     max_workers_per_type_raw = kwargs.get("max_workers_per_type")
     # Parse --max-workers-per-type early so malformed input fails BEFORE any
@@ -852,6 +856,7 @@ def build_config(cmd: Command, **kwargs: Optional[Any]) -> Configuration:
         transient_failure_threshold_pct=transient_failure_threshold_pct,
         destination_logs_intake_url=destination_logs_intake_url,
         skip_monitors_with_restricted_roles=skip_monitors_with_restricted_roles,
+        repair_metric_tag_configuration_metadata_type_conflicts=repair_metric_tag_configuration_metadata_type_conflicts,
     )
 
     # Initialize resource classes

tests/unit/test_metric_tag_config_metadata_type_repair_cli.py

@@ -0,0 +1,36 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the 3-clause BSD style license (see LICENSE).
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2019 Datadog, Inc.
+
+import importlib
+from unittest.mock import patch
+
+import pytest
+from click.testing import CliRunner
+
+from datadog_sync.cli import cli
+from datadog_sync.constants import Command
+
+
+@pytest.mark.parametrize(
+    "command,module_name,expected_command",
+    [
+        ("sync", "datadog_sync.commands.sync", Command.SYNC),
+        ("migrate", "datadog_sync.commands.migrate", Command.MIGRATE),
+    ],
+)
+def test_cli_accepts_metric_tag_configuration_metadata_type_repair_flag(
+    command, module_name, expected_command
+):
+    runner = CliRunner(mix_stderr=False)
+    command_module = importlib.import_module(module_name)
+
+    with patch.object(command_module, "run_cmd") as mock_run_cmd:
+        result = runner.invoke(cli, [command, "--repair-metric-tag-configuration-metadata-type-conflicts"])
+
+    assert result.exit_code == 0, result.output
+    mock_run_cmd.assert_called_once()
+    called_command, kwargs = mock_run_cmd.call_args.args[0], mock_run_cmd.call_args.kwargs
+    assert called_command == expected_command
+    assert kwargs["repair_metric_tag_configuration_metadata_type_conflicts"] is True

tests/unit/test_metric_tag_configurations.py

@@ -160,6 +160,58 @@ def test_create_resource_missing_metadata_type_retry_conflict_gets_existing_then
     assert data == {"id": "custom.metric", "attributes": {"tags": ["env", "service"]}}
 
 
+def test_create_resource_metadata_type_conflict_propagates_without_repair_flag(metric_tag_configurations):
+    client = metric_tag_configurations.config.destination_client
+    client.post = AsyncMock(
+        side_effect=_http_error(
+            400,
+            "Cannot configure tags for custom.metric with a metric_type of count "
+            "when the metric has metadata set to type rate.",
+        )
+    )
+    client.put = AsyncMock()
+    client.get = AsyncMock()
+    client.patch = AsyncMock()
+    metric_tag_configurations.config.state.source["metric_tag_configurations"]["custom.metric"] = _resource()
+
+    with pytest.raises(CustomClientHTTPError) as exc_info:
+        _run(metric_tag_configurations.create_resource("custom.metric", _resource()))
+
+    assert exc_info.value.status_code == 400
+    client.post.assert_awaited_once()
+    client.put.assert_not_awaited()
+    client.get.assert_not_awaited()
+    client.patch.assert_not_awaited()
+
+
+def test_create_resource_metadata_type_conflict_repairs_when_flag_enabled(metric_tag_configurations):
+    metric_tag_configurations.config.repair_metric_tag_configuration_metadata_type_conflicts = True
+    client = metric_tag_configurations.config.destination_client
+    client.post = AsyncMock(
+        side_effect=[
+            _http_error(
+                400,
+                "Cannot configure tags for custom.metric with a metric_type of count "
+                "when the metric has metadata set to type rate.",
+            ),
+            {"data": _resource()},
+        ]
+    )
+    client.put = AsyncMock(return_value={"type": "count"})
+    client.get = AsyncMock()
+    client.patch = AsyncMock()
+    metric_tag_configurations.config.state.source["metric_tag_configurations"]["custom.metric"] = _resource()
+
+    _id, data = _run(metric_tag_configurations.create_resource("custom.metric", _resource()))
+
+    assert _id == "custom.metric"
+    assert data == _resource()
+    client.put.assert_awaited_once_with("/api/v1/metrics/custom.metric", {"type": "count"})
+    assert client.post.await_count == 2
+    client.get.assert_not_awaited()
+    client.patch.assert_not_awaited()
+
+
 def test_create_resource_non_matching_409_propagates(metric_tag_configurations):
     client = metric_tag_configurations.config.destination_client
     client.post = AsyncMock(side_effect=_http_error(409, "conflict"))
@@ -230,3 +282,29 @@ def test_update_resource_missing_metadata_type_sets_metric_type_then_retries(met
     assert client.patch.await_count == 2
     assert client.patch.await_args_list[0].args[1]["data"]["attributes"] == {"tags": ["env", "service"]}
     assert resource["attributes"]["metric_type"] == "count"
+
+
+def test_update_resource_metadata_type_conflict_repairs_when_flag_enabled(metric_tag_configurations):
+    metric_tag_configurations.config.repair_metric_tag_configuration_metadata_type_conflicts = True
+    client = metric_tag_configurations.config.destination_client
+    client.patch = AsyncMock(
+        side_effect=[
+            _http_error(
+                400,
+                "Cannot configure tags for custom.metric with a metric_type of count "
+                "when the metric has metadata set to type rate.",
+            ),
+            {"data": {"id": "custom.metric", "attributes": {"tags": ["env", "service"]}}},
+        ]
+    )
+    client.put = AsyncMock(return_value={"type": "count"})
+    resource = _resource()
+    metric_tag_configurations.config.state.destination["metric_tag_configurations"]["custom.metric"] = _resource()
+
+    _id, data = _run(metric_tag_configurations.update_resource("custom.metric", resource))
+
+    assert _id == "custom.metric"
+    assert data == {"id": "custom.metric", "attributes": {"tags": ["env", "service"]}}
+    client.put.assert_awaited_once_with("/api/v1/metrics/custom.metric", {"type": "count"})
+    assert client.patch.await_count == 2
+    assert client.patch.await_args_list[0].args[1]["data"]["attributes"] == {"tags": ["env", "service"]}