diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index a7788df73d..0ece58fa4c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -119,6 +119,7 @@ Formatting settings for the `sqlmesh format` command and UI. | `max_text_width` | The maximum text width in a segment before creating new lines (Default: 80) | int | N | | `append_newline` | Whether to append a newline to the end of the file (Default: False) | boolean | N | | `no_rewrite_casts` | Preserve the existing casts, without rewriting them to use the :: syntax. (Default: False) | boolean | N | +| `transpile_meta` | Whether to render the `MODEL`/`AUDIT`/`METRIC` header with the model's dialect instead of keeping it dialect-agnostic. Headers are dialect-agnostic by default because SQLMesh properties are not warehouse SQL, but projects that author headers in their warehouse dialect can enable this to preserve dialect-specific values such as column types. Note that enabling it also transpiles SQLMesh's own properties, so a boolean such as `allow_partials TRUE` is rendered as `(1 = 1)` in T-SQL. (Default: False) | boolean | N | ## Janitor diff --git a/sqlmesh/core/config/format.py b/sqlmesh/core/config/format.py index 5ec6da47cd..77f57c944e 100644 --- a/sqlmesh/core/config/format.py +++ b/sqlmesh/core/config/format.py @@ -31,6 +31,11 @@ class FormatConfig(BaseConfig): max_text_width: The maximum text width in a segment before creating new lines. append_newline: Whether to append a newline to the end of the file or not. no_rewrite_casts: Preserve the existing casts, without rewriting them to use the :: syntax. + transpile_meta: Whether to render the MODEL/AUDIT/METRIC header with the model's + dialect instead of keeping it dialect-agnostic. Headers are dialect-agnostic + by default because SQLMesh properties are not warehouse SQL, but projects that + author headers in their warehouse dialect can opt in to preserve + dialect-specific values such as column types. """ normalize: bool = False @@ -41,6 +46,7 @@ class FormatConfig(BaseConfig): max_text_width: int = 80 append_newline: bool = False no_rewrite_casts: bool = False + transpile_meta: bool = False @property def generator_options(self) -> t.Dict[str, t.Any]: @@ -49,4 +55,4 @@ def generator_options(self) -> t.Dict[str, t.Any]: Returns: The generator options. """ - return self.dict(exclude={"append_newline", "no_rewrite_casts"}) + return self.dict(exclude={"append_newline", "no_rewrite_casts", "transpile_meta"}) diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index 7b435f4c62..785298039f 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -1307,6 +1307,7 @@ def _format( transpile: t.Optional[str] = None, rewrite_casts: t.Optional[bool] = None, append_newline: t.Optional[bool] = None, + transpile_meta: t.Optional[bool] = None, **kwargs: t.Any, ) -> str: expressions = parse(before, default_dialect=self.config_for_node(target).dialect) @@ -1327,6 +1328,9 @@ def _format( rewrite_casts=( rewrite_casts if rewrite_casts is not None else not format_config.no_rewrite_casts ), + transpile_meta=( + transpile_meta if transpile_meta is not None else format_config.transpile_meta + ), **{**format_config.generator_options, **kwargs}, ) diff --git a/sqlmesh/core/dialect.py b/sqlmesh/core/dialect.py index 67918b6d14..f4e05e9a29 100644 --- a/sqlmesh/core/dialect.py +++ b/sqlmesh/core/dialect.py @@ -791,6 +791,7 @@ def format_model_expressions( dialect: t.Optional[str] = None, rewrite_casts: bool = True, normalize_functions: t.Union[str, bool, None] = False, + transpile_meta: bool = False, **kwargs: t.Any, ) -> str: """Format a model's expressions into a standardized format. @@ -814,17 +815,25 @@ def format_model_expressions( called via ``FormatConfig``, ``None`` is excluded by Pydantic's ``exclude_none`` serialization and this function receives its own ``False`` default instead — so the two paths are not equivalent. + transpile_meta: Whether to render the MODEL/AUDIT/METRIC header with ``dialect`` + instead of keeping it dialect-agnostic. Headers are dialect-agnostic by + default because SQLMesh properties are not warehouse SQL, but projects that + author headers in their warehouse dialect can opt in to preserve + dialect-specific values such as column types. **kwargs: Additional keyword arguments to pass to the sql generator. Returns: A string representing the formatted model. """ + # Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL, not standard SQL, so by + # default they are not transpiled to the target dialect (e.g. tsql would rewrite a + # boolean property like `allow_partials TRUE` to `(1 = 1)`). Projects that author + # their headers in the warehouse dialect can opt in via `transpile_meta`. + meta_dialect = dialect if transpile_meta else None + if len(expressions) == 1 and is_meta_expression(expressions[0]): - # Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL, not standard SQL, - # so they must never be transpiled to the target dialect (e.g. tsql would - # rewrite a boolean property like `allow_partials TRUE` to `(1 = 1)`). return expressions[0].sql( - pretty=True, dialect=None, normalize_functions=normalize_functions + pretty=True, dialect=meta_dialect, normalize_functions=normalize_functions ) if rewrite_casts: @@ -857,11 +866,9 @@ def cast_to_colon(node: exp.Expr) -> exp.Expr: expressions = new_expressions return ";\n\n".join( - # Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL and must stay - # dialect-agnostic; only the actual query/statement expressions transpile. expression.sql( pretty=True, - dialect=None if is_meta_expression(expression) else dialect, + dialect=meta_dialect if is_meta_expression(expression) else dialect, normalize_functions=normalize_functions, **kwargs, ) diff --git a/sqlmesh/magics.py b/sqlmesh/magics.py index 3a59fc4f7b..aeae796af8 100644 --- a/sqlmesh/magics.py +++ b/sqlmesh/magics.py @@ -303,6 +303,7 @@ def model(self, context: Context, line: str, sql: t.Optional[str] = None) -> Non expressions, model.dialect, rewrite_casts=not config.format.no_rewrite_casts, + transpile_meta=config.format.transpile_meta, **config.format.generator_options, ) diff --git a/tests/core/test_dialect.py b/tests/core/test_dialect.py index e2f1daba3d..4450db8e15 100644 --- a/tests/core/test_dialect.py +++ b/tests/core/test_dialect.py @@ -342,6 +342,68 @@ def test_format_model_expressions(): ) +def test_format_model_expressions_transpile_meta(): + """The opt-in `transpile_meta` flag renders MODEL/AUDIT/METRIC headers with the + target dialect, which projects that authored their headers in warehouse dialect + need in order to preserve dialect-specific values such as column types. + """ + expressions = parse( + """ + MODEL( + name a.b, + kind FULL, + dialect tsql, + allow_partials true, + columns (ts DATETIME2(6)) + ); + SELECT CAST(x AS INT) AS y FROM t + """ + ) + + # Default: the header stays dialect-agnostic, so tsql's DATETIME2 is generalized. + assert "ts TIMESTAMP(6)" in format_model_expressions(expressions, dialect="tsql") + + x = format_model_expressions(expressions, dialect="tsql", transpile_meta=True) + + # Opting in renders the header with tsql, preserving DATETIME2 at the cost of + # tsql's boolean representation. The query body transpiles either way. + assert ( + x + == """MODEL ( + name a.b, + kind FULL, + dialect tsql, + allow_partials (1 = 1), + columns ( + ts DATETIME2(6) + ) +); + +SELECT + x::INTEGER AS y +FROM t""" + ) + + +def test_format_model_expressions_transpile_meta_single_expression(): + """The single meta expression path (no query) must honor the flag too.""" + expressions = parse("MODEL(name a.b, kind FULL, dialect tsql, columns (ts DATETIME2(6)))") + + assert "ts TIMESTAMP(6)" in format_model_expressions(expressions, dialect="tsql") + assert "ts DATETIME2(6)" in format_model_expressions( + expressions, dialect="tsql", transpile_meta=True + ) + + +def test_format_config_transpile_meta(): + """`transpile_meta` is a SQLMesh-level option, so it must not leak into the + options forwarded to SQLGlot's generator. + """ + assert FormatConfig().transpile_meta is False + assert FormatConfig(transpile_meta=True).transpile_meta is True + assert "transpile_meta" not in FormatConfig(transpile_meta=True).generator_options + + def test_format_model_expressions_normalize_functions(): """Regression: formatter function-name casing behavior. diff --git a/tests/core/test_format.py b/tests/core/test_format.py index 5a44e1b381..56df59a50f 100644 --- a/tests/core/test_format.py +++ b/tests/core/test_format.py @@ -9,6 +9,7 @@ from tests.utils.test_filesystem import create_temp_file from unittest.mock import call from sqlmesh.core.config import ModelDefaultsConfig +from sqlmesh.core.config.format import FormatConfig def test_format_files(tmp_path: pathlib.Path, mocker: MockerFixture): @@ -103,6 +104,21 @@ def test_format_files(tmp_path: pathlib.Path, mocker: MockerFixture): ) +def test_format_transpile_meta(tmp_path: pathlib.Path): + model_text = ( + "MODEL(name this.model, dialect 'tsql', columns (ts DATETIME2(6))); SELECT 1 AS col" + ) + models_dir = pathlib.Path("models") + + model = create_temp_file(tmp_path, pathlib.Path(models_dir, "model_1.sql"), model_text) + Context(paths=tmp_path, config=Config()).format() + assert "ts TIMESTAMP(6)" in model.read_text(encoding="utf-8") + + model = create_temp_file(tmp_path, pathlib.Path(models_dir, "model_1.sql"), model_text) + Context(paths=tmp_path, config=Config(format=FormatConfig(transpile_meta=True))).format() + assert "ts DATETIME2(6)" in model.read_text(encoding="utf-8") + + def test_ignore_formating_files(tmp_path: pathlib.Path): models_dir = pathlib.Path("models") audits_dir = pathlib.Path("audits") diff --git a/web/server/api/endpoints/files.py b/web/server/api/endpoints/files.py index db58fce55e..33b14aed56 100644 --- a/web/server/api/endpoints/files.py +++ b/web/server/api/endpoints/files.py @@ -77,7 +77,10 @@ async def write_file( try: expressions = parse(content, default_dialect=default_dialect) content = format_model_expressions( - expressions, dialect, **config.format.generator_options + expressions, + dialect, + transpile_meta=config.format.transpile_meta, + **config.format.generator_options, ) if config.format.append_newline: content += "\n"