Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ Formatting settings for the `sqlmesh format` command and UI.
| `max_text_width` | The maximum text width in a segment before creating new lines (Default: 80) | int | N |
| `append_newline` | Whether to append a newline to the end of the file (Default: False) | boolean | N |
| `no_rewrite_casts` | Preserve the existing casts, without rewriting them to use the :: syntax. (Default: False) | boolean | N |
| `transpile_meta` | Whether to render the `MODEL`/`AUDIT`/`METRIC` header with the model's dialect instead of keeping it dialect-agnostic. Headers are dialect-agnostic by default because SQLMesh properties are not warehouse SQL, but projects that author headers in their warehouse dialect can enable this to preserve dialect-specific values such as column types. Note that enabling it also transpiles SQLMesh's own properties, so a boolean such as `allow_partials TRUE` is rendered as `(1 = 1)` in T-SQL. (Default: False) | boolean | N |


## Janitor
Expand Down
8 changes: 7 additions & 1 deletion sqlmesh/core/config/format.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ class FormatConfig(BaseConfig):
max_text_width: The maximum text width in a segment before creating new lines.
append_newline: Whether to append a newline to the end of the file or not.
no_rewrite_casts: Preserve the existing casts, without rewriting them to use the :: syntax.
transpile_meta: Whether to render the MODEL/AUDIT/METRIC header with the model's
dialect instead of keeping it dialect-agnostic. Headers are dialect-agnostic
by default because SQLMesh properties are not warehouse SQL, but projects that
author headers in their warehouse dialect can opt in to preserve
dialect-specific values such as column types.
"""

normalize: bool = False
Expand All @@ -41,6 +46,7 @@ class FormatConfig(BaseConfig):
max_text_width: int = 80
append_newline: bool = False
no_rewrite_casts: bool = False
transpile_meta: bool = False

@property
def generator_options(self) -> t.Dict[str, t.Any]:
Expand All @@ -49,4 +55,4 @@ def generator_options(self) -> t.Dict[str, t.Any]:
Returns:
The generator options.
"""
return self.dict(exclude={"append_newline", "no_rewrite_casts"})
return self.dict(exclude={"append_newline", "no_rewrite_casts", "transpile_meta"})
4 changes: 4 additions & 0 deletions sqlmesh/core/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -1307,6 +1307,7 @@ def _format(
transpile: t.Optional[str] = None,
rewrite_casts: t.Optional[bool] = None,
append_newline: t.Optional[bool] = None,
transpile_meta: t.Optional[bool] = None,
**kwargs: t.Any,
) -> str:
expressions = parse(before, default_dialect=self.config_for_node(target).dialect)
Expand All @@ -1327,6 +1328,9 @@ def _format(
rewrite_casts=(
rewrite_casts if rewrite_casts is not None else not format_config.no_rewrite_casts
),
transpile_meta=(
transpile_meta if transpile_meta is not None else format_config.transpile_meta
),
**{**format_config.generator_options, **kwargs},
)

Expand Down
21 changes: 14 additions & 7 deletions sqlmesh/core/dialect.py
Original file line number Diff line number Diff line change
Expand Up @@ -791,6 +791,7 @@ def format_model_expressions(
dialect: t.Optional[str] = None,
rewrite_casts: bool = True,
normalize_functions: t.Union[str, bool, None] = False,
transpile_meta: bool = False,
**kwargs: t.Any,
) -> str:
"""Format a model's expressions into a standardized format.
Expand All @@ -814,17 +815,25 @@ def format_model_expressions(
called via ``FormatConfig``, ``None`` is excluded by Pydantic's
``exclude_none`` serialization and this function receives its own ``False``
default instead — so the two paths are not equivalent.
transpile_meta: Whether to render the MODEL/AUDIT/METRIC header with ``dialect``
instead of keeping it dialect-agnostic. Headers are dialect-agnostic by
default because SQLMesh properties are not warehouse SQL, but projects that
author headers in their warehouse dialect can opt in to preserve
dialect-specific values such as column types.
**kwargs: Additional keyword arguments to pass to the sql generator.

Returns:
A string representing the formatted model.
"""
# Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL, not standard SQL, so by
# default they are not transpiled to the target dialect (e.g. tsql would rewrite a
# boolean property like `allow_partials TRUE` to `(1 = 1)`). Projects that author
# their headers in the warehouse dialect can opt in via `transpile_meta`.
meta_dialect = dialect if transpile_meta else None

if len(expressions) == 1 and is_meta_expression(expressions[0]):
# Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL, not standard SQL,
# so they must never be transpiled to the target dialect (e.g. tsql would
# rewrite a boolean property like `allow_partials TRUE` to `(1 = 1)`).
return expressions[0].sql(
pretty=True, dialect=None, normalize_functions=normalize_functions
pretty=True, dialect=meta_dialect, normalize_functions=normalize_functions
)

if rewrite_casts:
Expand Down Expand Up @@ -857,11 +866,9 @@ def cast_to_colon(node: exp.Expr) -> exp.Expr:
expressions = new_expressions

return ";\n\n".join(
# Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL and must stay
# dialect-agnostic; only the actual query/statement expressions transpile.
expression.sql(
pretty=True,
dialect=None if is_meta_expression(expression) else dialect,
dialect=meta_dialect if is_meta_expression(expression) else dialect,
normalize_functions=normalize_functions,
**kwargs,
)
Expand Down
1 change: 1 addition & 0 deletions sqlmesh/magics.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ def model(self, context: Context, line: str, sql: t.Optional[str] = None) -> Non
expressions,
model.dialect,
rewrite_casts=not config.format.no_rewrite_casts,
transpile_meta=config.format.transpile_meta,
**config.format.generator_options,
)

Expand Down
62 changes: 62 additions & 0 deletions tests/core/test_dialect.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,68 @@ def test_format_model_expressions():
)


def test_format_model_expressions_transpile_meta():
"""The opt-in `transpile_meta` flag renders MODEL/AUDIT/METRIC headers with the
target dialect, which projects that authored their headers in warehouse dialect
need in order to preserve dialect-specific values such as column types.
"""
expressions = parse(
"""
MODEL(
name a.b,
kind FULL,
dialect tsql,
allow_partials true,
columns (ts DATETIME2(6))
);
SELECT CAST(x AS INT) AS y FROM t
"""
)

# Default: the header stays dialect-agnostic, so tsql's DATETIME2 is generalized.
assert "ts TIMESTAMP(6)" in format_model_expressions(expressions, dialect="tsql")

x = format_model_expressions(expressions, dialect="tsql", transpile_meta=True)

# Opting in renders the header with tsql, preserving DATETIME2 at the cost of
# tsql's boolean representation. The query body transpiles either way.
assert (
x
== """MODEL (
name a.b,
kind FULL,
dialect tsql,
allow_partials (1 = 1),
columns (
ts DATETIME2(6)
)
);

SELECT
x::INTEGER AS y
FROM t"""
)


def test_format_model_expressions_transpile_meta_single_expression():
"""The single meta expression path (no query) must honor the flag too."""
expressions = parse("MODEL(name a.b, kind FULL, dialect tsql, columns (ts DATETIME2(6)))")

assert "ts TIMESTAMP(6)" in format_model_expressions(expressions, dialect="tsql")
assert "ts DATETIME2(6)" in format_model_expressions(
expressions, dialect="tsql", transpile_meta=True
)


def test_format_config_transpile_meta():
"""`transpile_meta` is a SQLMesh-level option, so it must not leak into the
options forwarded to SQLGlot's generator.
"""
assert FormatConfig().transpile_meta is False
assert FormatConfig(transpile_meta=True).transpile_meta is True
assert "transpile_meta" not in FormatConfig(transpile_meta=True).generator_options


def test_format_model_expressions_normalize_functions():
"""Regression: formatter function-name casing behavior.

Expand Down
16 changes: 16 additions & 0 deletions tests/core/test_format.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from tests.utils.test_filesystem import create_temp_file
from unittest.mock import call
from sqlmesh.core.config import ModelDefaultsConfig
from sqlmesh.core.config.format import FormatConfig


def test_format_files(tmp_path: pathlib.Path, mocker: MockerFixture):
Expand Down Expand Up @@ -103,6 +104,21 @@ def test_format_files(tmp_path: pathlib.Path, mocker: MockerFixture):
)


def test_format_transpile_meta(tmp_path: pathlib.Path):
model_text = (
"MODEL(name this.model, dialect 'tsql', columns (ts DATETIME2(6))); SELECT 1 AS col"
)
models_dir = pathlib.Path("models")

model = create_temp_file(tmp_path, pathlib.Path(models_dir, "model_1.sql"), model_text)
Context(paths=tmp_path, config=Config()).format()
assert "ts TIMESTAMP(6)" in model.read_text(encoding="utf-8")

model = create_temp_file(tmp_path, pathlib.Path(models_dir, "model_1.sql"), model_text)
Context(paths=tmp_path, config=Config(format=FormatConfig(transpile_meta=True))).format()
assert "ts DATETIME2(6)" in model.read_text(encoding="utf-8")


def test_ignore_formating_files(tmp_path: pathlib.Path):
models_dir = pathlib.Path("models")
audits_dir = pathlib.Path("audits")
Expand Down
5 changes: 4 additions & 1 deletion web/server/api/endpoints/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ async def write_file(
try:
expressions = parse(content, default_dialect=default_dialect)
content = format_model_expressions(
expressions, dialect, **config.format.generator_options
expressions,
dialect,
transpile_meta=config.format.transpile_meta,
**config.format.generator_options,
)
if config.format.append_newline:
content += "\n"
Expand Down