feat: make the hash input cast datatype configurable per adapter

dbt_project.yml

@@ -31,6 +31,13 @@ vars:
   datavault4dbt.hashdiff_input_case_sensitive: TRUE
   datavault4dbt.hashdiff_use_trim: TRUE
 
+  # Datatypes that the hash inputs are casted to, before they are handed over to the hash function.
+  # 'attribute' applies to one single column, 'concat' applies to the fully concatenated payload.
+  # Multi Active Satellites are not affected by these variables, they keep the hardcoded datatype.
+  # CAUTION: A datatype shorter than the actual hash input truncates it silently, which changes the resulting hashes.
+  datavault4dbt.hash_input_attribute_dtype: {"bigquery":"STRING","snowflake":"STRING", "exasol": "VARCHAR(20000) UTF8", "postgres": "VARCHAR", "synapse": "VARCHAR(4000)", "fabric": "VARCHAR(4000)", "oracle":"VARCHAR2(2000)", databricks: "STRING", trino: "VARCHAR", "sqlserver": "VARCHAR(MAX)"}
+  datavault4dbt.hash_input_concat_dtype: {"bigquery":"STRING","snowflake":"STRING", "exasol": "VARCHAR(2000000) UTF8", "postgres": "VARCHAR", "redshift": "VARCHAR", "synapse": "VARCHAR(4000)", "fabric": "VARCHAR(4000)", "oracle":"VARCHAR2(2000)", databricks: "STRING", trino: "VARCHAR", "sqlserver": "VARCHAR(MAX)"}
+
   # Delimiters used when concatenating columns before hashing
   datavault4dbt.concat_string: '||'
   datavault4dbt.quote_character: '"'

docs/26_general-usage-notes/29_global-variables/29_global-variables.md

@@ -59,6 +59,14 @@ All the following variables are **prefixed with `datavault4dbt`**.
 | concat_string_replacement      | Stage | Token substituted for any occurrence of `concat_string` found *inside* the input data, so real values can never collide with the structural delimiter. Defaults to `dv4dbt-concat-replacement`. |
 | quote_character_replacement    | Stage | Token substituted for any occurrence of `quote_character` found *inside* the input data. Defaults to `dv4dbt-quote-replacement`. |
 | null_placeholder_string_replacement | Stage | Token substituted for any occurrence of `null_placeholder_string` found *inside* the input data. Defaults to `dv4dbt-null-replacement`. |
+| hash_input_attribute_dtype     | Stage | A mapping dictionary that defines, per database adapter, the datatype that a **single input column** is casted to inside `attribute_standardise`, before it is concatenated. |
+| hash_input_concat_dtype        | Stage | A mapping dictionary that defines, per database adapter, the datatype that the **fully concatenated payload** is casted to inside `concattenated_standardise`, before it is hashed. |
+
+Multi Active Satellites are **not** affected by either variable. Their payload is aggregated across all active records of one group before it is hashed, which makes a shortened datatype much more likely to be exceeded, so `multi_active_concattenated_standardise` keeps its hardcoded datatype. This lets you shorten the cast for all non-multi-active entities without putting your Multi Active Satellites at risk.
+
+:::warning
+Shortening `hash_input_attribute_dtype` or `hash_input_concat_dtype` below the actual length of your hash input truncates that input silently on most adapters. Truncated input produces different hash values, and two rows that only differ behind the truncation point collapse into the same hashkey or hashdiff. Only lower these values if you are certain that your concatenated input stays below the chosen limit, and treat any later change as a full reload of the affected entities.
+:::
 
 ### STAGE CONFIGURATION
 

docs/26_general-usage-notes/33_adapter-specific-notes/40_sqlserver/40_sqlserver.md

@@ -18,3 +18,25 @@ The SQL Server macros fall back to `VARBINARY(16)` internally, but if you copied
 
 - **Switch to a binary hash type:** Set `datavault4dbt.hash_datatype` in your `dbt_project.yml` to `VARBINARY(16)` (for MD5). This ensures hash keys and hashdiffs are stored efficiently and compared correctly on SQL Server.
 
+## HASH INPUT DATA TYPE
+
+Before hashing, both the single input columns and the concatenated payload are casted to a string datatype. On SQL Server this defaults to `VARCHAR(MAX)`, which is a large-value type: it is stored off-row, cannot be held in memory the same way as a regular `VARCHAR(n)`, and blocks several optimizations for the `REPLACE()`, `UPPER()` and `HASHBYTES()` calls wrapped around it. On wide satellites this can dominate the runtime of a load.
+
+Both casts are configurable per adapter through the global variables `datavault4dbt.hash_input_attribute_dtype` and `datavault4dbt.hash_input_concat_dtype`:
+
+```yaml
+vars:
+  datavault4dbt.hash_input_attribute_dtype: {"sqlserver": "VARCHAR(8000)"}
+  datavault4dbt.hash_input_concat_dtype: {"sqlserver": "VARCHAR(8000)"}
+```
+
+:::warning
+`VARCHAR(8000)` is the largest non-large-value `VARCHAR` on SQL Server. Any hash input longer than the configured length is truncated **silently**, which changes the resulting hashkeys and hashdiffs. Two rows that only differ behind the truncation point produce the same hash. Before lowering these values, verify that the concatenated input of your widest entity stays below the limit, and treat a later change as a full reload of all affected entities.
+:::
+
+### MULTI ACTIVE SATELLITES
+
+Multi Active Satellites are excluded from both variables and always stay on `VARCHAR(MAX)`, so you can shorten the cast for all other entities without touching them.
+
+The reason is `STRING_AGG()`, which aggregates the payload of all active records of one group before it is hashed. `STRING_AGG()` only returns `VARCHAR(MAX)` if its input expression is `VARCHAR(MAX)`; with a shorter input it returns `VARCHAR(8000)` and **raises an error** as soon as the aggregated result of a single group exceeds 8000 bytes. Since that limit applies to a whole group instead of a single record, it is far easier to hit than the per-record limit of a regular Satellite.
+

macros/supporting/hash_input_dtype.sql

@@ -0,0 +1,55 @@
+{#
+    Returns the string datatype that hash inputs are casted to, before they are handed over to the hash function.
+
+    Two different casts exist, which can be configured independently:
+      - 'attribute': The cast of one single column inside `attribute_standardise`.
+      - 'concat':    The cast of the fully concatenated payload inside `concattenated_standardise`.
+
+    Multi Active Satellites are deliberately NOT covered here. Their payload is aggregated across all
+    active records of one group before it is hashed, so a shortened datatype is far more likely to be
+    exceeded there. `multi_active_concattenated_standardise` therefore keeps its hardcoded datatype.
+
+    CAUTION: Choosing a datatype that is shorter than the actual hash input leads to a silent
+    truncation on most adapters, which changes the resulting hash values.
+#}
+
+{%- macro hash_input_dtype(type='concat') %}
+
+    {{ return(adapter.dispatch('hash_input_dtype', 'datavault4dbt')(type=type)) }}
+
+{%- endmacro -%}
+
+
+{%- macro default__hash_input_dtype(type) %}
+
+    {%- if type == 'attribute' -%}
+        {%- set var_name = 'datavault4dbt.hash_input_attribute_dtype' -%}
+        {%- set fallbacks = {"bigquery": "STRING", "snowflake": "STRING", "exasol": "VARCHAR(20000) UTF8", "postgres": "VARCHAR", "synapse": "VARCHAR(4000)", "fabric": "VARCHAR(4000)", "oracle": "VARCHAR2(2000)", "databricks": "STRING", "trino": "VARCHAR", "sqlserver": "VARCHAR(MAX)"} -%}
+    {%- else -%}
+        {%- set var_name = 'datavault4dbt.hash_input_concat_dtype' -%}
+        {%- set fallbacks = {"bigquery": "STRING", "snowflake": "STRING", "exasol": "VARCHAR(2000000) UTF8", "postgres": "VARCHAR", "redshift": "VARCHAR", "synapse": "VARCHAR(4000)", "fabric": "VARCHAR(4000)", "oracle": "VARCHAR2(2000)", "databricks": "STRING", "trino": "VARCHAR", "sqlserver": "VARCHAR(MAX)"} -%}
+    {%- endif -%}
+
+    {%- set global_var = var(var_name, none) -%}
+
+    {%- if global_var is mapping and target.type in global_var.keys()|map('lower') -%}
+
+        {%- set hash_input_dtype = global_var[target.type] -%}
+
+    {%- elif global_var is not mapping and datavault4dbt.is_something(global_var) -%}
+
+        {%- set hash_input_dtype = global_var -%}
+
+    {%- else -%}
+
+        {%- set hash_input_dtype = fallbacks.get(target.type, 'STRING') -%}
+
+        {%- if execute -%}
+            {%- do exceptions.warn("Warning: Adapter '"~ target.type ~"' not found in '" ~ var_name ~ "' variable. Defaulting to '" ~ hash_input_dtype ~ "'.") -%}
+        {%- endif -%}
+
+    {%- endif -%}
+
+    {{ return(hash_input_dtype) }}
+
+{%- endmacro -%}