From 2b2591c9b3e861ae3a1c8f6553ea6261a5245efd Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Mon, 3 Nov 2025 10:28:48 -0500 Subject: [PATCH 1/6] docs: clarify valid URNs --- proto/substrait/extensions/extensions.proto | 1 + site/docs/extensions/index.md | 2 +- site/docs/serialization/binary_serialization.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/proto/substrait/extensions/extensions.proto b/proto/substrait/extensions/extensions.proto index 66982e6a1..420733d3c 100644 --- a/proto/substrait/extensions/extensions.proto +++ b/proto/substrait/extensions/extensions.proto @@ -29,6 +29,7 @@ message SimpleExtensionURN { // The extension URN that uniquely identifies this extension. This must follow the // format extension:: and serves as the "namespace" of this extension. + // Otherwise, the urn is a RFC 8141 valid URN with the "urn:" prefix taken off. string urn = 2; } diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index 16aa248e3..c7f345920 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -13,7 +13,7 @@ Some kinds of primitives are so frequently extended that Substrait defines a sta * Window Functions * Table Functions -To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), they will be referred to as `extension URNs` for clarity. +To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), they will be referred to as `extension URNs` for clarity. (These URNs are otherwise valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) with the `"urn:` prefix removed from the front.) This extension URN uses the format `extension::`, where: diff --git a/site/docs/serialization/binary_serialization.md b/site/docs/serialization/binary_serialization.md index 7670377e7..464db9c30 100644 --- a/site/docs/serialization/binary_serialization.md +++ b/site/docs/serialization/binary_serialization.md @@ -22,7 +22,7 @@ For simple extensions, a plan references the extension URNs associated with the Simple extensions within a plan are split into three components: an extension URN, an extension declaration and a number of references. -* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. +* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The URN is neccessarily a valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) but with the `"urn:"` prefix cutoff. * **Extension Declaration**: A specific extension within a single YAML document. The declaration combines a reference to the associated extension URN along with a unique key identifying the specific item within that YAML document (see [Function Signature](../extensions/index.md#function-signature)). It also defines a declaration anchor. The anchor is a plan-specific unique value that the producer creates as a key to be referenced elsewhere. * **Extension Reference**: A specific instance or use of an extension declaration within the plan body. From 99e8a7fc41667c57f226bba075ceef348bdecb19 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Mon, 3 Nov 2025 16:13:37 -0500 Subject: [PATCH 2/6] docs: improve documentation surrounding urns --- proto/substrait/extensions/extensions.proto | 2 +- site/docs/extensions/index.md | 4 +++- site/docs/serialization/binary_serialization.md | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/proto/substrait/extensions/extensions.proto b/proto/substrait/extensions/extensions.proto index 420733d3c..9521a06f0 100644 --- a/proto/substrait/extensions/extensions.proto +++ b/proto/substrait/extensions/extensions.proto @@ -29,7 +29,7 @@ message SimpleExtensionURN { // The extension URN that uniquely identifies this extension. This must follow the // format extension:: and serves as the "namespace" of this extension. - // Otherwise, the urn is a RFC 8141 valid URN with the "urn:" prefix taken off. + // The URN must be valid RFC 8141 format without the urn: prefix. string urn = 2; } diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index c7f345920..846ddd0cc 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -13,13 +13,15 @@ Some kinds of primitives are so frequently extended that Substrait defines a sta * Window Functions * Table Functions -To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), they will be referred to as `extension URNs` for clarity. (These URNs are otherwise valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) with the `"urn:` prefix removed from the front.) +To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), they will be referred to as `extension URNs` for clarity. These URNs must be valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) format without the `urn:` prefix. This extension URN uses the format `extension::`, where: - `OWNER` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions - `ID` is the specific identifier for the extension (e.g., `functions_arithmetic`, `custom_types`) +Thus, if `extension::` is our URN, then `urn:extension::` must be a valid [RFC 8141 URN](https://www.rfc-editor.org/rfc/rfc8141.html). + The YAML file is constructed according to the [YAML Schema](https://github.com/substrait-io/substrait/blob/main/text/simple_extensions_schema.yaml). Each definition in the file corresponds to the YAML-based serialization of the relevant data structure. If a user only wants to extend one of these types of objects (e.g. types), a developer does not have to provide definitions for the other extension points. A Substrait plan can reference one or more YAML files via their extension URN. In the places where these entities are referenced, they will be referenced using an extension URN + name reference. The name scheme per type works as follows: diff --git a/site/docs/serialization/binary_serialization.md b/site/docs/serialization/binary_serialization.md index 464db9c30..77750b259 100644 --- a/site/docs/serialization/binary_serialization.md +++ b/site/docs/serialization/binary_serialization.md @@ -22,7 +22,7 @@ For simple extensions, a plan references the extension URNs associated with the Simple extensions within a plan are split into three components: an extension URN, an extension declaration and a number of references. -* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The URN is neccessarily a valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) but with the `"urn:"` prefix cutoff. +* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The URN with the `urn:` prefix added must conform to [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html). * **Extension Declaration**: A specific extension within a single YAML document. The declaration combines a reference to the associated extension URN along with a unique key identifying the specific item within that YAML document (see [Function Signature](../extensions/index.md#function-signature)). It also defines a declaration anchor. The anchor is a plan-specific unique value that the producer creates as a key to be referenced elsewhere. * **Extension Reference**: A specific instance or use of an extension declaration within the plan body. From 82476071295a32be3ed752a8a98615156ee19d99 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Tue, 18 Nov 2025 14:50:52 -0500 Subject: [PATCH 3/6] docs: clarify URN required format with regex The required regex is: ^extension:[^:]+:[^:]+$ --- proto/substrait/extensions/extensions.proto | 2 +- site/docs/extensions/index.md | 4 ++-- site/docs/serialization/binary_serialization.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/proto/substrait/extensions/extensions.proto b/proto/substrait/extensions/extensions.proto index 9521a06f0..ed306e2a7 100644 --- a/proto/substrait/extensions/extensions.proto +++ b/proto/substrait/extensions/extensions.proto @@ -29,7 +29,7 @@ message SimpleExtensionURN { // The extension URN that uniquely identifies this extension. This must follow the // format extension:: and serves as the "namespace" of this extension. - // The URN must be valid RFC 8141 format without the urn: prefix. + // This must conform to the following regex: ^extension:[^:]+:[^:]+$ string urn = 2; } diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index 846ddd0cc..d8de1ff61 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -13,14 +13,14 @@ Some kinds of primitives are so frequently extended that Substrait defines a sta * Window Functions * Table Functions -To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), they will be referred to as `extension URNs` for clarity. These URNs must be valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) format without the `urn:` prefix. +To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), and they will be referred to as `extension URNs` for clarity. This extension URN uses the format `extension::`, where: - `OWNER` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions - `ID` is the specific identifier for the extension (e.g., `functions_arithmetic`, `custom_types`) -Thus, if `extension::` is our URN, then `urn:extension::` must be a valid [RFC 8141 URN](https://www.rfc-editor.org/rfc/rfc8141.html). +These URNs must match the regex `^extension:[^:]+:[^:]+$`. The YAML file is constructed according to the [YAML Schema](https://github.com/substrait-io/substrait/blob/main/text/simple_extensions_schema.yaml). Each definition in the file corresponds to the YAML-based serialization of the relevant data structure. If a user only wants to extend one of these types of objects (e.g. types), a developer does not have to provide definitions for the other extension points. diff --git a/site/docs/serialization/binary_serialization.md b/site/docs/serialization/binary_serialization.md index 77750b259..2191064fc 100644 --- a/site/docs/serialization/binary_serialization.md +++ b/site/docs/serialization/binary_serialization.md @@ -22,7 +22,7 @@ For simple extensions, a plan references the extension URNs associated with the Simple extensions within a plan are split into three components: an extension URN, an extension declaration and a number of references. -* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The URN with the `urn:` prefix added must conform to [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html). +* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The extension URN must conform to the regex `^extension:[^:]+:[^:]+$`. * **Extension Declaration**: A specific extension within a single YAML document. The declaration combines a reference to the associated extension URN along with a unique key identifying the specific item within that YAML document (see [Function Signature](../extensions/index.md#function-signature)). It also defines a declaration anchor. The anchor is a plan-specific unique value that the producer creates as a key to be referenced elsewhere. * **Extension Reference**: A specific instance or use of an extension declaration within the plan body. From 6ff417243e85fa0338b4132c28582bc5aa99e9b2 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Thu, 20 Nov 2025 15:28:24 -0500 Subject: [PATCH 4/6] tweak: tighten up urn regex and add to extension schema --- site/docs/extensions/index.md | 2 +- text/simple_extensions_schema.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index 8f7b7d35f..e9b98c212 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -20,7 +20,7 @@ This extension URN uses the format `extension::`, where: - `OWNER` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions - `ID` is the specific identifier for the extension (e.g., `functions_arithmetic`, `custom_types`) -These URNs must match the regex `^extension:[^:]+:[^:]+$`. +These URNs must match the regex `^extension:[a-zA-Z0-9_.-]+:[a-zA-Z0-9_.-]+$`. The YAML file is constructed according to the [YAML Schema](https://github.com/substrait-io/substrait/blob/main/text/simple_extensions_schema.yaml). Each definition in the file corresponds to the YAML-based serialization of the relevant data structure. If a user only wants to extend one of these types of objects (e.g. types), a developer does not have to provide definitions for the other extension points. diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index f6bef9862..fce810bc9 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -7,6 +7,7 @@ required: [urn] properties: urn: type: string + pattern: "^extension:[a-zA-Z0-9_.-]+:[a-zA-Z0-9_.-]+$" dependencies: # For reusing type classes and type variations from other extension files. # The keys are namespace identifiers that you can then use as dot-separated From 03a9d7f6f1fdfd29e6ec985494e4104128ef6b6c Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Fri, 21 Nov 2025 15:33:45 -0500 Subject: [PATCH 5/6] tweak: use more restricive URN regex w/o capital letters `^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$` --- proto/substrait/extensions/extensions.proto | 2 +- site/docs/extensions/index.md | 2 +- site/docs/serialization/binary_serialization.md | 2 +- text/simple_extensions_schema.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/proto/substrait/extensions/extensions.proto b/proto/substrait/extensions/extensions.proto index ed306e2a7..48788b502 100644 --- a/proto/substrait/extensions/extensions.proto +++ b/proto/substrait/extensions/extensions.proto @@ -29,7 +29,7 @@ message SimpleExtensionURN { // The extension URN that uniquely identifies this extension. This must follow the // format extension:: and serves as the "namespace" of this extension. - // This must conform to the following regex: ^extension:[^:]+:[^:]+$ + // This must conform to the following regex: ^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$ string urn = 2; } diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index e9b98c212..5367b95f1 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -20,7 +20,7 @@ This extension URN uses the format `extension::`, where: - `OWNER` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions - `ID` is the specific identifier for the extension (e.g., `functions_arithmetic`, `custom_types`) -These URNs must match the regex `^extension:[a-zA-Z0-9_.-]+:[a-zA-Z0-9_.-]+$`. +These URNs must match the regex `^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$`. The YAML file is constructed according to the [YAML Schema](https://github.com/substrait-io/substrait/blob/main/text/simple_extensions_schema.yaml). Each definition in the file corresponds to the YAML-based serialization of the relevant data structure. If a user only wants to extend one of these types of objects (e.g. types), a developer does not have to provide definitions for the other extension points. diff --git a/site/docs/serialization/binary_serialization.md b/site/docs/serialization/binary_serialization.md index 2191064fc..5b3fa16ae 100644 --- a/site/docs/serialization/binary_serialization.md +++ b/site/docs/serialization/binary_serialization.md @@ -22,7 +22,7 @@ For simple extensions, a plan references the extension URNs associated with the Simple extensions within a plan are split into three components: an extension URN, an extension declaration and a number of references. -* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The extension URN must conform to the regex `^extension:[^:]+:[^:]+$`. +* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The extension URN must conform to the regex `^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$`. * **Extension Declaration**: A specific extension within a single YAML document. The declaration combines a reference to the associated extension URN along with a unique key identifying the specific item within that YAML document (see [Function Signature](../extensions/index.md#function-signature)). It also defines a declaration anchor. The anchor is a plan-specific unique value that the producer creates as a key to be referenced elsewhere. * **Extension Reference**: A specific instance or use of an extension declaration within the plan body. diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index fce810bc9..a635e4943 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -7,7 +7,7 @@ required: [urn] properties: urn: type: string - pattern: "^extension:[a-zA-Z0-9_.-]+:[a-zA-Z0-9_.-]+$" + pattern: "^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$" dependencies: # For reusing type classes and type variations from other extension files. # The keys are namespace identifiers that you can then use as dot-separated From 1e9860cf07ad3114482b8eb6028fe7c266bd8d78 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Wed, 18 Mar 2026 17:48:02 -0400 Subject: [PATCH 6/6] docs: adopt canonical URN format urn:substrait:extension:: - Update documentation to define canonical URN format - Add backwards compatibility note for extension: prefix - Update all extension YAML files to use canonical form - Update schema to enforce canonical format --- extensions/extension_types.yaml | 2 +- extensions/functions_aggregate_approx.yaml | 2 +- extensions/functions_aggregate_decimal_output.yaml | 2 +- extensions/functions_aggregate_generic.yaml | 2 +- extensions/functions_arithmetic.yaml | 2 +- extensions/functions_arithmetic_decimal.yaml | 2 +- extensions/functions_boolean.yaml | 2 +- extensions/functions_comparison.yaml | 2 +- extensions/functions_datetime.yaml | 2 +- extensions/functions_geometry.yaml | 2 +- extensions/functions_list.yaml | 2 +- extensions/functions_logarithmic.yaml | 2 +- extensions/functions_rounding.yaml | 2 +- extensions/functions_rounding_decimal.yaml | 2 +- extensions/functions_set.yaml | 2 +- extensions/functions_string.yaml | 2 +- extensions/type_variations.yaml | 2 +- extensions/unknown.yaml | 2 +- proto/substrait/extensions/extensions.proto | 8 +++++--- site/docs/extensions/index.md | 10 ++++++---- site/docs/serialization/binary_serialization.md | 2 +- site/examples/extensions/any1_type_function.yaml | 2 +- site/examples/extensions/any_type_function.yaml | 2 +- site/examples/extensions/distance_functions.yaml | 2 +- site/examples/extensions/double_function.yaml | 2 +- site/examples/extensions/lambda_function_example.yaml | 2 +- site/examples/extensions/metadata_example.yaml | 2 +- site/examples/types/point_with_datatype_param.yaml | 2 +- site/examples/types/point_with_enum_param.yaml | 2 +- site/examples/types/point_with_nstruct.yaml | 2 +- site/examples/types/point_with_structure.yaml | 2 +- site/examples/types/point_with_two_params.yaml | 2 +- site/examples/types/tuple_optional_variadic.yaml | 2 +- site/examples/types/union_variadic.yaml | 2 +- site/examples/types/user_defined_point.yaml | 2 +- site/examples/types/vector_with_constraints.yaml | 2 +- text/simple_extensions_schema.yaml | 2 +- 37 files changed, 46 insertions(+), 42 deletions(-) diff --git a/extensions/extension_types.yaml b/extensions/extension_types.yaml index f05ad2c0d..0b1bc5183 100644 --- a/extensions/extension_types.yaml +++ b/extensions/extension_types.yaml @@ -1,5 +1,5 @@ --- -urn: extension:io.substrait:extension_types +urn: urn:substrait:extension:io.substrait:extension_types types: - name: point structure: diff --git a/extensions/functions_aggregate_approx.yaml b/extensions/functions_aggregate_approx.yaml index d858accf4..622324fe8 100644 --- a/extensions/functions_aggregate_approx.yaml +++ b/extensions/functions_aggregate_approx.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_aggregate_approx +urn: urn:substrait:extension:io.substrait:functions_aggregate_approx aggregate_functions: - name: "approx_count_distinct" description: >- diff --git a/extensions/functions_aggregate_decimal_output.yaml b/extensions/functions_aggregate_decimal_output.yaml index 85a1959b9..89afa2c58 100644 --- a/extensions/functions_aggregate_decimal_output.yaml +++ b/extensions/functions_aggregate_decimal_output.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_aggregate_decimal_output +urn: urn:substrait:extension:io.substrait:functions_aggregate_decimal_output aggregate_functions: - name: "count" description: Count a set of values. Result is returned as a decimal instead of i64. diff --git a/extensions/functions_aggregate_generic.yaml b/extensions/functions_aggregate_generic.yaml index 63553f391..d1ac73615 100644 --- a/extensions/functions_aggregate_generic.yaml +++ b/extensions/functions_aggregate_generic.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_aggregate_generic +urn: urn:substrait:extension:io.substrait:functions_aggregate_generic aggregate_functions: - name: "count" description: Count a set of values diff --git a/extensions/functions_arithmetic.yaml b/extensions/functions_arithmetic.yaml index 8746f0392..57908ac1c 100644 --- a/extensions/functions_arithmetic.yaml +++ b/extensions/functions_arithmetic.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_arithmetic +urn: urn:substrait:extension:io.substrait:functions_arithmetic scalar_functions: - name: "add" diff --git a/extensions/functions_arithmetic_decimal.yaml b/extensions/functions_arithmetic_decimal.yaml index bc131a266..16ffe685a 100644 --- a/extensions/functions_arithmetic_decimal.yaml +++ b/extensions/functions_arithmetic_decimal.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_arithmetic_decimal +urn: urn:substrait:extension:io.substrait:functions_arithmetic_decimal scalar_functions: - name: "add" diff --git a/extensions/functions_boolean.yaml b/extensions/functions_boolean.yaml index fcbe56ca9..5b779f832 100644 --- a/extensions/functions_boolean.yaml +++ b/extensions/functions_boolean.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_boolean +urn: urn:substrait:extension:io.substrait:functions_boolean scalar_functions: - name: or diff --git a/extensions/functions_comparison.yaml b/extensions/functions_comparison.yaml index a07fd29e6..060bd141c 100644 --- a/extensions/functions_comparison.yaml +++ b/extensions/functions_comparison.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_comparison +urn: urn:substrait:extension:io.substrait:functions_comparison scalar_functions: - name: "not_equal" diff --git a/extensions/functions_datetime.yaml b/extensions/functions_datetime.yaml index 754454a9b..71371dc76 100644 --- a/extensions/functions_datetime.yaml +++ b/extensions/functions_datetime.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_datetime +urn: urn:substrait:extension:io.substrait:functions_datetime scalar_functions: - name: extract diff --git a/extensions/functions_geometry.yaml b/extensions/functions_geometry.yaml index 127978c58..1d202221a 100644 --- a/extensions/functions_geometry.yaml +++ b/extensions/functions_geometry.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_geometry +urn: urn:substrait:extension:io.substrait:functions_geometry types: - name: geometry structure: "BINARY" diff --git a/extensions/functions_list.yaml b/extensions/functions_list.yaml index f7776350f..7818f82b7 100644 --- a/extensions/functions_list.yaml +++ b/extensions/functions_list.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_list +urn: urn:substrait:extension:io.substrait:functions_list scalar_functions: - name: "transform" description: >- diff --git a/extensions/functions_logarithmic.yaml b/extensions/functions_logarithmic.yaml index d1f8dd93b..b05f5a581 100644 --- a/extensions/functions_logarithmic.yaml +++ b/extensions/functions_logarithmic.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_logarithmic +urn: urn:substrait:extension:io.substrait:functions_logarithmic scalar_functions: - name: "ln" diff --git a/extensions/functions_rounding.yaml b/extensions/functions_rounding.yaml index 0ca87133b..7220e2ae9 100644 --- a/extensions/functions_rounding.yaml +++ b/extensions/functions_rounding.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_rounding +urn: urn:substrait:extension:io.substrait:functions_rounding scalar_functions: - name: "ceil" diff --git a/extensions/functions_rounding_decimal.yaml b/extensions/functions_rounding_decimal.yaml index 057dc6099..837239f92 100644 --- a/extensions/functions_rounding_decimal.yaml +++ b/extensions/functions_rounding_decimal.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_rounding_decimal +urn: urn:substrait:extension:io.substrait:functions_rounding_decimal scalar_functions: - name: "ceil" diff --git a/extensions/functions_set.yaml b/extensions/functions_set.yaml index 3331b29f0..86cabbecb 100644 --- a/extensions/functions_set.yaml +++ b/extensions/functions_set.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_set +urn: urn:substrait:extension:io.substrait:functions_set scalar_functions: - name: "index_in" diff --git a/extensions/functions_string.yaml b/extensions/functions_string.yaml index 2cd5f84da..2c44a7eea 100644 --- a/extensions/functions_string.yaml +++ b/extensions/functions_string.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_string +urn: urn:substrait:extension:io.substrait:functions_string scalar_functions: - name: concat diff --git a/extensions/type_variations.yaml b/extensions/type_variations.yaml index 6f129b573..39585e08c 100644 --- a/extensions/type_variations.yaml +++ b/extensions/type_variations.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:type_variations +urn: urn:substrait:extension:io.substrait:type_variations type_variations: - parent: string name: dict4 diff --git a/extensions/unknown.yaml b/extensions/unknown.yaml index 3603d6871..b1c24fd62 100644 --- a/extensions/unknown.yaml +++ b/extensions/unknown.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:unknown +urn: urn:substrait:extension:io.substrait:unknown types: - name: unknown scalar_functions: diff --git a/proto/substrait/extensions/extensions.proto b/proto/substrait/extensions/extensions.proto index 672b8638d..7f5d0b8ab 100644 --- a/proto/substrait/extensions/extensions.proto +++ b/proto/substrait/extensions/extensions.proto @@ -16,9 +16,11 @@ message SimpleExtensionURN { // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics. uint32 extension_urn_anchor = 1; - // The extension URN that uniquely identifies this extension. This must follow the - // format extension:: and serves as the "namespace" of this extension. - // This must conform to the following regex: ^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$ + // The extension URN that uniquely identifies this extension. The canonical format + // is urn:substrait:extension::. For backwards compatibility, URNs + // beginning with "extension:" have "urn:substrait:" prepended before validation. + // The canonical form must be a valid RFC 8141 URN and conform to the regex: + // ^urn:substrait:extension:[a-z0-9_.-]+:[a-z0-9_.-]+$ string urn = 2; } diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index b93893eea..675e84801 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -13,14 +13,16 @@ Some kinds of primitives are so frequently extended that Substrait defines a sta * Window Functions * Table Functions -To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), and they will be referred to as `extension URNs` for clarity. +To extend these items, users can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. -This extension URN uses the format `extension::`, where: +The canonical form of an extension URN is `urn:substrait:extension::`, where: -- `OWNER` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions +- `NAMESPACE` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions - `ID` is the specific identifier for the extension (e.g., `functions_arithmetic`, `custom_types`) -These URNs must match the regex `^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$`. +For backwards compatibility, if a URN begins with `extension:` instead of `urn:substrait:extension:`, the prefix `urn:substrait:` is prepended before validation. This means `extension:io.substrait:functions_list` is equivalent to `urn:substrait:extension:io.substrait:functions_list`. + +Extension URNs in canonical form must be valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) URNs (with NID `substrait`) and must match the regex `^urn:substrait:extension:[a-z0-9_.-]+:[a-z0-9_.-]+$`. The YAML file is constructed according to the [YAML Schema](https://github.com/substrait-io/substrait/blob/main/text/simple_extensions_schema.yaml). Each definition in the file corresponds to the YAML-based serialization of the relevant data structure. If a user only wants to extend one of these types of objects (e.g. types), a developer does not have to provide definitions for the other extension points. diff --git a/site/docs/serialization/binary_serialization.md b/site/docs/serialization/binary_serialization.md index 64b30425e..533b97d66 100644 --- a/site/docs/serialization/binary_serialization.md +++ b/site/docs/serialization/binary_serialization.md @@ -22,7 +22,7 @@ For simple extensions, a plan references the extension URNs associated with the Simple extensions within a plan are split into three components: an extension URN, an extension declaration and a number of references. -* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. The extension URN must conform to the regex `^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$`. +* **Extension URN**: A unique identifier for the extension in the canonical format `urn:substrait:extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. For backwards compatibility, URNs beginning with `extension:` have `urn:substrait:` prepended before validation. See [Simple Extensions](../extensions/index.md#simple-extensions) for details. * **Extension Declaration**: A specific extension within a single YAML document. The declaration combines a reference to the associated extension URN along with a unique key identifying the specific item within that YAML document (see [Function Signature](../extensions/index.md#function-signature)). It also defines a declaration anchor. The anchor is a plan-specific unique value that the producer creates as a key to be referenced elsewhere. * **Extension Reference**: A specific instance or use of an extension declaration within the plan body. diff --git a/site/examples/extensions/any1_type_function.yaml b/site/examples/extensions/any1_type_function.yaml index 2263c2db5..edef731c6 100644 --- a/site/examples/extensions/any1_type_function.yaml +++ b/site/examples/extensions/any1_type_function.yaml @@ -1,5 +1,5 @@ # Example showing the 'any1' type - arguments must be of the same type -urn: extension:example:any1_type +urn: urn:substrait:extension:example:any1_type scalar_functions: - name: bar impls: diff --git a/site/examples/extensions/any_type_function.yaml b/site/examples/extensions/any_type_function.yaml index 1e0636462..0f5ffd517 100644 --- a/site/examples/extensions/any_type_function.yaml +++ b/site/examples/extensions/any_type_function.yaml @@ -1,5 +1,5 @@ # Example showing the 'any' type - arguments can be of any type -urn: extension:example:any_type +urn: urn:substrait:extension:example:any_type scalar_functions: - name: foo impls: diff --git a/site/examples/extensions/distance_functions.yaml b/site/examples/extensions/distance_functions.yaml index 214f1433c..bdee96ec3 100644 --- a/site/examples/extensions/distance_functions.yaml +++ b/site/examples/extensions/distance_functions.yaml @@ -1,4 +1,4 @@ -urn: extension:example:distance_functions +urn: urn:substrait:extension:example:distance_functions dependencies: ext: extension:io.substrait:extension_types scalar_functions: diff --git a/site/examples/extensions/double_function.yaml b/site/examples/extensions/double_function.yaml index f428fb1c7..f191be750 100644 --- a/site/examples/extensions/double_function.yaml +++ b/site/examples/extensions/double_function.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:example:double_function +urn: urn:substrait:extension:example:double_function scalar_functions: - name: "double" diff --git a/site/examples/extensions/lambda_function_example.yaml b/site/examples/extensions/lambda_function_example.yaml index cb28d0d23..2c24c4151 100644 --- a/site/examples/extensions/lambda_function_example.yaml +++ b/site/examples/extensions/lambda_function_example.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_list +urn: urn:substrait:extension:io.substrait:functions_list scalar_functions: - name: "transform" description: >- diff --git a/site/examples/extensions/metadata_example.yaml b/site/examples/extensions/metadata_example.yaml index a890def74..269b9f8fc 100644 --- a/site/examples/extensions/metadata_example.yaml +++ b/site/examples/extensions/metadata_example.yaml @@ -1,5 +1,5 @@ # Example showing the metadata field at multiple levels -urn: extension:io.substrait:metadata_examples +urn: urn:substrait:extension:io.substrait:metadata_examples metadata: version: 2.0 maintainer: example-team diff --git a/site/examples/types/point_with_datatype_param.yaml b/site/examples/types/point_with_datatype_param.yaml index d16d77319..737a5be23 100644 --- a/site/examples/types/point_with_datatype_param.yaml +++ b/site/examples/types/point_with_datatype_param.yaml @@ -1,5 +1,5 @@ # Compound user-defined type with a data type parameter -urn: extension:example:point_parameterized +urn: urn:substrait:extension:example:point_parameterized types: - name: point parameters: diff --git a/site/examples/types/point_with_enum_param.yaml b/site/examples/types/point_with_enum_param.yaml index f4e3b8fd0..ed8d3ca39 100644 --- a/site/examples/types/point_with_enum_param.yaml +++ b/site/examples/types/point_with_enum_param.yaml @@ -1,5 +1,5 @@ # Compound user-defined type with an enumeration parameter -urn: extension:example:point_enum_param +urn: urn:substrait:extension:example:point_enum_param types: - name: point parameters: diff --git a/site/examples/types/point_with_nstruct.yaml b/site/examples/types/point_with_nstruct.yaml index b320e0cab..eb0e04cdc 100644 --- a/site/examples/types/point_with_nstruct.yaml +++ b/site/examples/types/point_with_nstruct.yaml @@ -1,5 +1,5 @@ # Alternative way to define point structure using NSTRUCT syntax -urn: extension:example:point_nstruct +urn: urn:substrait:extension:example:point_nstruct types: - name: point structure: "NSTRUCT" diff --git a/site/examples/types/point_with_structure.yaml b/site/examples/types/point_with_structure.yaml index f1999e6f0..4b7b89c11 100644 --- a/site/examples/types/point_with_structure.yaml +++ b/site/examples/types/point_with_structure.yaml @@ -1,5 +1,5 @@ # User-defined point type with structure information -urn: extension:example:point_with_structure +urn: urn:substrait:extension:example:point_with_structure types: - name: point structure: diff --git a/site/examples/types/point_with_two_params.yaml b/site/examples/types/point_with_two_params.yaml index fd7585b32..36a0c65c0 100644 --- a/site/examples/types/point_with_two_params.yaml +++ b/site/examples/types/point_with_two_params.yaml @@ -1,5 +1,5 @@ # Compound user-defined type with two data type parameters -urn: extension:example:point_two_params +urn: urn:substrait:extension:example:point_two_params types: - name: point parameters: diff --git a/site/examples/types/tuple_optional_variadic.yaml b/site/examples/types/tuple_optional_variadic.yaml index f72565dd3..213c939fd 100644 --- a/site/examples/types/tuple_optional_variadic.yaml +++ b/site/examples/types/tuple_optional_variadic.yaml @@ -1,5 +1,5 @@ # Tuple type with optional variadic parameter (zero or more types) -urn: extension:example:tuple_variadic +urn: urn:substrait:extension:example:tuple_variadic types: - name: tuple parameters: diff --git a/site/examples/types/union_variadic.yaml b/site/examples/types/union_variadic.yaml index 4b39d23cd..bf8922403 100644 --- a/site/examples/types/union_variadic.yaml +++ b/site/examples/types/union_variadic.yaml @@ -1,5 +1,5 @@ # Union type with variadic parameter (one or more types) -urn: extension:example:union_variadic +urn: urn:substrait:extension:example:union_variadic types: - name: union parameters: diff --git a/site/examples/types/user_defined_point.yaml b/site/examples/types/user_defined_point.yaml index 311e99278..199dcb26e 100644 --- a/site/examples/types/user_defined_point.yaml +++ b/site/examples/types/user_defined_point.yaml @@ -1,5 +1,5 @@ # User-defined type example: a point type with two scalar functions -urn: extension:example:point_type +urn: urn:substrait:extension:example:point_type types: - name: "point" diff --git a/site/examples/types/vector_with_constraints.yaml b/site/examples/types/vector_with_constraints.yaml index 60f72c953..d962ad6f9 100644 --- a/site/examples/types/vector_with_constraints.yaml +++ b/site/examples/types/vector_with_constraints.yaml @@ -1,5 +1,5 @@ # Vector type with integer parameter constrained to 2 or 3 dimensions -urn: extension:example:vector_constrained +urn: urn:substrait:extension:example:vector_constrained types: - name: vector parameters: diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index dd3dca8d7..9d8961044 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -7,7 +7,7 @@ required: [urn] properties: urn: type: string - pattern: "^extension:[a-z0-9_.-]+:[a-z0-9_.-]+$" + pattern: "^urn:substrait:extension:[a-z0-9_.-]+:[a-z0-9_.-]+$" dependencies: # For reusing type classes and type variations from other extension files. # The keys are namespace identifiers that you can then use as dot-separated