From 40db4e0da61b42a42bc8e76ebcfa36e20e508297 Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Tue, 10 Sep 2024 20:41:44 +0300 Subject: [PATCH] Remove forward-compatibility promise and all reserved & private syntax (#883) * Remove forwards compatibility from stability guarantee * Drop reserved statements and expressions * Drop private-use annotations * Update tests * Clarify that deprecation is not removal --- spec/README.md | 34 +--- spec/data-model/README.md | 73 ++------ spec/data-model/message.dtd | 18 +- spec/data-model/message.json | 76 +------- spec/errors.md | 35 ---- spec/formatting.md | 58 ++---- spec/message.abnf | 38 +--- spec/registry.md | 4 +- spec/syntax.md | 174 +++-------------- test/README.md | 2 - test/schemas/v0/tests.schema.json | 2 - test/tests/syntax-errors.json | 70 ++++++- test/tests/syntax.json | 238 ------------------------ test/tests/unsupported-expressions.json | 53 ------ test/tests/unsupported-statements.json | 18 -- 15 files changed, 149 insertions(+), 744 deletions(-) delete mode 100644 test/tests/unsupported-expressions.json delete mode 100644 test/tests/unsupported-statements.json diff --git a/spec/README.md b/spec/README.md index b85cfa56f..76cc998cf 100644 --- a/spec/README.md +++ b/spec/README.md @@ -90,16 +90,7 @@ Updates to this specification MUST NOT specify the use of a fallback value for a that previously did not specify a fallback value. Updates to this specification will not change the syntactical meaning -of any syntax defined in this specification except for that syntax marked as -"reserved for future standardization". - -Updates to this specification will not assign any meaning to or change the syntactical -requirements for any private-use annotation. - -Updates to this specification will not remove any reserved keywords or sigils. - -Updates to this specification will not add any additional Unicode code points to -those in `reserved-annotation-start`. +of any syntax defined in this specification. Updates to this specification will not remove any functions defined in the default registry. @@ -114,13 +105,6 @@ defined in the default registry. > differences in implementation or changes to locale data > (such as due to the release of new CLDR versions). -Updates to this specification will not introduce message syntax that, -when parsed according to earlier versions of this specification, -would produce syntax or data model errors. -Messages that use syntax introduced in a future version of this specification -could produce resolution or message function errors -when formatted according to an earlier version of this specification. - Updates to this specification will only reserve, define, or require function names or function option names consisting of characters in the ranges a-z, A-Z, and 0-9. @@ -137,29 +121,19 @@ based on this version being invalid. > For example, existing interfaces or fields will not be removed. -Future versions of this specification will not introduce syntax that cannot be -represented by this version of the data model. - -> For example, a future version could introduce a new keyword. -> The future version's data model would provide an interface for that keyword -> while this version of the data model would parse the value into -> the interface `UnsupportedStatement`. -> Both data models would be "valid" in their context, -> but this version's would be missing any functionality for the new statement type. - > [!IMPORTANT] > This stability policy allows any of the following, non-exhaustive list, of changes > in future versions of this specification: +> - Future versions may define new syntax and structures +> that would not be supported by this version of the specification. > - Future versions may add additional structure or meaning to existing syntax. > - Future versions may define new keywords. -> - Future versions may define annotations that use portions of the `reserved-annotation` -> syntax. > - Future versions may make previously invalid messages valid. > - Future versions may define additional functions in the default registry > or may reserve the names of functions for the purposes of interoperability. > - Future versions may define additional options to existing functions. > - Future versions may define additional option values for existing options. -> - Future versions may deprecate functions, options, or option values. +> - Future versions may deprecate (but not remove) keywords, functions, options, or option values. > - Future versions of this specification may introduce changes > to the data model that would result in future data model representations > not being valid for implementations of this version of the data model. diff --git a/spec/data-model/README.md b/spec/data-model/README.md index 9084e0203..23ae649e2 100644 --- a/spec/data-model/README.md +++ b/spec/data-model/README.md @@ -17,11 +17,10 @@ Implementations that expose APIs supporting the production, consumption, or tran _message_ as a data structure are encouraged to use this data model. This data model provides these capabilities: -- any MessageFormat 2 message (including future versions) - can be parsed into this representation +- any MessageFormat 2.0 message can be parsed into this representation - this data model representation can be serialized as a well-formed -MessageFormat 2 message -- parsing a MessageFormat 2 message into a data model representation +MessageFormat 2.0 message +- parsing a MessageFormat 2.0 message into a data model representation and then serializing it results in an equivalently functional message This data model might also be used to: @@ -98,21 +97,8 @@ The `name` does not include the initial `$` of the _variable_. The `name` of an `InputDeclaration` MUST be the same as the `name` in the `VariableRef` of its `VariableExpression` `value`. -An `UnsupportedStatement` represents a statement not supported by the implementation. -Its `keyword` is a non-empty string name (i.e. not including the initial `.`). -If not empty, the `body` is the "raw" value (i.e. escape sequences are not processed) -starting after the keyword and up to the first _expression_, -not including leading or trailing whitespace. -The non-empty `expressions` correspond to the trailing _expressions_ of the _reserved statement_. - -> [!NOTE] -> Be aware that future versions of this specification -> might assign meaning to _reserved statement_ values. -> This would result in new interfaces being added to -> this data model. - ```ts -type Declaration = InputDeclaration | LocalDeclaration | UnsupportedStatement; +type Declaration = InputDeclaration | LocalDeclaration; interface InputDeclaration { type: "input"; @@ -125,13 +111,6 @@ interface LocalDeclaration { name: string; value: Expression; } - -interface UnsupportedStatement { - type: "unsupported-statement"; - keyword: string; - body?: string; - expressions: Expression[]; -} ``` In a `SelectMessage`, @@ -173,34 +152,26 @@ type Pattern = Array; type Expression = | LiteralExpression | VariableExpression - | FunctionExpression - | UnsupportedExpression; + | FunctionExpression; interface LiteralExpression { type: "expression"; arg: Literal; - annotation?: FunctionAnnotation | UnsupportedAnnotation; + function?: FunctionRef; attributes: Attributes; } interface VariableExpression { type: "expression"; arg: VariableRef; - annotation?: FunctionAnnotation | UnsupportedAnnotation; + function?: FunctionRef; attributes: Attributes; } interface FunctionExpression { type: "expression"; arg?: never; - annotation: FunctionAnnotation; - attributes: Attributes; -} - -interface UnsupportedExpression { - type: "expression"; - arg?: never; - annotation: UnsupportedAnnotation; + function: FunctionRef; attributes: Attributes; } ``` @@ -209,7 +180,7 @@ interface UnsupportedExpression { The `Literal` and `VariableRef` correspond to the the _literal_ and _variable_ syntax rules. When they are used as the `body` of an `Expression`, -they represent _expression_ values with no _annotation_. +they represent _expression_ values with no _function_. `Literal` represents all literal values, both _quoted literal_ and _unquoted literal_. The presence or absence of quotes is not preserved by the data model. @@ -229,14 +200,14 @@ interface VariableRef { } ``` -A `FunctionAnnotation` represents a _function_ _annotation_. +A `FunctionRef` represents a _function_. The `name` does not include the `:` starting sigil. `Options` is a key-value mapping containing options, -and is used to represent the _annotation_ and _markup_ _options_. +and is used to represent the _function_ and _markup_ _options_. ```ts -interface FunctionAnnotation { +interface FunctionRef { type: "function"; name: string; options: Options; @@ -245,31 +216,13 @@ interface FunctionAnnotation { type Options = Map; ``` -An `UnsupportedAnnotation` represents a -_private-use annotation_ not supported by the implementation or a _reserved annotation_. -The `source` is the "raw" value (i.e. escape sequences are not processed), -including the starting sigil. - -When parsing the syntax of a _message_ that includes a _private-use annotation_ -supported by the implementation, -the implementation SHOULD represent it in the data model -using an interface appropriate for the semantics and meaning -that the implementation attaches to that _annotation_. - -```ts -interface UnsupportedAnnotation { - type: "unsupported-annotation"; - source: string; -} -``` - ## Markup A `Markup` object has a `kind` of either `"open"`, `"standalone"`, or `"close"`, each corresponding to _open_, _standalone_, and _close_ _markup_. The `name` in these does not include the starting sigils `#` and `/` or the ending sigil `/`. -The `options` for markup use the same key-value mapping as `FunctionAnnotation`. +The `options` for markup use the same key-value mapping as `FunctionRef`. ```ts interface Markup { diff --git a/spec/data-model/message.dtd b/spec/data-model/message.dtd index 6c49c358a..2990da1a3 100644 --- a/spec/data-model/message.dtd +++ b/spec/data-model/message.dtd @@ -1,5 +1,5 @@ @@ -10,12 +10,6 @@ name NMTOKEN #REQUIRED > - - - @@ -24,8 +18,8 @@ @@ -33,14 +27,12 @@ - - + + - - diff --git a/spec/data-model/message.json b/spec/data-model/message.json index ae2867d2c..bcf3cd8d7 100644 --- a/spec/data-model/message.json +++ b/spec/data-model/message.json @@ -36,7 +36,7 @@ } }, - "function-annotation": { + "function": { "type": "object", "properties": { "type": { "const": "function" }, @@ -45,65 +45,17 @@ }, "required": ["type", "name"] }, - "unsupported-annotation": { - "type": "object", - "properties": { - "type": { "const": "unsupported-annotation" }, - "source": { "type": "string" } - }, - "required": ["type", "source"] - }, - "annotation": { - "oneOf": [ - { "$ref": "#/$defs/function-annotation" }, - { "$ref": "#/$defs/unsupported-annotation" } - ] - }, - - "literal-expression": { - "type": "object", - "properties": { - "type": { "const": "expression" }, - "arg": { "$ref": "#/$defs/literal" }, - "annotation": { "$ref": "#/$defs/annotation" }, - "attributes": { "$ref": "#/$defs/attributes" } - }, - "required": ["type", "arg"] - }, - "variable-expression": { - "type": "object", - "properties": { - "type": { "const": "expression" }, - "arg": { "$ref": "#/$defs/variable" }, - "annotation": { "$ref": "#/$defs/annotation" }, - "attributes": { "$ref": "#/$defs/attributes" } - }, - "required": ["type", "arg"] - }, - "function-expression": { - "type": "object", - "properties": { - "type": { "const": "expression" }, - "annotation": { "$ref": "#/$defs/function-annotation" }, - "attributes": { "$ref": "#/$defs/attributes" } - }, - "required": ["type", "annotation"] - }, - "unsupported-expression": { + "expression": { "type": "object", "properties": { "type": { "const": "expression" }, - "annotation": { "$ref": "#/$defs/unsupported-annotation" }, + "arg": { "$ref": "#/$defs/literal-or-variable" }, + "function": { "$ref": "#/$defs/function" }, "attributes": { "$ref": "#/$defs/attributes" } }, - "required": ["type", "annotation"] - }, - "expression": { "oneOf": [ - { "$ref": "#/$defs/literal-expression" }, - { "$ref": "#/$defs/variable-expression" }, - { "$ref": "#/$defs/function-expression" }, - { "$ref": "#/$defs/unsupported-expression" } + { "required": ["type", "arg"] }, + { "required": ["type", "function"] } ] }, @@ -148,26 +100,12 @@ }, "required": ["type", "name", "value"] }, - "unsupported-statement": { - "type": "object", - "properties": { - "type": { "const": "unsupported-statement" }, - "keyword": { "type": "string" }, - "body": { "type": "string" }, - "expressions": { - "type": "array", - "items": { "$ref": "#/$defs/expression" } - } - }, - "required": ["type", "keyword", "expressions"] - }, "declarations": { "type": "array", "items": { "oneOf": [ { "$ref": "#/$defs/input-declaration" }, - { "$ref": "#/$defs/local-declaration" }, - { "$ref": "#/$defs/unsupported-statement" } + { "$ref": "#/$defs/local-declaration" } ] } }, diff --git a/spec/errors.md b/spec/errors.md index b437be695..619f22558 100644 --- a/spec/errors.md +++ b/spec/errors.md @@ -254,41 +254,6 @@ a reference to a function which cannot be resolved. > * {{The value is not one.}} > ``` -### Unsupported Expression - -An **_Unsupported Expression_** error occurs when an expression uses -syntax reserved for future standardization, -or for private implementation use that is not supported by the current implementation. - -> For example, attempting to format this message -> would result in an _Unsupported Expression_ error -> because it includes a _reserved annotation_. -> -> ``` -> The value is {!horse}. -> ``` -> -> Attempting to format this message would result in an _Unsupported Expression_ error -> if done within a context that does not support the `^` private use sigil: -> -> ``` -> .match {|horse| ^private} -> 1 {{The value is one.}} -> * {{The value is not one.}} -> ``` - -### Unsupported Statement - -An **_Unsupported Statement_** error occurs when a message includes a _reserved statement_. - -> For example, attempting to format this message -> would result in an _Unsupported Statement_ error: -> -> ``` -> .some {|horse|} -> {{The message body}} -> ``` - ### Bad Selector A **_Bad Selector_** error occurs when a message includes a _selector_ diff --git a/spec/formatting.md b/spec/formatting.md index 03e8eb549..7d54b790f 100644 --- a/spec/formatting.md +++ b/spec/formatting.md @@ -115,18 +115,10 @@ and different implementations MAY choose to perform different levels of resoluti > Alternatively, it could be an instance of an ICU4J `FormattedNumber`, > or some other locally appropriate value. -Depending on the presence or absence of a _variable_ or _literal_ operand -and a _function_, _private-use annotation_, or _reserved annotation_, +Depending on the presence or absence of a _variable_ or _literal_ operand and a _function_, the resolved value of the _expression_ is determined as follows: -If the _expression_ contains a _reserved annotation_, -an _Unsupported Expression_ error is emitted and -a _fallback value_ is used as the resolved value of the _expression_. - -Else, if the _expression_ contains a _private-use annotation_, -its resolved value is defined according to the implementation's specification. - -Else, if the _expression_ contains an _annotation_, +If the _expression_ contains a _function_, its resolved value is defined by _function resolution_. Else, if the _expression_ consists of a _variable_, @@ -152,10 +144,10 @@ Else, the _expression_ consists of a _literal_. Its resolved value is defined by _literal resolution_. > **Note** -> This means that a _literal_ value with no _annotation_ +> This means that a _literal_ value with no _function_ > is always treated as a string. > To represent values that are not strings as a _literal_, -> an _annotation_ needs to be provided: +> a _function_ needs to be provided: > > ``` > .local $aNumber = {1234 :number} @@ -195,7 +187,7 @@ this MUST also be considered a failure. ### Function Resolution -To resolve an _expression_ with a _function_ _annotation_, +To resolve an _expression_ with a _function_, the following steps are taken: 1. If the _expression_ includes an _operand_, resolve its value. @@ -267,8 +259,8 @@ the following steps are taken: > > is currently implementation-dependent. > Depending on whether the options are preserved -> between the resolution of the first `:number` _annotation_ -> and the resolution of the second `:number` _annotation_, +> between the resolution of the first `:number` _function_ +> and the resolution of the second `:number` _function_, > a conformant implementation > could produce either "001.000" or "1.000" > @@ -350,12 +342,9 @@ A **_fallback value_** is the resolved value for an _expression_ that An _expression_ fails to resolve when: -- A _variable_ used as an _operand_ (with or without an _annotation_) fails to resolve. +- A _variable_ used as an _operand_ (with or without a _function_) fails to resolve. * Note that this does not include a _variable_ used as an _option_ value. -- A _function_ _annotation_ fails to resolve. -- A _private-use annotation_ is unsupported by the implementation or if - a _private-use annotation_ fails to resolve. -- The _expression_ has a _reserved annotation_. +- A _function_ fails to resolve. The _fallback value_ depends on the contents of the _expression_: @@ -369,9 +358,8 @@ The _fallback value_ depends on the contents of the _expression_: > In a context where `:func` fails to resolve, > `{42 :func}` resolves to the _fallback value_ `|42|` and > `{|C:\\| :func}` resolves to the _fallback value_ `|C:\\|`. - > In any context, `{|| @reserved}` resolves to the _fallback value_ `||`. -- _expression_ with _variable_ _operand_ referring to a local _declaration_ (with or without an _annotation_): +- _expression_ with _variable_ _operand_ referring to a local _declaration_ (with or without a _function_): the _value_ to which it resolves (which may already be a _fallback value_) > Examples: @@ -388,12 +376,12 @@ The _fallback value_ depends on the contents of the _expression_: > (transitively) resolves to the _fallback value_ `:now` and > the message formats to `{:now}`. -- _expression_ with _variable_ _operand_ not referring to a local _declaration_ (with or without an _annotation_): +- _expression_ with _variable_ _operand_ not referring to a local _declaration_ (with or without a _function_): U+0024 DOLLAR SIGN `$` followed by the _name_ of the _variable_ > Examples: - > In a context where `$var` fails to resolve, `{$var}` and `{$var :number}` and `{$var @reserved}` - > all resolve to the _fallback value_ `$var`. + > In a context where `$var` fails to resolve, `{$var}` and `{$var :number}` + > both resolve to the _fallback value_ `$var`. > In a context where `:func` fails to resolve, > the _pattern_'s _expression_ in `.input $arg {{{$arg :func}}}` > resolves to the _fallback value_ `$arg` and @@ -406,21 +394,6 @@ The _fallback value_ depends on the contents of the _expression_: > In a context where `:func` fails to resolve, `{:func}` resolves to the _fallback value_ `:func`. > In a context where `:ns:func` fails to resolve, `{:ns:func}` resolves to the _fallback value_ `:ns:func`. -- unsupported _private-use annotation_ or _reserved annotation_ with no _operand_: - the _annotation_ starting sigil - - > Examples: - > In any context, `{@reserved}` and `{@reserved |...|}` both resolve to the _fallback value_ `@`. - -- supported _private-use annotation_ with no _operand_: - the _annotation_ starting sigil, optionally followed by implementation-defined details - conforming with patterns in the other cases (such as quoting literals). - If details are provided, they SHOULD NOT leak potentially private information. - - > Examples: - > In a context where `^` expressions are used for comments, `{^▽^}` might resolve to the _fallback value_ `^`. - > In a context where `&` expressions are _function_-like macro invocations, `{&foo |...|}` might resolve to the _fallback value_ `&foo`. - - Otherwise: the U+FFFD REPLACEMENT CHARACTER `�` This is not currently used by any expression, but may apply in future revisions. @@ -431,9 +404,6 @@ _Pattern selection_ is not supported for _fallback values_. ## Pattern Selection -If the _message_ contains any _reserved statements_, -emit an _Unsupported Statement_ error. - If the _message_ being formatted is not _well-formed_ and _valid_, the result of pattern selection is a _pattern_ consisting of a single _fallback value_ using the _message_'s fallback string defined in the _formatting context_ @@ -622,7 +592,7 @@ _This section is non-normative._ #### Example 1 -Presuming a minimal implementation which only supports `:string` annotation +Presuming a minimal implementation which only supports `:string` _function_ which matches keys by using string comparison, and a formatting context in which the variable reference `$foo` resolves to the string `'foo'` and diff --git a/spec/message.abnf b/spec/message.abnf index 3377275da..8d26906ba 100644 --- a/spec/message.abnf +++ b/spec/message.abnf @@ -6,7 +6,7 @@ pattern = *(text-char / escaped-char / placeholder) placeholder = expression / markup complex-message = [s] *(declaration [s]) complex-body [s] -declaration = input-declaration / local-declaration / reserved-statement +declaration = input-declaration / local-declaration complex-body = quoted-pattern / matcher input-declaration = input [s] variable-expression @@ -21,16 +21,12 @@ variant = key *(s key) [s] quoted-pattern key = literal / "*" ; Expressions -expression = literal-expression - / variable-expression - / annotation-expression -literal-expression = "{" [s] literal [s annotation] *(s attribute) [s] "}" -variable-expression = "{" [s] variable [s annotation] *(s attribute) [s] "}" -annotation-expression = "{" [s] annotation *(s attribute) [s] "}" - -annotation = function - / private-use-annotation - / reserved-annotation +expression = literal-expression + / variable-expression + / function-expression +literal-expression = "{" [s] literal [s function] *(s attribute) [s] "}" +variable-expression = "{" [s] variable [s function] *(s attribute) [s] "}" +function-expression = "{" [s] function *(s attribute) [s] "}" markup = "{" [s] "#" identifier *(s option) *(s attribute) [s] ["/"] "}" ; open and standalone / "{" [s] "/" identifier *(s option) *(s attribute) [s] "}" ; close @@ -38,7 +34,7 @@ markup = "{" [s] "#" identifier *(s option) *(s attribute) [s] ["/"] "}" ; open ; Expression and literal parts function = ":" identifier *(s option) option = identifier [s] "=" [s] (literal / variable) -; Attributes are reserved for future standardization + attribute = "@" identifier [[s] "=" [s] (literal / variable)] variable = "$" name @@ -54,23 +50,6 @@ input = %s".input" local = %s".local" match = %s".match" -; Reserve additional .keywords for use by future versions of this specification. -reserved-statement = reserved-keyword [s reserved-body] 1*([s] expression) -; Note that the following production is a simplification, -; as this rule MUST NOT be considered to match existing keywords -; (`.input`, `.local`, and `.match`). -reserved-keyword = "." name - -; Reserve additional sigils for use by future versions of this specification. -reserved-annotation = reserved-annotation-start [[s] reserved-body] -reserved-annotation-start = "!" / "%" / "*" / "+" / "<" / ">" / "?" / "~" - -; Reserve sigils for private-use by implementations. -private-use-annotation = private-start [[s] reserved-body] -private-start = "^" / "&" -reserved-body = reserved-body-part *([s] reserved-body-part) -reserved-body-part = reserved-char / escaped-char / quoted-literal - ; Names and identifiers ; identifier matches https://www.w3.org/TR/REC-xml-names/#NT-QName ; name matches https://www.w3.org/TR/REC-xml-names/#NT-NCName but excludes U+FFFD @@ -89,7 +68,6 @@ name-char = name-start / DIGIT / "-" / "." simple-start-char = content-char / "@" / "|" text-char = content-char / s / "." / "@" / "|" quoted-char = content-char / s / "." / "@" / "{" / "}" -reserved-char = content-char / "." content-char = %x01-08 ; omit NULL (%x00), HTAB (%x09) and LF (%x0A) / %x0B-0C ; omit CR (%x0D) / %x0E-1F ; omit SP (%x20) diff --git a/spec/registry.md b/spec/registry.md index 1138f7e8a..9abedbb91 100644 --- a/spec/registry.md +++ b/spec/registry.md @@ -483,7 +483,7 @@ for examples. > [!IMPORTANT] > The exact behavior of exact literal match is only defined for non-zero-filled > integer values. -> Annotations that use fraction digits or significant digits might work in specific +> Functions that use fraction digits or significant digits might work in specific > implementation-defined ways. > Users should avoid depending on these types of keys in message selection. @@ -563,7 +563,7 @@ The function `:datetime` has these _style options_. _Field options_ describe which fields to include in the formatted output and what format to use for that field. -The implementation may use this _annotation_ to configure which fields +The implementation may use this _function_ to configure which fields appear in the formatted output. > [!NOTE] diff --git a/spec/syntax.md b/spec/syntax.md index 80b704bcb..da212e21a 100644 --- a/spec/syntax.md +++ b/spec/syntax.md @@ -187,15 +187,12 @@ _Declarations_ are optional: many messages will not contain any _declarations_. An **_input-declaration_** binds a _variable_ to an external input value. The _variable-expression_ of an _input-declaration_ -MAY include an _annotation_ that is applied to the external value. +MAY include a _function_ that is applied to the external value. A **_local-declaration_** binds a _variable_ to the resolved value of an _expression_. -For compatibility with later MessageFormat 2 specification versions, -_declarations_ MAY also include _reserved statements_. - ```abnf -declaration = input-declaration / local-declaration / reserved-statement +declaration = input-declaration / local-declaration input-declaration = input [s] variable-expression local-declaration = local s variable [s] "=" [s] expression ``` @@ -206,7 +203,7 @@ _Duplicate Declaration_ error during processing: - A _declaration_ MUST NOT bind a _variable_ that appears as a _variable_ anywhere within a previous _declaration_. - An _input-declaration_ MUST NOT bind a _variable_ - that appears anywhere within the _annotation_ of its _variable-expression_. + that appears anywhere within the _function_ of its _variable-expression_. - A _local-declaration_ MUST NOT bind a _variable_ that appears in its _expression_. A _local-declaration_ MAY overwrite an external input value as long as the @@ -214,46 +211,17 @@ external input value does not appear in a previous _declaration_. > [!NOTE] > These restrictions only apply to _declarations_. -> A _placeholder_ or _selector_ can apply a different annotation to a _variable_ +> A _placeholder_ or _selector_ can apply a different _function_ to a _variable_ > than one applied to the same _variable_ named in a _declaration_. > For example, this message is _valid_: > ``` > .input {$var :number maximumFractionDigits=0} > .match {$var :number maximumFractionDigits=2} -> 0 {{The selector can apply a different annotation to {$var} for the purposes of selection}} -> * {{A placeholder in a pattern can apply a different annotation to {$var :number maximumFractionDigits=3}}} +> 0 {{The selector can apply a different function to {$var} for the purposes of selection}} +> * {{A placeholder in a pattern can apply a different function to {$var :number maximumFractionDigits=3}}} > ``` > (See the [Errors](./errors.md) section for examples of invalid messages) -#### Reserved Statements - -A **_reserved statement_** reserves additional `.keywords` -for use by future versions of this specification. -Any such future keyword must start with `.`, -followed by two or more lower-case ASCII characters. - -The rest of the statement supports -a similarly wide range of content as _reserved annotations_, -but it MUST end with one or more _expressions_. - -```abnf -reserved-statement = reserved-keyword [s reserved-body] 1*([s] expression) -reserved-keyword = "." name -``` - -> [!NOTE] -> The `reserved-keyword` ABNF rule is a simplification, -> as it MUST NOT be considered to match any of the existing keywords -> `.input`, `.local`, or `.match`. - -This allows flexibility in future standardization, -as future definitions MAY define additional semantics and constraints -on the contents of these _reserved statements_. - -Implementations MUST NOT assign meaning or semantics to a _reserved statement_: -these are reserved for future standardization. -Implementations MUST NOT remove or alter the contents of a _reserved statement_. - ### Complex Body The **_complex body_** of a _complex message_ is the part that will be formatted. @@ -318,7 +286,6 @@ be preserved during formatting. simple-start-char = content-char / "@" / "|" text-char = content-char / s / "." / "@" / "|" quoted-char = content-char / s / "." / "@" / "{" / "}" -reserved-char = content-char / "." content-char = %x01-08 ; omit NULL (%x00), HTAB (%x09) and LF (%x0A) / %x0B-0C ; omit CR (%x0D) / %x0E-1F ; omit SP (%x20) @@ -375,9 +342,9 @@ otherwise, a corresponding _Data Model Error_ will be produced during processing The number of _keys_ on each _variant_ MUST be equal to the number of _selectors_. - _Missing Fallback Variant_: At least one _variant_ MUST exist whose _keys_ are all equal to the "catch-all" key `*`. -- _Missing Selector Annotation_: - Each _selector_ MUST have an _annotation_, - or contain a _variable_ that directly or indirectly references a _declaration_ with an _annotation_. +- _Missing Selector Function_: + Each _selector_ MUST have a _function_, + or contain a _variable_ that directly or indirectly references a _declaration_ with a _function_. - _Duplicate Variant_: Each _variant_ MUST use a list of _keys_ that is unique from that of all other _variants_ in the _message_. @@ -481,20 +448,20 @@ An _expression_ cannot contain another _expression_. An _expression_ MAY contain one more _attributes_. A **_literal-expression_** contains a _literal_, -optionally followed by an _annotation_. +optionally followed by a _function_. A **_variable-expression_** contains a _variable_, -optionally followed by an _annotation_. +optionally followed by a _function_. -An **_annotation-expression_** contains an _annotation_ without an _operand_. +A **_function-expression_** contains a _function_ without an _operand_. ```abnf -expression = literal-expression - / variable-expression - / annotation-expression -literal-expression = "{" [s] literal [s annotation] *(s attribute) [s] "}" -variable-expression = "{" [s] variable [s annotation] *(s attribute) [s] "}" -annotation-expression = "{" [s] annotation *(s attribute) [s] "}" +expression = literal-expression + / variable-expression + / function-expression +literal-expression = "{" [s] literal [s function] *(s attribute) [s] "}" +variable-expression = "{" [s] variable [s function] *(s attribute) [s] "}" +function-expression = "{" [s] function *(s attribute) [s] "}" ``` There are several types of _expression_ that can appear in a _message_. @@ -530,30 +497,20 @@ Additionally, an _input-declaration_ can contain a _variable-expression_. > This placeholder contains a function expression with a variable-valued option: {:function option=$variable} > ``` -### Annotation - -An **_annotation_** is part of an _expression_ containing either -a _function_ together with its associated _options_, or -a _private-use annotation_ or a _reserved annotation_. - -```abnf -annotation = function - / private-use-annotation - / reserved-annotation -``` +### Operand An **_operand_** is the _literal_ of a _literal-expression_ or the _variable_ of a _variable-expression_. -An _annotation_ can appear in an _expression_ by itself or following a single _operand_. -When following an _operand_, the _operand_ serves as input to the _annotation_. - #### Function -A **_function_** is named functionality in an _annotation_. +A **_function_** is named functionality in an _expression_. _Functions_ are used to evaluate, format, select, or otherwise process data values during formatting. +A _function_ can appear in an _expression_ by itself or following a single _operand_. +When following an _operand_, the _operand_ serves as input to the _function_. + Each _function_ is defined by the runtime's _function registry_. A _function_'s entry in the _function registry_ will define whether the _function_ is a _selector_ or formatter (or both), @@ -587,11 +544,11 @@ The _identifier_ is separated from the _value_ by an U+003D EQUALS SIGN `=` alon optional whitespace. The value of an _option_ can be either a _literal_ or a _variable_. -Multiple _options_ are permitted in an _annotation_. +Multiple _options_ are permitted in a _function_. _Options_ are separated from the preceding _function_ _identifier_ and from each other by whitespace. -Each _option_'s _identifier_ MUST be unique within the _annotation_: -an _annotation_ with duplicate _option_ _identifiers_ is not _valid_ +Each _option_'s _identifier_ MUST be unique within the _function_: +a _function_ with duplicate _option_ _identifiers_ is not _valid_ and will produce a _Duplicate Option Name_ error during processing. The order of _options_ is not significant. @@ -616,82 +573,6 @@ option = identifier [s] "=" [s] (literal / variable) > Today is {$date :datetime weekday=$dateStyle}! > ``` -#### Private-Use Annotations - -A **_private-use annotation_** is an _annotation_ whose syntax is reserved -for use by a specific implementation or by private agreement between multiple implementations. -Implementations MAY define their own meaning and semantics for _private-use annotations_. - -A _private-use annotation_ starts with either U+0026 AMPERSAND `&` or U+005E CIRCUMFLEX ACCENT `^`. - -Characters, including whitespace, are assigned meaning by the implementation. -The definition of escapes in the `reserved-body` production, used for the body of -a _private-use annotation_ is an affordance to implementations that -wish to use a syntax exactly like other functions. Specifically: - -- The characters `\`, `{`, and `}` MUST be escaped as `\\`, `\{`, and `\}` respectively - when they appear in the body of a _private-use annotation_. -- The character `|` is special: it SHOULD be escaped as `\|` in a _private-use annotation_, - but can appear unescaped as long as it is paired with another `|`. - This is an affordance to allow _literals_ to appear in the private use syntax. - -A _private-use annotation_ MAY be empty after its introducing sigil. - -```abnf -private-use-annotation = private-start [[s] reserved-body] -private-start = "^" / "&" -``` - -> [!NOTE] -> Users are cautioned that _private-use annotations_ cannot be reliably exchanged -> and can result in errors during formatting. -> It is generally a better idea to use the function registry -> to define additional formatting or annotation options. - -> Here are some examples of what _private-use_ sequences might look like: -> -> ``` -> Here's private use with an operand: {$foo &bar} -> Here's a placeholder that is entirely private-use: {&anything here} -> Here's a private-use function that uses normal function syntax: {$operand ^foo option=|literal|} -> The character \| has to be paired or escaped: {&private || |something between| or isolated: \| } -> Stop {& "translate 'stop' as a verb" might be a translator instruction or comment } -> Protect stuff in {^ph}{^/ph}private use{^ph}{^/ph} -> ``` - -#### Reserved Annotations - -A **_reserved annotation_** is an _annotation_ whose syntax is reserved -for future standardization. - -A _reserved annotation_ starts with a reserved character. -The remaining part of a _reserved annotation_, called a _reserved body_, -MAY be empty or contain arbitrary text that starts and ends with -a non-whitespace character. - -This allows maximum flexibility in future standardization, -as future definitions MAY define additional semantics and constraints -on the contents of these _annotations_. - -Implementations MUST NOT assign meaning or semantics to -an _annotation_ starting with `reserved-annotation-start`: -these are reserved for future standardization. -Whitespace before or after a _reserved body_ is not part of the _reserved body_. -Implementations MUST NOT remove or alter the contents of a _reserved body_, -including any interior whitespace, -but MAY remove or alter whitespace before or after the _reserved body_. - -While a reserved sequence is technically "well-formed", -unrecognized _reserved-annotations_ or _private-use-annotations_ have no meaning. - -```abnf -reserved-annotation = reserved-annotation-start [[s] reserved-body] -reserved-annotation-start = "!" / "%" / "*" / "+" / "<" / ">" / "?" / "~" - -reserved-body = reserved-body-part *([s] reserved-body-part) -reserved-body-part = reserved-char / escaped-char / quoted-literal -``` - ## Markup **_Markup_** _placeholders_ are _pattern_ parts @@ -907,8 +788,7 @@ An **_escape sequence_** is a two-character sequence starting with U+005C REVERSE SOLIDUS `\`. An _escape sequence_ allows the appearance of lexically meaningful characters -in the body of _text_, _quoted literal_, or _reserved_ -(which includes, in this case, _private-use_) sequences. +in the body of _text_ or _quoted literal_ sequences. Each _escape sequence_ represents the literal character immediately following the initial `\`. ```abnf diff --git a/test/README.md b/test/README.md index 8942185fa..dabde2aa2 100644 --- a/test/README.md +++ b/test/README.md @@ -40,8 +40,6 @@ to the error names used in ["MessageFormat 2.0 Errors"](../spec/errors.md) in th | Syntax Error | syntax-error | | Unknown Function | unknown-function | | Unresolved Variable | unresolved-variable | -| Unsupported Expression | unsupported-expression | -| Unsupported Statement | unsupported-statement | | Variant Key Mismatch | variant-key-mismatch | The "Message Function Error" error name used in the spec diff --git a/test/schemas/v0/tests.schema.json b/test/schemas/v0/tests.schema.json index 7b2056292..a0dd0a56e 100644 --- a/test/schemas/v0/tests.schema.json +++ b/test/schemas/v0/tests.schema.json @@ -345,8 +345,6 @@ "duplicate-variant", "unresolved-variable", "unknown-function", - "unsupported-expression", - "unsupported-statement", "bad-selector", "bad-operand", "bad-option", diff --git a/test/tests/syntax-errors.json b/test/tests/syntax-errors.json index 9f9f69e84..f01f5f1dc 100644 --- a/test/tests/syntax-errors.json +++ b/test/tests/syntax-errors.json @@ -178,6 +178,74 @@ }, { "src": ".match |x| * {{foo}}" - } + }, + { "src": ".n{a}{{}}" }, + { "src": "{^}" }, + { "src": "{!}" }, + { "src": ".n .{a}{{}}" }, + { "src": ".n. {a}{{}}" }, + { "src": ".n.{a}{b}{{}}" }, + { "src": "{!.}" }, + { "src": "{! .}" }, + { "src": "{%}" }, + { "src": "{*}" }, + { "src": "{+}" }, + { "src": "{<}" }, + { "src": "{>}" }, + { "src": "{?}" }, + { "src": "{~}" }, + { "src": "{^.}" }, + { "src": "{^ .}" }, + { "src": "{&}" }, + { "src": "{!.\\{}" }, + { "src": "{!. \\{}" }, + { "src": "{!|a|}" }, + { "src": "foo {+reserved}" }, + { "src": "foo {&private}" }, + { "src": "foo {?reserved @a @b=c}" }, + { "src": ".foo {42} {{bar}}" }, + { "src": ".foo{42}{{bar}}" }, + { "src": ".foo |}lit{| {42}{{bar}}" }, + { "src": ".i {1} {{}}" }, + { "src": ".l $y = {|bar|} {{}}" }, + { "src": ".l $x.y = {|bar|} {{}}" }, + { "src": "hello {|4.2| %number}" }, + { "src": "hello {|4.2| %n|um|ber}" }, + { "src": "{+42}" }, + { "src": "hello {|4.2| &num|be|r}" }, + { "src": "hello {|4.2| ^num|be|r}" }, + { "src": "hello {|4.2| +num|be|r}" }, + { "src": "hello {|4.2| ?num|be||r|s}" }, + { "src": "hello {|foo| !number}" }, + { "src": "hello {|foo| *number}" }, + { "src": "hello {?number}" }, + { "src": "{xyzz }" }, + { "src": "hello {$foo ~xyzz }" }, + { "src": "hello {$x xyzz }" }, + { "src": "{ !xyzz }" }, + { "src": "{~xyzz }" }, + { "src": "{ num x \\\\ abcde |aaa||3.14||42| r }" }, + { "src": "hello {$foo >num x \\\\ abcde |aaa||3.14| |42| r }" }, + { "src" : ".input{ $n ~ }{{{$n}}}" } ] } diff --git a/test/tests/syntax.json b/test/tests/syntax.json index 1bde167eb..2bb634680 100644 --- a/test/tests/syntax.json +++ b/test/tests/syntax.json @@ -169,12 +169,6 @@ "src": ".local $x ={a}.input{$y}{{}}", "exp": "" }, - { - "description": "message -> complex-message -> *(declaration [s]) complex-body -> declaration complex-body -> reserved-statement complex-body -> reserved-keyword expression -> \".\" name expression complex-body", - "src": ".n{a}{{}}", - "exp": "", - "expErrors": [ { "type": "unsupported-statement" } ] - }, { "description": "message -> complex-message -> complex-body -> matcher -> match-statement variant -> match selector key quoted-pattern -> \".match\" expression literal quoted-pattern", "src": ".match{a :f}a{{}}*{{}}", @@ -304,18 +298,6 @@ "exp": "{:f}", "expErrors": [{ "type": "unknown-function" }] }, - { - "description": "... annotation -> private-use-annotation -> private-start", - "src": "{^}", - "exp": "{^}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... annotation -> reserved-annotation -> reserved-annotation-start", - "src": "{!}", - "exp": "{!}", - "expErrors": [{ "type": "unsupported-expression" }] - }, { "description": "message -> simple-message -> simple-start pattern -> placeholder -> markup -> \"{\" s \"#\" identifier \"}\"", "src": "{ #a}", @@ -507,114 +489,6 @@ "src": "{0E-1}", "exp": "0E-1" }, - { - "description": "... reserved-statement -> reserved-keyword s reserved-body 1*([s] expression) -> reserved-keyword s reserved-body expression -> \".\" name s reserved-body-part expression -> \".\" name s reserved-char expression ...", - "src": ".n .{a}{{}}", - "exp": "", - "expErrors": [ { "type": "unsupported-statement" } ] - }, - { - "description": "... reserved-statement -> reserved-keyword reserved-body 1*([s] expression) -> reserved-keyword s reserved-body s expression -> \".\" name s reserved-body-part expression -> \".\" name s reserved-char expression ...", - "src": ".n. {a}{{}}", - "exp": "", - "expErrors": [ { "type": "unsupported-statement" } ] - }, - { - "description": "... reserved-statement -> reserved-keyword reserved-body 1*([s] expression) -> reserved-keyword reserved-body expression expression -> \".\" name reserved-body-part expression expression -> \".\" name s reserved-char expression expression ...", - "src": ".n.{a}{b}{{}}", - "exp": "", - "expErrors": [ { "type": "unsupported-statement" } ] - }, - { - "description": "... reserved-annotation -> reserved-annotation-start reserved-body -> \"!\" reserved-body-part -> \"!\" reserved-char ...", - "src": "{!.}", - "exp": "{!}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation -> reserved-annotation-start s reserved-body -> \"!\" s reserved-body-part -> \"!\" s reserved-char ...", - "src": "{! .}", - "exp": "{!}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation-start ...", - "src": "{%}", - "exp": "{%}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation-start ...", - "src": "{*}", - "exp": "{*}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation-start ...", - "src": "{+}", - "exp": "{+}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation-start ...", - "src": "{<}", - "exp": "{<}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation-start ...", - "src": "{>}", - "exp": "{>}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation-start ...", - "src": "{?}", - "exp": "{?}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation-start ...", - "src": "{~}", - "exp": "{~}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... private-use-annotation -> private-start reserved-body -> \"^\" reserved-body-part -> \"^\" reserved-char ...", - "src": "{^.}", - "exp": "{^}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... private-use-annotation -> private-start s reserved-body -> \"^\" s reserved-body-part -> \"^\" s reserved-char ...", - "src": "{^ .}", - "exp": "{^}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... private-start ...", - "src": "{&}", - "exp": "{&}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation -> reserved-annotation-start reserved-body -> \"!\" reserved-body-part reserved-body-part -> \"!\" reserved-char escaped-char ...", - "src": "{!.\\{}", - "exp": "{!}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation -> reserved-annotation-start reserved-body -> \"!\" reserved-body-part s reserved-body-part -> \"!\" reserved-char s escaped-char ...", - "src": "{!. \\{}", - "exp": "{!}", - "expErrors": [{ "type": "unsupported-expression" }] - }, - { - "description": "... reserved-annotation -> reserved-annotation-start reserved-body -> \"!\" reserved-body-part -> \"!\" quoted-literal ...", - "src": "{!|a|}", - "exp": "{!}", - "expErrors": [{ "type": "unsupported-expression" }] - }, { "src": "hello { world\t\n}", "exp": "hello world" @@ -820,118 +694,6 @@ } ] }, - { - "src": "foo {+reserved}", - "exp": "foo {+}", - "expParts": [ - { - "type": "literal", - "value": "foo " - }, - { - "type": "fallback", - "source": "+" - } - ], - "expErrors": [ - { - "type": "unsupported-expression" - } - ] - }, - { - "src": "foo {&private}", - "exp": "foo {&}", - "expParts": [ - { - "type": "literal", - "value": "foo " - }, - { - "type": "fallback", - "source": "&" - } - ], - "expErrors": [ - { - "type": "unsupported-expression" - } - ] - }, - { - "src": "foo {?reserved @a @b=c}", - "exp": "foo {?}", - "expParts": [ - { - "type": "literal", - "value": "foo " - }, - { - "type": "fallback", - "source": "?" - } - ], - "expErrors": [ - { - "type": "unsupported-expression" - } - ] - }, - { - "src": ".foo {42} {{bar}}", - "exp": "bar", - "expParts": [ - { - "type": "literal", - "value": "bar" - } - ], - "expErrors": [ - { - "type": "unsupported-statement" - } - ] - }, - { - "src": ".foo{42}{{bar}}", - "exp": "bar", - "expParts": [ - { - "type": "literal", - "value": "bar" - } - ], - "expErrors": [ - { - "type": "unsupported-statement" - } - ] - }, - { - "src": ".foo |}lit{| {42}{{bar}}", - "exp": "bar", - "expParts": [ - { - "type": "literal", - "value": "bar" - } - ], - "expErrors": [ - { - "type": "unsupported-statement" - } - ] - }, - { - "src": ".l $y = {|bar|} {{}}", - "exp": "", - "expParts": [], - "expErrors": [ - { - "type": "unsupported-statement" - } - ] - }, { "src": "{{trailing whitespace}} \n", "exp": "trailing whitespace" diff --git a/test/tests/unsupported-expressions.json b/test/tests/unsupported-expressions.json deleted file mode 100644 index f7d611509..000000000 --- a/test/tests/unsupported-expressions.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "scenario": "Reserved and private annotations", - "description": "Tests for unsupported expressions (reserved/private)", - "defaultTestProperties": { - "locale": "en-US", - "expErrors": [ - { - "type": "unsupported-expression" - } - ] - }, - "tests": [ - { "src": "hello {|4.2| %number}" }, - { "src": "hello {|4.2| %n|um|ber}" }, - { "src": "{+42}" }, - { "src": "hello {|4.2| &num|be|r}" }, - { "src": "hello {|4.2| ^num|be|r}" }, - { "src": "hello {|4.2| +num|be|r}" }, - { "src": "hello {|4.2| ?num|be||r|s}" }, - { "src": "hello {|foo| !number}" }, - { "src": "hello {|foo| *number}" }, - { "src": "hello {?number}" }, - { "src": "{xyzz }" }, - { "src": "hello {$foo ~xyzz }" }, - { "src": "hello {$x xyzz }" }, - { "src": "{ !xyzz }" }, - { "src": "{~xyzz }" }, - { "src": "{ num x \\\\ abcde |aaa||3.14||42| r }" }, - { "src": "hello {$foo >num x \\\\ abcde |aaa||3.14| |42| r }" }, - { "src" : ".input{ $n ~ }{{{$n}}}" } - ] -} - diff --git a/test/tests/unsupported-statements.json b/test/tests/unsupported-statements.json deleted file mode 100644 index d944aa0f7..000000000 --- a/test/tests/unsupported-statements.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "scenario": "Reserved statements", - "description": "Tests for unsupported statements", - "defaultTestProperties": { - "locale": "en-US", - "expErrors": [ - { - "type": "unsupported-statement" - } - ] - }, - "tests": [ - { "src" : ".i {1} {{}}" }, - { "src" : ".l $y = {|bar|} {{}}" }, - { "src" : ".l $x.y = {|bar|} {{}}" } - ] -} -