diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala index 927bb38987..e662e1ce6b 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/ChoiceGroup.scala @@ -159,7 +159,7 @@ abstract class ChoiceTermBase( * Open issues: * 1) Is alignment or leading/trailing skip to be considered syntax. Alignment might not be there. * 2) What about an empty sequence that only carries statement annotations such as dfdl:assert or - * dfdl:setVariable + * dfdl:setVariable * * This latter need to be allowed, because while they do not have known required syntax they do * have to be executed for side-effect. diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala index 0e9aaf57e9..d00cbc2b3f 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/LocalElementMixin.scala @@ -61,10 +61,8 @@ trait LocalElementMixin extends ParticleMixin with LocalElementGrammarMixin { else if (representation =:= Representation.Binary) true else false } - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + // we can rarely statically know if an endOfParent element must have non-zero length + case LengthKind.EndOfParent => false } res }.value diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala index a6a9bc0037..210797ed01 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/SequenceGroup.scala @@ -269,7 +269,6 @@ abstract class SequenceGroupTermBase(xml: Node, lexicalParent: SchemaComponent, case SequenceKind.Ordered => true case SequenceKind.Unordered => false } - } /** diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala index 7558483d5b..e093bbf223 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/Term.scala @@ -27,9 +27,7 @@ import org.apache.daffodil.lib.iapi.WarnID import org.apache.daffodil.lib.schema.annotation.props.Found import org.apache.daffodil.lib.schema.annotation.props.NotFound import org.apache.daffodil.lib.schema.annotation.props.SeparatorSuppressionPolicy -import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind -import org.apache.daffodil.lib.schema.annotation.props.gen.OccursCountKind -import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo +import org.apache.daffodil.lib.schema.annotation.props.gen.* /** * Mixin for objects that are shared, but have consistency checks to be run @@ -562,4 +560,11 @@ trait Term } } + final lazy val realElementChildren: Seq[ElementBase] = { + termChildren.flatMap { + case eb: ElementBase => Seq(eb) + case c: Choice => Nil + case mg: ModelGroup => mg.realElementChildren + } + } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala index 00ebdee130..721211a71b 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/AlignedMixin.scala @@ -456,7 +456,15 @@ trait AlignedMixin extends GrammarMixin { self: Term => } case LengthKind.Delimited => encodingLengthApprox case LengthKind.Pattern => encodingLengthApprox - case LengthKind.EndOfParent => LengthMultipleOf(1) // NYI + case LengthKind.EndOfParent => + // Technically, the approximate length of an EndOfParent element is the length + // of its parent minus the length of prior siblings. However, this value is only + // used to compute the ending alignment available to subsequent represented terms. + // Per spec (12.3.6), no represented element or model group may follow an + // EndOfParent element (only unrepresented elements such as IVC elements are + // permitted), so the ending alignment is never consumed. LengthMultipleOf(1) + // is a safe conservative value to return. + LengthMultipleOf(1) // If an element is lengthKind="prefixed", the element's length is the length // of the value of the prefix element, which can't be known till runtime case LengthKind.Prefixed => LengthMultipleOf(1) // NYI (see DAFFODIL-3066) diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala index 3a33fdfb07..0a27f8db80 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala @@ -23,6 +23,7 @@ import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.dsom.ExpressionCompilers import org.apache.daffodil.core.dsom.InitiatedTerminatedMixin import org.apache.daffodil.core.dsom.PrefixLengthQuasiElementDecl +import org.apache.daffodil.core.dsom.Root import org.apache.daffodil.core.grammar.primitives.* import org.apache.daffodil.core.runtime1.ElementBaseRuntime1Mixin import org.apache.daffodil.lib.exceptions.Assert @@ -252,6 +253,59 @@ trait ElementBaseGrammarMixin } final lazy val prefixedLengthBody = prefixedLengthElementDecl.parsedValue + def checkEndOfParentRestrictionsOnCurrentElement(optParentELU: Option[LengthUnits]): Unit = { + Assert.invariant(optParentELU.isDefined) + val parentELU = optParentELU.get + val currentElement: ElementBase = this + Assert.invariant(currentElement.lengthKind == LengthKind.EndOfParent) + schemaDefinitionWhen( + currentElement.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a dfdl:terminator." + ) + schemaDefinitionWhen( + currentElement.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a non-zero dfdl:trailingSkip." + ) + schemaDefinitionWhen( + currentElement.maxOccurs > 1, + "element is specified as dfdl:lengthKind=\"endOfParent\", but specifies a maxOccurs greater than 1." + ) + schemaDefinitionWhen( + currentElement.impliedRepresentation == Representation.Text + && (!currentElement.isKnownEncoding || !currentElement.knownEncodingIsFixedWidth || currentElement.knownEncodingWidthInBits != 8) + && (parentELU != LengthUnits.Characters), + "element is specified as dfdl:lengthKind=\"endOfParent\", but the element has text representation, and does not have a single-byte character set encoding, and the effective length units of the parent is not 'characters'." + ) + if (currentElement.isSimpleType) { + val meetsSimpleTypeRestrictions = (currentElement.primType eq PrimType.String) + || (currentElement.representation == Representation.Text) + || (currentElement.primType eq PrimType.HexBinary) + || (currentElement.representation == Representation.Binary + && ((currentElement.binaryNumberRep match { + case BinaryNumberRep.Packed => true + case BinaryNumberRep.Bcd => true + case BinaryNumberRep.Ibm4690Packed => true + case _ => false + }) || ( + currentElement.optionBinaryCalendarRep match { + case Some(bcr) => + bcr match { + case BinaryCalendarRep.Packed => true + case BinaryCalendarRep.Bcd => true + case BinaryCalendarRep.Ibm4690Packed => true + case _ => false + } + case None => + false // if no binary calendar rep, then we dont care about what it returns + } + ))) + schemaDefinitionWhen( + !meetsSimpleTypeRestrictions, + "element is a simple type specified as dfdl:lengthKind=\"endOfParent\", but isn't a string type, doesn't have text representation, isn't a hexbinary type, or doesn't have binary representation with packed decimal representation." + ) + } + } + /** * Quite tricky when we add padding or fill * @@ -604,8 +658,20 @@ trait ElementBaseGrammarMixin } case LengthKind.Pattern => schemaDefinitionError("Binary data elements cannot have lengthKind='pattern'.") + case LengthKind.EndOfParent if optionBinaryCalendarRep.isDefined => + binaryCalendarRep match { + case BinaryCalendarRep.BinaryMilliseconds | BinaryCalendarRep.BinarySeconds => + SDE("lengthKind='endOfParent' only supported for packed binary formats.") + case _ => -1 + } + case LengthKind.EndOfParent if optionBinaryNumberRep.isDefined => + binaryNumberRep match { + case BinaryNumberRep.Binary => + SDE("lengthKind='endOfParent' only supported for packed binary formats.") + case _ => -1 + } case LengthKind.EndOfParent => - schemaDefinitionError("Binary data elements cannot have lengthKind='endOfParent'.") + SDE("lengthKind='endOfParent' only supported for packed binary formats.") } private def explicitBinaryLengthInBits() = { @@ -648,10 +714,7 @@ trait ElementBaseGrammarMixin Assert.invariant(pt == PrimType.String) StringOfSpecifiedLength(this) } - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + case LengthKind.EndOfParent => StringOfSpecifiedLength(this) } } @@ -667,6 +730,10 @@ trait ElementBaseGrammarMixin new HexBinaryLengthPrefixed(this) } + private lazy val hexBinaryLengthEndOfParent = prod("hexBinaryLengthEndOfParent") { + new HexBinaryLengthEndOfParent(this) + } + private lazy val hexBinaryValue = prod("hexBinaryValue") { schemaDefinitionWhen( lengthUnits == LengthUnits.Characters, @@ -678,10 +745,7 @@ trait ElementBaseGrammarMixin case LengthKind.Delimited => hexBinaryDelimitedEndOfData case LengthKind.Pattern => hexBinaryLengthPattern case LengthKind.Prefixed => hexBinaryLengthPrefixed - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + case LengthKind.EndOfParent => hexBinaryLengthEndOfParent } } @@ -793,6 +857,10 @@ trait ElementBaseGrammarMixin prod("bcdPrefixedLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Bcd) { ConvertZonedCombinator(this, new BCDIntegerPrefixedLength(this), textConverter) } + private lazy val bcdEndOfParentLengthCalendar = + prod("bcdEndOfParentLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Bcd) { + ConvertZonedCombinator(this, new BCDIntegerEndOfParentLength(this), textConverter) + } private lazy val ibm4690PackedKnownLengthCalendar = prod( "ibm4690PackedKnownLengthCalendar", @@ -834,6 +902,16 @@ trait ElementBaseGrammarMixin textConverter ) } + private lazy val ibm4690PackedEndOfParentLengthCalendar = prod( + "ibm4690PackedEndOfParentLengthCalendar", + binaryCalendarRep == BinaryCalendarRep.Ibm4690Packed + ) { + ConvertZonedCombinator( + this, + new IBM4690PackedIntegerEndOfParentLength(this), + textConverter + ) + } private lazy val packedKnownLengthCalendar = prod("packedKnownLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Packed) { @@ -871,6 +949,14 @@ trait ElementBaseGrammarMixin textConverter ) } + private lazy val packedEndOfParentLengthCalendar = + prod("packedPrefixedLengthCalendar", binaryCalendarRep == BinaryCalendarRep.Packed) { + ConvertZonedCombinator( + this, + new PackedIntegerEndOfParentLength(this, packedSignCodes), + textConverter + ) + } def primType: PrimType @@ -939,6 +1025,8 @@ trait ElementBaseGrammarMixin new PackedIntegerDelimitedEndOfData(this, packedSignCodes) case (BinaryNumberRep.Packed, LengthKind.Prefixed, -1) => new PackedIntegerPrefixedLength(this, packedSignCodes) + case (BinaryNumberRep.Packed, LengthKind.EndOfParent, -1) => + new PackedIntegerEndOfParentLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, -1) => new PackedIntegerRuntimeLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, _) => @@ -951,6 +1039,8 @@ trait ElementBaseGrammarMixin new IBM4690PackedIntegerDelimitedEndOfData(this) case (BinaryNumberRep.Ibm4690Packed, LengthKind.Prefixed, -1) => new IBM4690PackedIntegerPrefixedLength(this) + case (BinaryNumberRep.Ibm4690Packed, LengthKind.EndOfParent, -1) => + new IBM4690PackedIntegerEndOfParentLength(this) case (BinaryNumberRep.Ibm4690Packed, _, -1) => new IBM4690PackedIntegerRuntimeLength(this) case (BinaryNumberRep.Ibm4690Packed, _, _) => @@ -963,6 +1053,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => new BCDIntegerDelimitedEndOfData(this) case (LengthKind.Prefixed, -1) => new BCDIntegerPrefixedLength(this) + case (LengthKind.EndOfParent, -1) => new BCDIntegerEndOfParentLength(this) case (_, -1) => new BCDIntegerRuntimeLength(this) case (_, _) => new BCDIntegerKnownLength(this, binaryNumberKnownLengthInBits) } @@ -1030,6 +1121,8 @@ trait ElementBaseGrammarMixin ) byteOrderRaw // must have or SDE (binaryNumberRep, lengthKind, binaryNumberKnownLengthInBits) match { + case (BinaryNumberRep.Binary, LengthKind.EndOfParent, _) => + SDE("lengthKind='endOfParent' is not allowed with binary number representation") case (BinaryNumberRep.Binary, LengthKind.Prefixed, _) => new BinaryDecimalPrefixedLength(this) case (BinaryNumberRep.Binary, _, -1) => new BinaryDecimalRuntimeLength(this) @@ -1047,6 +1140,8 @@ trait ElementBaseGrammarMixin new PackedDecimalDelimitedEndOfData(this, packedSignCodes) case (BinaryNumberRep.Packed, LengthKind.Prefixed, _) => new PackedDecimalPrefixedLength(this, packedSignCodes) + case (BinaryNumberRep.Packed, LengthKind.EndOfParent, _) => + new PackedDecimalEndOfParentLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, -1) => new PackedDecimalRuntimeLength(this, packedSignCodes) case (BinaryNumberRep.Packed, _, _) => @@ -1055,6 +1150,8 @@ trait ElementBaseGrammarMixin new BCDDecimalDelimitedEndOfData(this) case (BinaryNumberRep.Bcd, LengthKind.Prefixed, _) => new BCDDecimalPrefixedLength(this) + case (BinaryNumberRep.Bcd, LengthKind.EndOfParent, _) => + new BCDDecimalEndOfParentLength(this) case (BinaryNumberRep.Bcd, _, -1) => new BCDDecimalRuntimeLength(this) case (BinaryNumberRep.Bcd, _, _) => new BCDDecimalKnownLength(this, binaryNumberKnownLengthInBits) @@ -1062,6 +1159,8 @@ trait ElementBaseGrammarMixin new IBM4690PackedDecimalDelimitedEndOfData(this) case (BinaryNumberRep.Ibm4690Packed, LengthKind.Prefixed, _) => new IBM4690PackedDecimalPrefixedLength(this) + case (BinaryNumberRep.Ibm4690Packed, LengthKind.EndOfParent, _) => + new IBM4690PackedDecimalEndOfParentLength(this) case (BinaryNumberRep.Ibm4690Packed, _, -1) => new IBM4690PackedDecimalRuntimeLength(this) case (BinaryNumberRep.Ibm4690Packed, _, _) => @@ -1072,6 +1171,13 @@ trait ElementBaseGrammarMixin case PrimType.Boolean => { lengthKind match { case LengthKind.Prefixed => new BinaryBooleanPrefixedLength(this) + case LengthKind.EndOfParent => + // xs:boolean is not one of the allowed simple types for lengthKind='endOfParent' + // (spec 12.3.6: only xs:string, text representation, xs:hexBinary, or binary + // with packed decimal representation are permitted). + SDE( + "lengthKind='endOfParent' is not supported for xs:boolean with binary representation." + ) case _ => new BinaryBoolean(this) } } @@ -1125,6 +1231,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => bcdDelimitedLengthCalendar case (LengthKind.Prefixed, -1) => bcdPrefixedLengthCalendar + case (LengthKind.EndOfParent, -1) => bcdEndOfParentLengthCalendar case (_, -1) => bcdRuntimeLengthCalendar case (_, _) => bcdKnownLengthCalendar } @@ -1133,6 +1240,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => ibm4690PackedDelimitedLengthCalendar case (LengthKind.Prefixed, -1) => ibm4690PackedPrefixedLengthCalendar + case (LengthKind.EndOfParent, -1) => ibm4690PackedEndOfParentLengthCalendar case (_, -1) => ibm4690PackedRuntimeLengthCalendar case (_, _) => ibm4690PackedKnownLengthCalendar } @@ -1141,6 +1249,7 @@ trait ElementBaseGrammarMixin (lengthKind, binaryNumberKnownLengthInBits) match { case (LengthKind.Delimited, -1) => packedDelimitedLengthCalendar case (LengthKind.Prefixed, -1) => packedPrefixedLengthCalendar + case (LengthKind.EndOfParent, -1) => packedEndOfParentLengthCalendar case (_, -1) => packedRuntimeLengthCalendar case (_, _) => packedKnownLengthCalendar } @@ -1280,10 +1389,7 @@ trait ElementBaseGrammarMixin case LengthKind.Implicit => LiteralValueNilOfSpecifiedLength(this) case LengthKind.Prefixed => LiteralValueNilOfSpecifiedLength(this) - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") - case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + case LengthKind.EndOfParent => LiteralValueNilOfSpecifiedLength(this) } } case NilKind.LiteralCharacter => { @@ -1341,18 +1447,29 @@ trait ElementBaseGrammarMixin // non-explicit lengthKind val body = bodyArg + val eopSimpleTypeElementThatNeedsBitLimit = + (isSimpleType && lengthKind == LengthKind.EndOfParent) + && !this.isInstanceOf[Root] + && impliedRepresentation != Representation.Text + && !isNillable + && primType != PrimType.HexBinary // there are essentially two categories of processors that read/write data input/output - // stream: those that calculate lengths themselves and those that expect another - // processor to calculate the length and set the bit limit which this processor will use as - // the length. The following determines if this element requires another processor to - // calculate and set the bit limit, and if so adds the appropriate grammar to do that - val bodyRequiresSpecifiedLengthBitLimit = lengthKind != LengthKind.Delimited && ( - isSimpleType && impliedRepresentation == Representation.Text || - isSimpleType && isNillable || - isComplexType && lengthKind != LengthKind.Implicit || - lengthKind == LengthKind.Prefixed || - isSimpleType && primType == PrimType.HexBinary && lengthKind == LengthKind.Pattern - ) + // stream: those that calculate lengths themselves (ex: binary numeric parsers) and those + // that expect another processor to calculate the length and set the bit limit which + // this processor will use as the length (such as text parsers). The following determines + // if this element requires another processor to calculate and set the bit limit, and if so + // adds the appropriate grammar to do that + val bodyRequiresSpecifiedLengthBitLimit = lengthKind != LengthKind.Delimited + // Note for non-root EndOfParent simple types, we don't wish to duplicate the length + // calculation efforts unless we know it needs the bit limit set by a parent + && !eopSimpleTypeElementThatNeedsBitLimit + && ( + isSimpleType && impliedRepresentation == Representation.Text || + isSimpleType && isNillable || + isComplexType && lengthKind != LengthKind.Implicit || + lengthKind == LengthKind.Prefixed || + isSimpleType && primType == PrimType.HexBinary && lengthKind == LengthKind.Pattern + ) if (!bodyRequiresSpecifiedLengthBitLimit) { body } else { @@ -1396,10 +1513,8 @@ trait ElementBaseGrammarMixin case LengthKind.Implicit if isSimpleType && impliedRepresentation == Representation.Binary => new SpecifiedLengthImplicit(this, body, implicitBinaryLengthInBits) - case LengthKind.EndOfParent if isComplexType => - notYetImplemented("lengthKind='endOfParent' for complex type") case LengthKind.EndOfParent => - notYetImplemented("lengthKind='endOfParent' for simple type") + new SpecifiedLengthEndOfParent(this, body) case LengthKind.Delimited | LengthKind.Implicit => Assert.impossibleCase( "Delimited and ComplexType Implicit cases should not be reached" diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala index c1c3febadb..0d2733f441 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementDeclGrammarMixin.scala @@ -19,11 +19,14 @@ package org.apache.daffodil.core.grammar import org.apache.daffodil.core.dsom.Root import org.apache.daffodil.core.grammar.primitives.UnicodeByteOrderMark +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits trait RootGrammarMixin extends LocalElementGrammarMixin // can be repeating if not root { self: Root => + requiredEvaluationsAlways(checkEndOfParentRestrictions(Some(LengthUnits.Characters))) + final lazy val document = prod("document") { schemaDefinitionUnless(isScalar, "The document element cannot be an array.") UnicodeByteOrderMark(this) ~ documentElement diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala index 995a97cd87..f489adf7a8 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/TermGrammarMixin.scala @@ -17,9 +17,20 @@ package org.apache.daffodil.core.grammar +import org.apache.daffodil.core.dsom.ChoiceTermBase +import org.apache.daffodil.core.dsom.ElementBase +import org.apache.daffodil.core.dsom.ModelGroup +import org.apache.daffodil.core.dsom.Root +import org.apache.daffodil.core.dsom.SequenceTermBase import org.apache.daffodil.core.dsom.Term import org.apache.daffodil.core.grammar.primitives.MandatoryTextAlignment import org.apache.daffodil.core.runtime1.TermRuntime1Mixin +import org.apache.daffodil.lib.schema.annotation.props.gen.ChoiceLengthKind +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthKind +import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits +import org.apache.daffodil.lib.schema.annotation.props.gen.SeparatorPosition +import org.apache.daffodil.lib.schema.annotation.props.gen.SequenceKind +import org.apache.daffodil.lib.schema.annotation.props.gen.YesNo ///////////////////////////////////////////////////////////////// // Common to all Terms (Elements and ModelGroups) @@ -84,4 +95,149 @@ trait TermGrammarMixin extends AlignedMixin with BitOrderMixin with TermRuntime1 MandatoryTextAlignment(this, knownEncodingAlignmentInBits, true) } + def optEffectiveLengthUnits(optLastNonEOPELU: Option[LengthUnits]): Option[LengthUnits] = { + this match { + case e: ElementBase => + e.lengthKind match { + case LengthKind.EndOfParent => optLastNonEOPELU + case LengthKind.Explicit | LengthKind.Prefixed => Some(e.lengthUnits) + case LengthKind.Pattern => Some(LengthUnits.Characters) + case LengthKind.Implicit | LengthKind.Delimited => + None // invalid parent; SDE fires separately + } + case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => + Some(LengthUnits.Bytes) + // Sequences are transparent — the ELU is inherited from the nearest enclosing box. + case _: SequenceTermBase => optLastNonEOPELU + case _: ChoiceTermBase => None // implicit-length choice; SDE fires separately + } + } + + final lazy val childrenEndOfParent: Seq[ElementBase] = LV(Symbol("childrenEndOfParent")) { + val gms = termChildren + val chls = gms.flatMap { + case eb: ElementBase if eb.lengthKind == LengthKind.EndOfParent => Seq(eb) + case eb: ElementBase => Nil + case c: ChoiceTermBase => Nil + case mg: ModelGroup => mg.childrenEndOfParent + } + chls + }.value + + def checkEndOfParentRestrictions(lastNonEOPELU: Option[LengthUnits]): Boolean = { + val term = this + lazy val eopChildren = this.childrenEndOfParent + lazy val optParentELU = term.optEffectiveLengthUnits(lastNonEOPELU) + // checks + term match { + case rootElem: Root if rootElem.lengthKind == LengthKind.EndOfParent => { + rootElem.checkEndOfParentRestrictionsOnCurrentElement(Some(LengthUnits.Characters)) + eopChildren.foreach { child => + child.checkEndOfParentRestrictionsOnCurrentElement(lastNonEOPELU) + } + } + case e: ElementBase if eopChildren.nonEmpty => { + eopChildren.foreach { child => + child.schemaDefinitionWhen( + e.lengthKind == LengthKind.Implicit || e.lengthKind == LengthKind.Delimited, + "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is an element with dfdl:lengthKind 'implicit' or 'delimited'." + ) + child.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) + } + } + case s: SequenceTermBase if eopChildren.nonEmpty => { + eopChildren.foreach { child => + child.schemaDefinitionWhen( + s.separatorPosition == SeparatorPosition.Postfix, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:separatorPosition defined as 'postfix'." + ) + child.schemaDefinitionWhen( + s.sequenceKind != SequenceKind.Ordered, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with dfdl:sequenceKind defined as 'unordered'." + ) + child.schemaDefinitionWhen( + s.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a dfdl:terminator." + ) + child.schemaDefinitionWhen( + s.realElementChildren.exists(e => e.floating == YesNo.Yes), + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with elements defining dfdl:floating='yes'." + ) + child.schemaDefinitionWhen( + s.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a sequence with a non-zero dfdl:trailingSkip." + ) + } + + } + case c: ChoiceTermBase if eopChildren.nonEmpty => { + // TODO: The DFDL spec (12.3.6) explicitly mentions that an EndOfParent element + // can be terminated by a choice with choiceLengthKind='explicit', with no reference + // to implicit length choices. Later on in 12.3.6, it specified what the parent Effective + // Length Units would be for an explicit length choice, again with no reference to + // implicit length choices. The only way to get an EndOfParent element within an + // implicit length choice would be to have it actually be terminated by the choice's + // parent, similar to how we treat elements within an element that is also EndOfParent. + // The only mention of implicit length choices in the DFDL Spec is to mention SDEs when + // an EndOfParent element is enclosed by a choice with choiceLengthKind='implicit'. We + // think that may be a typo, so for now we disallow ChoiceLengthKind.Implicit + // enclosing an EndOfParent element. + // See Daffodil-3080 + eopChildren.foreach { child => + child.schemaDefinitionWhen( + c.choiceLengthKind == ChoiceLengthKind.Implicit, + "element is specified as dfdl:lengthKind=\"endOfParent\", but its parent is a choice with dfdl:choiceLengthKind 'implicit'." + ) + child.schemaDefinitionWhen( + c.hasTerminator, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a dfdl:terminator." + ) + child.schemaDefinitionWhen( + c.trailingSkip != 0, + "element is specified as dfdl:lengthKind=\"endOfParent\", but is in a choice with a non-zero dfdl:trailingSkip." + ) + child.checkEndOfParentRestrictionsOnCurrentElement(optParentELU) + } + } + case _ => // do nothing + } + // end checks + val sawEOP = term.termChildren.foldLeft(false) { case (sawEOP, child) => + if (sawEOP) { + // Choice branches are alternatives, not sequential data — the after-EOP SDE must + // not fire across branches. Only sequences and elements have sequential ordering. + term match { + case _: ChoiceTermBase => // has alternatives which can all be EOP; skip + case _ => + child.schemaDefinitionWhen( + child.isInstanceOf[ModelGroup], + "element is specified as dfdl:lengthKind=\"endOfParent\", but a model group is defined between this element and the end of the enclosing component" + ) + child.schemaDefinitionWhen( + child.isRepresented, + "element is specified as dfdl:lengthKind=\"endOfParent\", but a represented element is defined between this element and the end of the enclosing component" + ) + } + } + val res = child.checkEndOfParentRestrictions(optParentELU) + term match { + // Branches of a choice are alternatives, so should never carry sawEOP from one branch to the next. + case _: ChoiceTermBase => false + case _ => + child match { + case e: ElementBase if e.lengthKind == LengthKind.EndOfParent => true + // A non-EOP element forms a hard length boundary. EOP elements nested inside it are + // scoped to that element's length, so they must not affect sibling scanning here. + case _: ElementBase => false + // An explicit-length choice owns a fixed byte span, so EOP elements inside it are + // scoped to that span. After the choice ends the parent continues normally. + case c: ChoiceTermBase if c.choiceLengthKind == ChoiceLengthKind.Explicit => false + // Sequences are transparent (no own length boundary), + // so EOP state propagates through them. + case _ => res + } + } + } + sawEOP + } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala index e9e0996b00..3052f25e4d 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesBCD.scala @@ -20,9 +20,11 @@ package org.apache.daffodil.core.grammar.primitives import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDDecimalRuntimeLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerRuntimeLengthParser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -52,7 +54,18 @@ class BCDIntegerKnownLength(val e: ElementBase, lengthInBits: Long) extends Term class BCDIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, true) { - override lazy val parser = new BCDIntegerBitLimitLengthParser(e.elementRuntimeData) + override lazy val parser = + new BCDIntegerBitLimitLengthParser(e.elementRuntimeData) + + override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser( + e.elementRuntimeData + ) +} + +class BCDIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + + override lazy val parser = + new BCDIntegerEndOfParentLengthParser(e.elementRuntimeData) override lazy val unparser: Unparser = new BCDIntegerMinimumLengthUnparser( e.elementRuntimeData @@ -93,10 +106,19 @@ class BCDDecimalKnownLength(val e: ElementBase, lengthInBits: Long) extends Term class BCDDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = - new BCDDecimalBitLimitLengthParser( + new BCDDecimalBitLimitLengthParser(e.elementRuntimeData, e.binaryDecimalVirtualPoint) + + override lazy val unparser: Unparser = + new BCDDecimalMinimumLengthUnparser( e.elementRuntimeData, e.binaryDecimalVirtualPoint ) +} + +class BCDDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + + override lazy val parser = + new BCDDecimalEndOfParentLengthParser(e.elementRuntimeData, e.binaryDecimalVirtualPoint) override lazy val unparser: Unparser = new BCDDecimalMinimumLengthUnparser( diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala index 91d0d25efe..78332ef333 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesIBM4690Packed.scala @@ -20,9 +20,11 @@ package org.apache.daffodil.core.grammar.primitives import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalRuntimeLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerRuntimeLengthParser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -66,6 +68,15 @@ class IBM4690PackedIntegerPrefixedLength(val e: ElementBase) extends Terminal(e, ) } +class IBM4690PackedIntegerEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + override lazy val parser = + new IBM4690PackedIntegerEndOfParentLengthParser(e.elementRuntimeData) + + override lazy val unparser: Unparser = new IBM4690PackedIntegerMinimumLengthUnparser( + e.elementRuntimeData + ) +} + class IBM4690PackedDecimalRuntimeLength(val e: ElementBase) extends Terminal(e, true) { override lazy val parser = new IBM4690PackedDecimalRuntimeLengthParser( e.elementRuntimeData, @@ -115,3 +126,17 @@ class IBM4690PackedDecimalPrefixedLength(val e: ElementBase) extends Terminal(e, e.decimalSigned ) } + +class IBM4690PackedDecimalEndOfParentLength(val e: ElementBase) extends Terminal(e, true) { + override lazy val parser = new IBM4690PackedDecimalEndOfParentLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + e.decimalSigned + ) + + override lazy val unparser: Unparser = new IBM4690PackedDecimalMinimumLengthUnparser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + e.decimalSigned + ) +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala index e069353bd5..3c9ae08786 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesLengthKind.scala @@ -34,6 +34,7 @@ import org.apache.daffodil.runtime1.processors.parsers.BCDIntegerDelimitedParser import org.apache.daffodil.runtime1.processors.parsers.BlobSpecifiedLengthParser import org.apache.daffodil.runtime1.processors.parsers.HexBinaryDelimitedParser import org.apache.daffodil.runtime1.processors.parsers.HexBinaryEndOfBitLimitParser +import org.apache.daffodil.runtime1.processors.parsers.HexBinaryEndOfParentParser import org.apache.daffodil.runtime1.processors.parsers.HexBinarySpecifiedLengthParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedDecimalDelimitedParser import org.apache.daffodil.runtime1.processors.parsers.IBM4690PackedIntegerDelimitedParser @@ -191,6 +192,16 @@ case class HexBinaryLengthPrefixed(e: ElementBase) extends Terminal(e, true) { new HexBinaryMinLengthInBytesUnparser(e.minLength.longValue, e.elementRuntimeData) } +case class HexBinaryLengthEndOfParent(e: ElementBase) extends Terminal(e, true) { + + override lazy val parser: DaffodilParser = new HexBinaryEndOfParentParser( + e.elementRuntimeData + ) + + override lazy val unparser: DaffodilUnparser = + new HexBinaryMinLengthInBytesUnparser(e.minLength.longValue, e.elementRuntimeData) +} + abstract class PackedIntegerDelimited( e: ElementBase, packedSignCodes: PackedSignCodes diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala index f5c81be1f4..80d1280f17 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/PrimitivesPacked.scala @@ -21,9 +21,11 @@ import org.apache.daffodil.core.dsom.ElementBase import org.apache.daffodil.core.grammar.Terminal import org.apache.daffodil.lib.util.PackedSignCodes import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedDecimalRuntimeLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerBitLimitLengthParser +import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerEndOfParentLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerKnownLengthParser import org.apache.daffodil.runtime1.processors.parsers.PackedIntegerRuntimeLengthParser import org.apache.daffodil.runtime1.processors.unparsers.Unparser @@ -84,6 +86,18 @@ class PackedIntegerPrefixedLength( new PackedIntegerMinimumLengthUnparser(e.elementRuntimeData, packedSignCodes) } +class PackedIntegerEndOfParentLength( + val e: ElementBase, + packedSignCodes: PackedSignCodes +) extends Terminal(e, true) { + + override lazy val parser = + new PackedIntegerEndOfParentLengthParser(e.elementRuntimeData, packedSignCodes) + + override lazy val unparser: Unparser = + new PackedIntegerMinimumLengthUnparser(e.elementRuntimeData, packedSignCodes) +} + class PackedDecimalRuntimeLength(val e: ElementBase, packedSignCodes: PackedSignCodes) extends Terminal(e, true) { override lazy val parser = new PackedDecimalRuntimeLengthParser( @@ -145,3 +159,21 @@ class PackedDecimalPrefixedLength(val e: ElementBase, packedSignCodes: PackedSig e.decimalSigned ) } + +class PackedDecimalEndOfParentLength(val e: ElementBase, packedSignCodes: PackedSignCodes) + extends Terminal(e, true) { + + override lazy val parser = new PackedDecimalEndOfParentLengthParser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + packedSignCodes, + e.decimalSigned + ) + + override lazy val unparser: Unparser = new PackedDecimalMinimumLengthUnparser( + e.elementRuntimeData, + e.binaryDecimalVirtualPoint, + packedSignCodes, + e.decimalSigned + ) +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala index 0ae3e6a4d9..22edb1bccb 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/SpecifiedLength.scala @@ -132,6 +132,27 @@ class SpecifiedLengthExplicit(e: ElementBase, eGram: => Gram, bitsMultiplier: In } +class SpecifiedLengthEndOfParent(e: ElementBase, eGram: => Gram) + extends SpecifiedLengthCombinatorBase(e, eGram) { + + lazy val kind = "EndOfParent_" + e.lengthUnits.toString + + lazy val parser: Parser = { + if (eParser.isEmpty) eParser + else + new SpecifiedLengthEndOfParentParser( + eParser, + e.elementRuntimeData + ) + } + + // No length-management wrapper needed: the enclosing element's + // unparser handles RightFill/ElementUnused for any space + // the EOP child doesn't fill + lazy val unparser: Unparser = eUnparser + +} + class SpecifiedLengthImplicit(e: ElementBase, eGram: => Gram, nBits: Long) extends SpecifiedLengthCombinatorBase(e, eGram) with SpecifiedLengthExplicitImplicitUnparserMixin { diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala index c951fb2c7f..f393f337cf 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSource.scala @@ -181,6 +181,12 @@ abstract class InputSource { */ def releasePosition(bytePos0b: Long): Unit + /** + * Returns the 0-based exclusive byte position of the end of the data stream, + * or None if the end of data is not reachable within the implementation's cache size limit. + */ + def optEndOfDataPosition: Option[Long] + /** * Alerts the implementation to attempt to free data that is no longer used, * if possible. If possible, this should free any unlocked bytes. @@ -350,9 +356,19 @@ class BucketingInputSource( bytesFilledInLastBucket = 0 lastBucketIndex += 1 if ((lastBucketIndex - oldestBucketIndex) >= maxNumberOfNonNullBuckets) { - // This frees the oldest bucket, allowing it to be garbage collected. - buckets(oldestBucketIndex) = null - oldestBucketIndex += 1 + if (buckets(oldestBucketIndex).refCount == 0) { + // This frees the oldest bucket, allowing it to be garbage collected. + buckets(oldestBucketIndex) = null + oldestBucketIndex += 1 + } else { + // The oldest bucket has an active backtracking mark. We cannot evict + // it, and we cannot advance oldestBucketIndex past it (that would + // break the invariant checked in releasePosition). Stop filling here + // to prevent the window from growing without bound. Callers that need + // more data (e.g. optEndOfDataPosition) will receive a "goal not + // reached" result and handle the limitation appropriately. + needsMoreData = false + } } } @@ -564,6 +580,28 @@ class BucketingInputSource( headBucketBytePosition0b += bytesRemoved oldestBucketIndex = 0 } + + /** + * Returns the 0-based exclusive byte position of the end of the data stream + * by reading and bucketing data until EOF is reached. Does not affect the + * current read position. + * + * The scan window extends [[maxCacheSizeInBytes]] bytes beyond the last + * already-filled bucket (not from the current read position), maximising + * coverage without invalidating backtracking state. + * + * Returns None if EOF is not reached within that window. + */ + override lazy val optEndOfDataPosition: Option[Long] = { + val lastFilledBucketIndex = buckets.length - 1 + val goalBucketIndex = lastFilledBucketIndex + maxNumberOfNonNullBuckets - 1 + val endOfDataNotReached = fillBucketsToIndex(goalBucketIndex, bucketSize) + if (endOfDataNotReached) { + None + } else { + Some(totalBytesBucketed) + } + } } /** @@ -645,4 +683,12 @@ class ByteBufferInputSource(byteBuffer: ByteBuffer) extends InputSource { override def close(): Unit = { // do nothing. No resources to release. } + + /** + * Returns the 0-based exclusive byte position of the end of the buffer data. + */ + override def optEndOfDataPosition: Option[Long] = { + val finalPosition = position() + bb.remaining + Some(finalPosition) + } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala index 8e39d7db43..d73f854301 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/io/InputSourceDataInputStream.scala @@ -136,6 +136,8 @@ final class InputSourceDataInputStream private (val inputSource: InputSource) */ def hasReachedEndOfData: Boolean = inputSource.hasReachedEndOfData + def optEndOfDataPosition: Option[Long] = inputSource.optEndOfDataPosition + /** * Return the number of currently available bytes. * diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala index 36fc5ddeda..d832354c4c 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/BCDParsers.scala @@ -50,16 +50,22 @@ class BCDDecimalRuntimeLengthParser( } -class BCDDecimalBitLimitLengthParser( - e: ElementRuntimeData, - binaryDecimalVirtualPoint: Int -) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, None) +class BCDDecimalBitLimitLengthParser(e: ElementRuntimeData, binaryDecimalVirtualPoint: Int) + extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, None) with BitLengthFromBitLimitMixin { override def toNumber(num: Array[Byte]): JBigDecimal = DecimalUtils.bcdToBigDecimal(num, binaryDecimalVirtualPoint) } +class BCDDecimalEndOfParentLengthParser(e: ElementRuntimeData, binaryDecimalVirtualPoint: Int) + extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, None) + with EndOfParentBitLengthMixin { + + override def toNumber(num: Array[Byte]): JBigDecimal = + DecimalUtils.bcdToBigDecimal(num, binaryDecimalVirtualPoint) +} + class BCDIntegerRuntimeLengthParser( val e: ElementRuntimeData, val lengthEv: Evaluatable[JLong], @@ -85,3 +91,10 @@ class BCDIntegerBitLimitLengthParser(e: ElementRuntimeData) override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.bcdToBigInteger(num) } + +class BCDIntegerEndOfParentLengthParser(e: ElementRuntimeData) + extends PackedBinaryIntegerBaseParser(e) + with EndOfParentBitLengthMixin { + + override def toNumber(num: Array[Byte]): JBigInteger = DecimalUtils.bcdToBigInteger(num) +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala index 6ea612a57d..4c4d58c06e 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/IBM4690PackedDecimalParsers.scala @@ -65,6 +65,18 @@ class IBM4690PackedDecimalBitLimitLengthParser( } +class IBM4690PackedDecimalEndOfParentLengthParser( + e: ElementRuntimeData, + binaryDecimalVirtualPoint: Int, + decimalSigned: YesNo +) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) + with EndOfParentBitLengthMixin { + + override def toNumber(num: Array[Byte]): JBigDecimal = + DecimalUtils.ibm4690ToBigDecimal(num, binaryDecimalVirtualPoint) + +} + class IBM4690PackedIntegerRuntimeLengthParser( val e: ElementRuntimeData, val lengthEv: Evaluatable[JLong], @@ -96,3 +108,12 @@ class IBM4690PackedIntegerBitLimitLengthParser(e: ElementRuntimeData) DecimalUtils.ibm4690ToBigInteger(num) } + +class IBM4690PackedIntegerEndOfParentLengthParser(e: ElementRuntimeData) + extends PackedBinaryIntegerBaseParser(e) + with EndOfParentBitLengthMixin { + + override def toNumber(num: Array[Byte]): JBigInteger = + DecimalUtils.ibm4690ToBigInteger(num) + +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala index 46fc6c091d..823db43d99 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/PackedDecimalParsers.scala @@ -66,6 +66,18 @@ class PackedDecimalBitLimitLengthParser( DecimalUtils.packedToBigDecimal(num, binaryDecimalVirtualPoint, packedSignCodes) } +class PackedDecimalEndOfParentLengthParser( + e: ElementRuntimeData, + binaryDecimalVirtualPoint: Int, + packedSignCodes: PackedSignCodes, + decimalSigned: YesNo +) extends PackedBinaryDecimalBaseParser(e, binaryDecimalVirtualPoint, Some(decimalSigned)) + with EndOfParentBitLengthMixin { + + override def toNumber(num: Array[Byte]): JBigDecimal = + DecimalUtils.packedToBigDecimal(num, binaryDecimalVirtualPoint, packedSignCodes) +} + class PackedIntegerRuntimeLengthParser( val e: ElementRuntimeData, packedSignCodes: PackedSignCodes, @@ -101,3 +113,14 @@ class PackedIntegerBitLimitLengthParser( DecimalUtils.packedToBigInteger(num, packedSignCodes) } + +class PackedIntegerEndOfParentLengthParser( + e: ElementRuntimeData, + packedSignCodes: PackedSignCodes +) extends PackedBinaryIntegerBaseParser(e) + with EndOfParentBitLengthMixin { + + override def toNumber(num: Array[Byte]): JBigInteger = + DecimalUtils.packedToBigInteger(num, packedSignCodes) + +} diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala index a4a30de0bd..2df8114f7e 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryBooleanParsers.scala @@ -115,7 +115,5 @@ class BinaryBooleanBitLimitLengthParser( override def runtimeDependencies = Vector() - override def getBitLength(state: PState): Int = { - getLengthInBits(state).toInt - } + override def getBitLength(state: PState): Int = super.getBitLengthAsInt(state) } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala index 7c9f50efca..081a402aea 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/HexBinaryLengthParsers.scala @@ -21,6 +21,7 @@ import java.nio.ByteBuffer import org.apache.daffodil.runtime1.processors.ElementRuntimeData import org.apache.daffodil.runtime1.processors.LengthInBitsEv +import org.apache.daffodil.runtime1.processors.ParseOrUnparseState sealed abstract class HexBinaryLengthParser(override val context: ElementRuntimeData) extends PrimParser @@ -84,11 +85,15 @@ final class HexBinarySpecifiedLengthParser(erd: ElementRuntimeData, lengthEv: Le } final class HexBinaryEndOfBitLimitParser(erd: ElementRuntimeData) - extends HexBinaryLengthParser(erd) { + extends HexBinaryLengthParser(erd), + BitLengthFromBitLimitMixin { override def runtimeDependencies = Vector() +} - override def getLengthInBits(pstate: PState): Long = { - pstate.bitLimit0b.get - pstate.bitPos0b - } +final class HexBinaryEndOfParentParser(erd: ElementRuntimeData) + extends HexBinaryLengthParser(erd), + EndOfParentBitLengthMixin { + + override def runtimeDependencies = Vector() } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberTraits.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala similarity index 70% rename from daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberTraits.scala rename to daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala index 814a9ee31b..f37e3efc50 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/BinaryNumberTraits.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/ParserTraits.scala @@ -21,6 +21,7 @@ import java.lang.Long as JLong import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits +import org.apache.daffodil.lib.util.Maybe.One import org.apache.daffodil.lib.util.MaybeULong import org.apache.daffodil.lib.util.Numbers import org.apache.daffodil.runtime1.infoset.DIComplex @@ -170,18 +171,82 @@ trait PrefixedLengthParserMixin { * Some parsers do not calculate their own length, but instead expect another parser * to set the bit limit, and then they use that bit limit as the length. * An example of this is prefix length parsers. This trait can be used by those - * parsers to do determine the length based on the bitLimit and position. + * parsers to determine the length based on the bitLimit and position. + * + * For dfdl:lengthKind='endOfParent' parsers that need to scan to end-of-stream + * when no bit limit is set, mix in [[EndOfParentBitLengthMixin]] instead. */ trait BitLengthFromBitLimitMixin { - def getBitLength(s: ParseOrUnparseState): Int = { - val pState = s.asInstanceOf[PState] - val len = getLengthInBits(pState) - len.toInt + def getBitLength(s: ParseOrUnparseState): Int = getBitLengthAsInt(s.asInstanceOf[PState]) + + /** + * getLengthInBits converted to Int with a parse error on overflow. + * + * The default maxCacheSizeInBytes (256 MiB) yields a maximum bit length of + * exactly Int.MaxValue + 1, so a bare .toInt without this guard can overflow + * by one bit on a full-cache EOP element. + */ + def getBitLengthAsInt(pstate: PState): Int = { + val len = getLengthInBits(pstate) + if (pstate.processorStatus ne Success) return 0 + if (len > Int.MaxValue) { + pstate.setFailed( + new ParseError( + One(pstate.schemaFileLocation), + One(pstate.currentLocation), + "Bit length %d exceeds maximum (%d) for this parser.", + len, + Int.MaxValue + ) + ) + 0 + } else { + len.toInt + } } def getLengthInBits(pstate: PState): Long = { - val len = pstate.bitLimit0b.get - pstate.bitPos0b - len + if (pstate.bitLimit0b.isDefined) { + pstate.bitLimit0b.get - pstate.bitPos0b + } else { + Assert.invariantFailed("BitLimit not set for parser.") + } + } +} + +/** + * Extends [[BitLengthFromBitLimitMixin]] with dfdl:lengthKind='endOfParent' support. + * When no bit limit is set (i.e., the parser is at the root or outermost EOP element), + * falls back to [[org.apache.daffodil.io.InputSource#optEndOfDataPosition]] to locate + * the end of the data stream. + * + * Only mix this in for parsers that are instantiated specifically for EOP elements. + * Parsers for prefixed or explicitly-bounded elements should use the plain + * [[BitLengthFromBitLimitMixin]]; mixing in this trait by mistake would silently + * consume the entire remaining stream instead of failing at the missing bit limit. + */ +trait EndOfParentBitLengthMixin extends BitLengthFromBitLimitMixin { + + override def getLengthInBits(pstate: PState): Long = { + if (pstate.bitLimit0b.isDefined) { + pstate.bitLimit0b.get - pstate.bitPos0b + } else { + val dis = pstate.dataInputStream + dis.optEndOfDataPosition match { + case Some(endOfDataPosition) => + (endOfDataPosition * 8) - dis.bitPos0b + case None => + pstate.setFailed( + new ParseError( + One(pstate.schemaFileLocation), + One(pstate.currentLocation), + "Cannot determine end-of-data position for dfdl:lengthKind='endOfParent': " + + "data stream exceeds the maximum cache size. Consider increasing maxCacheSizeInBytes." + ) + ) + 0L // callers check pstate.processorStatus before using this value + } + } } } diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala index 2f3ee1f5dd..3f72372c9c 100644 --- a/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala +++ b/daffodil-core/src/main/scala/org/apache/daffodil/runtime1/processors/parsers/SpecifiedLengthParsers.scala @@ -20,6 +20,7 @@ package org.apache.daffodil.runtime1.processors.parsers import java.lang.Long as JLong import java.util.regex.Matcher +import org.apache.daffodil.io.InputSourceDataInputStream import org.apache.daffodil.lib.equality.* import org.apache.daffodil.lib.exceptions.Assert import org.apache.daffodil.lib.schema.annotation.props.gen.LengthUnits @@ -52,29 +53,18 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat */ protected def getBitLength(s: PState): MaybeULong - final def parse(pState: PState): Unit = { - - val maybeNBits = getBitLength(pState) - - if (pState.processorStatus._ne_(Success)) return - val nBits = maybeNBits.get - val dis = pState.dataInputStream - - val shouldCheckDefinedForLength = erd match { + def parse(pState: PState): Unit = { + lazy val shouldCheckDefinedForLength = erd match { case erd: ElementRuntimeData => !erd.isComplexType case _: ChoiceRuntimeData => false case _ => true } - if (shouldCheckDefinedForLength && !dis.isDefinedForLength(nBits)) { - PENotEnoughBits(pState, nBits, dis) - return - } - - val startingBitPos0b = dis.bitPos0b - dis.withBitLengthLimit(nBits) { - eParser.parse1(pState) - } + val (nBits: Long, dis: InputSourceDataInputStream, startingBitPos0b: Long) = + checkLengthAndParseWithinBitLimits(pState, shouldCheckDefinedForLength) match { + case Some(result) => result + case None => return + } // at this point the recursive parse of the children is finished // so if we're still successful we need to advance the position @@ -87,12 +77,39 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat // we want to capture the length before we do any skipping // value length of simple types is captured by the eParser if needed - // the SpecifiedLengthParserBase is extended by SpecifiedLengthChoiceParser which should not have its valueLength captured here + // the SpecifiedLengthParserBase is extended by SpecifiedLengthChoiceParser + // which should not have its valueLength captured here if (pState.infoset.isComplex && !erd.isInstanceOf[ChoiceRuntimeData]) captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) Assert.invariant(dis eq pState.dataInputStream) val bitsToSkip = finalEndPos0b - dis.bitPos0b + skipBits(pState, bitsToSkip, dis) + } + + protected def checkLengthAndParseWithinBitLimits( + pState: PState, + shouldCheckDefinedForLength: Boolean + ): Option[(Long, InputSourceDataInputStream, Long)] = { + val maybeNBits = getBitLength(pState) + + if (pState.processorStatus._ne_(Success)) return None + val nBits = maybeNBits.get + val dis = pState.dataInputStream + + if (shouldCheckDefinedForLength && !dis.isDefinedForLength(nBits)) { + PENotEnoughBits(pState, nBits, dis) + return None + } + + val startingBitPos0b = dis.bitPos0b + dis.withBitLengthLimit(nBits) { + eParser.parse1(pState) + } + Some((nBits, dis, startingBitPos0b)) + } + + def skipBits(pState: PState, bitsToSkip: Long, dis: InputSourceDataInputStream): Unit = { Assert.invariant( bitsToSkip >= 0 ) // if this is < 0, then the parsing of children went past the limit, which it isn't supposed to. @@ -104,6 +121,7 @@ sealed abstract class SpecifiedLengthParserBase(eParser: Parser, erd: RuntimeDat } } } + } class SpecifiedLengthPatternParser( @@ -132,6 +150,85 @@ class SpecifiedLengthPatternParser( } } +class SpecifiedLengthEndOfParentParser( + eParser: Parser, + erd: ElementRuntimeData +) extends SpecifiedLengthParserBase(eParser, erd), + EndOfParentBitLengthMixin { + + override protected def getBitLength(s: PState): MaybeULong = { + MaybeULong(getLengthInBits(s)) + } + + override def parse(pState: PState): Unit = { + val dis = pState.dataInputStream + + if (erd.isComplexType) { + if (dis.bitLimit0b.isDefined) { + val (nBits: Long, dis: InputSourceDataInputStream, startingBitPos0b: Long) = + checkLengthAndParseWithinBitLimits( + pState, + shouldCheckDefinedForLength = false + ) match { + case Some(result) => result + case None => return + } + + // at this point the recursive parse of the children is finished + // so if we're still successful we need to advance the position + // to skip past any bits that the recursive child parse did not + // consume at the end. That is, the specified length can be an + // outer constraint, but the children may not use it all up, leaving + // a section at the end. + if (pState.processorStatus ne Success) return + val finalEndPos0b = startingBitPos0b + nBits + + // we want to capture the length before we do any skipping + // value length of simple types is captured by the eParser + if (pState.infoset.isComplex) + captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) + + Assert.invariant(dis eq pState.dataInputStream) + val bitsToSkip = finalEndPos0b - dis.bitPos0b + // if this is < 0, then the parsing of children went past the limit, which it isn't supposed to. + skipBits(pState, bitsToSkip, dis) + } else { + val startingBitPos0b = dis.bitPos0b + + eParser.parse1(pState) + + // at this point the recursive parse of the children is finished + // so if we're still successful we need to advance the position + // to skip past any bits that the recursive child parse did not + // consume at the end. That is, the specified length can be an + // outer constraint, but the children may not use it all up, leaving + // a section at the end. + if (pState.processorStatus ne Success) return + + // we want to capture the length before we do any skipping + // value length of simple types is captured by the eParser + if (pState.infoset.isComplex) { + captureValueLength(pState, ULong(startingBitPos0b), ULong(dis.bitPos0b)) + } + + val maybeNBits = getBitLength(pState) + + if (pState.processorStatus._ne_(Success)) return + val nBits = maybeNBits.get + + if (!dis.isDefinedForLength(nBits)) { + PENotEnoughBits(pState, nBits, dis) + return + } + skipBits(pState, nBits, dis) + } + } else { + super.parse(pState) + } + } + +} + class SpecifiedLengthExplicitParser( eParser: Parser, erd: ElementRuntimeData, diff --git a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala index b6220d65af..3b8caf80a9 100644 --- a/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala +++ b/daffodil-core/src/test/scala/org/apache/daffodil/io/TestDaffodilDataInputSource.scala @@ -256,6 +256,41 @@ class TestBucketingInputSource { assertEquals(6, bis.get()) assertEquals(-1, bis.get()) } + + /** + * Regression for: optEndOfDataPosition must not evict a bucket that has an + * active backtracking mark (refCount > 0). + * + * Configuration: bucketSize=2 bytes, maxNumberOfNonNullBuckets=2 (4 bytes total + * cache). Locking position 0 pins bucket 0. When optEndOfDataPosition tries to + * scan beyond the 2-bucket window it would need to evict bucket 0 — which is + * forbidden. The fix stops the scan and returns None rather than nulling the + * locked bucket. + * + * Before fix: fillBucketsToIndex evicted bucket 0 unconditionally; the + * subsequent bis.position(0)/bis.get() threw BacktrackingException. + * After fix: optEndOfDataPosition returns None; the locked position is still + * readable after the scan. + */ + @Test def testOptEndOfDataPositionDoesNotEvictLockedBucket(): Unit = { + // bucketSizeExponent=1 -> bucketSize=2; maxCacheSizeInBytes=4 -> maxNumberOfNonNullBuckets=2 + val tis = new TestInputStream() + tis.setEOF(12) + val bis = new BucketingInputSource(tis, bucketSizeExponent = 1, maxCacheSizeInBytes = 4) + + bis.lockPosition(0) // simulates a choice-parser mark at the start of the stream + + // The oldest bucket (index 0) is locked. fillBucketsToIndex would exceed + // maxNumberOfNonNullBuckets after filling 2 buckets and be unable to evict + // bucket 0, so it stops the scan and returns None. + assertEquals(None, bis.optEndOfDataPosition) + + // Bucket 0 must still be live: backtracking to position 0 must succeed. + bis.position(0) + assertEquals(0, bis.get()) + + bis.releasePosition(0) + } } class TestByteBufferInputSource { diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml index c2be97ef3f..e97a545c6b 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/aligned_data/Aligned_Data.tdml @@ -639,6 +639,28 @@ + + + + + + + + + + + + + + + @@ -664,6 +686,30 @@ + + + + + + + + + + + + + + + + @@ -4357,6 +4403,21 @@ + + IaaBbb + + + + + + aa + bb + + + + + + IAaabbee @@ -4373,6 +4434,22 @@ + + IAaabbee + + + + + + aa + bb + + ee + + + + + aaTBbb @@ -4613,4 +4690,6 @@ + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml index fdbcc2e32d..1cbec0c1fa 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml @@ -20,9 +20,10 @@ description="Section 12 - lengthKind=endOfParent" xmlns:tdml="http://www.ibm.com/xmlns/dfdl/testData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:fn="http://www.w3.org/2005/xpath-functions" - xmlns:ex="http://example.com" defaultRoundTrip="true"> + xmlns:ex="http://example.com" defaultRoundTrip="onePass" + xmlns:dfdlx="http://www.ogf.org/dfdl/dfdl-1.0/extensions"> - + + + + + + + + + + + + + + + + + + + + + + - + - - + + - - + + + + + + + + + + + + + + + Schema Definition Error + endOfParent + but its parent + lengthKind + delimited + + + + + + + Schema Definition Error + endOfParent + but its parent + lengthKind + delimited + + + + + + + + + + + + + + + + + + + + + + + - + - - + + + + + + + + + - - + + Schema Definition Error - not - implemented endOfParent - complex - type + but its parent + lengthKind + implicit - - + + Schema Definition Error - not - implemented endOfParent - simple - type + but its parent + lengthKind + implicit + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + A + + + + + + + + + + + + X + YZ + + A + + + + + + + + + endOfParent + does not have + single-byte character set encoding + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + A + + + + + + + + + + + + X + YZ + + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + A + + + + + + + + + + + + X + YZ + + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + + + + + + + + + + + + X + YZ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + model group + between this element and + end of the enclosing component + + + + endOfParent + model group + between this element and + end of the enclosing component + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + A + + + + + + + + + + + + + X + YZ + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + specifies + terminator + + + + + + + endOfParent + specifies + terminator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + specifies + non-zero + trailingSkip + + + + + + + endOfParent + specifies + non-zero + trailingSkip + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + specifies + maxOccurs + greater than 1 + + + + + + + endOfParent + specifies + maxOccurs + greater than 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + does not have + single-byte character set encoding + + + + + + + endOfParent + does not have + single-byte character set encoding + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + separatorPosition + postfix + + + + + + + endOfParent + in a sequence with + separatorPosition + postfix + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + sequenceKind + unordered + + + + + + + endOfParent + in a sequence with + sequenceKind + unordered + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + elements + floating='yes + + + + + + + endOfParent + in a sequence with + elements + floating='yes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a sequence with + non-zero + trailingSkip + + + + + + + endOfParent + in a sequence with + non-zero + trailingSkip + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a choice with + non-zero + trailingSkip + + + + + + + endOfParent + in a choice with + non-zero + trailingSkip + + + + + + + endOfParent + a choice with + choiceLengthKind + implicit + + + + + + + endOfParent + a choice with + choiceLengthKind + implicit + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + in a choice with + terminator + + + + + + + endOfParent + in a choice with + terminator + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + xyzw + A + + + + + + + + + + + + xyzw + A + + + + + + + + + + + + + 1 + 1 + + A + + + + + + + + + + + + abcdef + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + X + YZ + COMPUTED + + A + + + + + + + + + + + + + + 1 + 1 + + + 2 + 2 + + COMPUTED + + A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + endOfParent + supported for + packed binary + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + X + 3132 + + + + + + + + + + + + + X + 12 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + 2 + 2 + + + 3 + 3 + + + + + + + + + + + + + X + YZ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + abcde + + + + abcde + + + + + + + ábcde + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + + + abcde + + + + + abcde + + + + + + + + abcde + + + + + + abcde + + + + + + + + + abcde + + + + abcde + + + + + + + abcde + + + + abcde + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 12 + + + + 12 + + + + + + + 123 + + + endOfParent + represented element + between this element and + end of the enclosing component + + + + + + 12 + + + + 12 + + + + + + + 12 + + + + 12 + + + + + + + 123.45 + + + + 123.45 + + + + + + + 123.45 + + + + 123.45 + + + + + + + 123.45 + + + + 123.45 + + + + + + + 2017-08-28T18:29:00 + + + + 2017-08-28T18:29:00 + + + + + + + 2017-08-28T18:29:00 + + + + 2017-08-28T18:29:00 + + + + + + + 2017-08-28T18:29:00 + + + + 2017-08-28T18:29:00 + + + + + + + TRUE + + + + true + + + + + + + TRUE + + + + true + + + + + + + TRUE + + + + true + + + + + + + + abcdefg + + + + + + abcd + + e + + + + + + + + + abcd + + + + + + abcd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 01234C + + + + 1234 + + + + + + + + 1234 + + + + 1234 + + + + + + + 1234 + + + + 1234 + + + + + + + 30 39 + + + endOfParent + only supported + packed binary formats + + + + + + 01234C + + + + 12.34 + + + + + + + 1234 + + + + 12.34 + + + + + + + 1234 + + + + 12.34 + + + + + + + + DE AD BE EF + + + + DEADBEEF + + + + + + + + 01 + + + endOfParent + not supported + boolean with binary representation + + + + + + + 01 22 51 64 51 93 65 0C + + + + 1645-12-25T19:36:50 + + + + + + + 06 14 20 04 18 56 03 + + + + 2004-06-14T18:56:03 + + + + + + + 11 30 20 07 04 15 08 + + + + 2007-11-30T04:15:08 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hello + + + + hello + + + + + + + hello + + + + hello + + + + + + + hello} + + + endOfParent + sequence with + terminator + + + + + + hello + + + + hello + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + helff + + + + hel + + + + + + + helff + + + + hel + + + + + + + helloA + + + + helloA + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0111 + 10101011 + + + + + + 7 + AB + + + + + + + + + + 0111 + 10101011 1010 + + + + + + 7 + 101010111010 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + AA BB CC DD EE + + + + + + AABBCCDDEE + + + + + + + + + + AA BB 00 00 00 + + + + + + AABB + + + + + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml index 6714a288c5..ce71d4d347 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section13/packed/packed.tdml @@ -259,6 +259,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1050,4 +1112,187 @@ + + + + 003C 123D + + + + + 3 + -123 + + + + + + + + + 003C 123D + + + + + 0.03 + -1.23 + + + + + + + + + + 3 + -123 + + + + + + 003C 123D + + + + + + + + + 0.03 + -1.23 + + + + + + 003C 123D + + + + + + + 0003 0123 + + + + + 3 + 123 + + + + + + + + + 0003 0123 + + + + + 0.03 + 1.23 + + + + + + + + + + 3 + 123 + + + + + + 0003 0123 + + + + + + + + 0.03 + 1.23 + + + + + + 0003 0123 + + + + + + + + FFF3 D123 + + + + + 3 + -123 + + + + + + + + + FFF3 D123 + + + + + 0.03 + -1.23 + + + + + + + + + + + 3 + -123 + + + + + + FFF3 D123 + + + + + + + + 0.03 + -1.23 + + + + + + FFF3 D123 + + + diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml index 099c6c5a1b..ff00047748 100644 --- a/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml +++ b/daffodil-test/src/test/resources/org/apache/daffodil/section15/choice_groups/ChoiceLengthExplicit.tdml @@ -84,6 +84,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + A + BC + + + + + + + + + + + + + + + + + A + BC + + + + + + + diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala index 3c4de0add4..ad587d2a09 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/aligned_data/TestAlignedData.scala @@ -209,6 +209,10 @@ class TestAlignedData extends TdmlTests { @Test def prior_siblings_3 = test @Test def prior_siblings_4 = test @Test def prior_siblings_5 = test + + // DAFFODIL-238 + @Test def test_init_alignment_1_eop = test + @Test def test_init_alignment_2_eop = test } class TestBinaryInput extends TdmlTests { diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala new file mode 100644 index 0000000000..cd72b674b9 --- /dev/null +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent.scala @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.daffodil.section12.lengthKind + +import org.apache.daffodil.junit.tdml.TdmlSuite +import org.apache.daffodil.junit.tdml.TdmlTests + +import org.junit.Test + +object TestLengthKindEndOfParent extends TdmlSuite { + val tdmlResource = "/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml" +} + +class TestLengthKindEndOfParent extends TdmlTests { + val tdmlSuite = TestLengthKindEndOfParent + + @Test def TestEndOfParentComplexTypesDelimited = test + @Test def TestEndOfParentSimpleTypesDelimited = test + @Test def TestEndOfParentComplexTypesImplicit = test + @Test def TestEndOfParentSimpleTypesImplicit = test + @Test def TestEndOfParentComplexTypesExplicit = test + @Test def TestEndOfParentSimpleTypesExplicit = test + @Test def TestEndOfParentCSVExplicit = test + @Test def TestEndOfParentComplexTypesPrefixed = test + @Test def TestEndOfParentSimpleTypesPrefixed = test + @Test def TestEndOfParentComplexTypesPattern = test + @Test def TestEndOfParentSimpleTypesPattern = test + @Test def TestEndOfParentComplexTypesEOP = test + @Test def TestEndOfParentSimpleTypesEOP = test + @Test def TestEndOfParentComplexTypes1 = test + @Test def TestEndOfParentSimpleTypes1 = test + @Test def TestEndOfParentComplexTypes2 = test + @Test def TestEndOfParentSimpleTypes2 = test + @Test def TestEndOfParentComplexTypes3 = test + @Test def TestEndOfParentSimpleTypes3 = test + @Test def TestEndOfParentComplexTypes4 = test + @Test def TestEndOfParentSimpleTypes4 = test + @Test def TestEndOfParentComplexTypes5 = test + @Test def TestEndOfParentSimpleTypes5 = test + @Test def TestEndOfParentComplexTypes6 = test + @Test def TestEndOfParentSimpleTypes6 = test + @Test def TestEndOfParentComplexTypes7 = test + @Test def TestEndOfParentSimpleTypes7 = test + @Test def TestEndOfParentComplexTypes8 = test + @Test def TestEndOfParentSimpleTypes8 = test + @Test def TestEndOfParentComplexTypes9 = test + @Test def TestEndOfParentSimpleTypes9 = test + @Test def TestEndOfParentComplexTypes10 = test + @Test def TestEndOfParentSimpleTypes10 = test + @Test def TestEndOfParentComplexTypes11 = test + @Test def TestEndOfParentSimpleTypes11 = test + @Test def TestEndOfParentComplexTypes12 = test + @Test def TestEndOfParentSimpleTypes12 = test + @Test def TestEndOfParentComplexTypes12Implicit = test + @Test def TestEndOfParentSimpleTypes12Implicit = test + @Test def TestEndOfParentComplexTypes13 = test + @Test def TestEndOfParentSimpleTypes13 = test + @Test def eopBranchFirst_ST_eopBranch = test + @Test def eopBranchFirst_ST_fixedBranch = test + @Test def eopBranchFirst_CT_eopBranch = test + @Test def eopBranchFirst_CT_fixedBranch = test + @Test def eopFollowedByIVC_ST = test + @Test def eopFollowedByIVC_CT = test + @Test def TestEndOfParentSimpleTypes14 = test + @Test def TestEndOfParentSimpleTypes16 = test + @Test def TestEndOfParentSimpleTypes17 = test + @Test def TestEndOfParentComplexTypesRootEOP = test + @Test def TestEndOfParentSimpleTypesRootEOP = test + + @Test def text_string_txt_bytes = test + @Test def text_string_txt_bits = test + @Test def text_string_txt_chars = test + @Test def text_string_txt_ref1 = test + @Test def text_string_txt_ref2 = test + @Test def text_string_txt_ref3 = test + @Test def text_string_txt_bytes_nil = test + @Test def text_string_txt_bits_nil = test + @Test def text_string_txt_chars_nil = test + @Test def text_int_txt_bytes = test + @Test def text_int_txt_bytes_group_ref = test + @Test def text_int_txt_bits = test + @Test def text_int_txt_chars = test + @Test def text_dec_txt_bytes = test + @Test def text_dec_txt_bits = test + @Test def text_dec_txt_chars = test + @Test def text_date_txt_bytes = test + @Test def text_date_txt_bits = test + @Test def text_date_txt_chars = test + @Test def text_bool_txt_bytes = test + @Test def text_bool_txt_bits = test + @Test def text_bool_txt_chars = test + @Test def check_eop_simple_elem_01 = test + @Test def check_eop_simple_elem_02 = test + @Test def bin_int_bin_bytes_packed = test + @Test def bin_int_bin_bytes_bcd = test + @Test def bin_int_bin_bytes_ibm4690 = test + @Test def bin_dec_bin_bytes = test + @Test def bin_dec_bin_bytes_packed = test + @Test def bin_dec_bin_bytes_bcd = test + @Test def bin_dec_bin_bytes_ibm4690 = test + @Test def bin_hex_bytes = test + @Test def bin_bool_bin_bytes = test + @Test def bin_date_bin_bytes_packed = test + @Test def bin_date_bin_bytes_bcd = test + @Test def bin_date_bin_bytes_ibm4690 = test + + @Test def nested_01 = test + @Test def nested_02 = test + @Test def nested_03 = test + @Test def nested_04 = test + @Test def checks_01 = test + @Test def checks_02 = test + @Test def checks_03 = test + + @Test def eop_not_byte_aligned_01 = test + @Test def eop_not_byte_aligned_02 = test + + @Test def eop_unparse_fill_exact = test + @Test def eop_unparse_fill_short = test +} diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent2.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent2.scala deleted file mode 100644 index 5928ed5f71..0000000000 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section12/lengthKind/TestLengthKindEndOfParent2.scala +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.daffodil.section12.lengthKind - -import org.apache.daffodil.junit.tdml.TdmlSuite -import org.apache.daffodil.junit.tdml.TdmlTests - -import org.junit.Test - -object TestLengthKindEndOfParent2 extends TdmlSuite { - val tdmlResource = "/org/apache/daffodil/section12/lengthKind/EndOfParentTests.tdml" -} - -class TestLengthKindEndOfParent2 extends TdmlTests { - val tdmlSuite = TestLengthKindEndOfParent2 - - @Test def TestEndOfParentNYIComplexTypes = test - @Test def TestEndOfParentNYISimpleTypes = test -} diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala index 48fbcf95cf..4c7b516881 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section13/packed/TestPacked.scala @@ -95,4 +95,18 @@ class TestPacked extends TdmlTests { // Daffodil-2961 @Test def bcdBigIntToLongExpr = test + + // Daffodil-238 + @Test def EOPPackedIntSeq = test + @Test def EOPPackedDecSeq = test + @Test def EOPPackedIntSeqUnparser = test + @Test def EOPPackedDecSeqUnparser = test + @Test def EOPBCDIntSeq = test + @Test def EOPBCDDecSeq = test + @Test def EOPBCDIntSeqUnparser = test + @Test def EOPBCDDecSeqUnparser = test + @Test def EOPIBM4690IntSeq = test + @Test def EOPIBM4690DecSeq = test + @Test def EOPIBM4690IntSeqUnparser = test + @Test def EOPIBM4690DecSeqUnparser = test } diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala index cfbb16a117..075f3a5183 100644 --- a/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala +++ b/daffodil-test/src/test/scala/org/apache/daffodil/section15/choice_groups/TestChoice.scala @@ -133,6 +133,8 @@ class TestChoiceLengthExplicit extends TdmlTests { @Test def explicit_07 = test @Test def explicit_08 = test @Test def explicit_09 = test + @Test def explicit_10 = test + @Test def explicit_11 = test @Test def explicit_multiple_choices = test