Skip to content

Commit d52c2d1

Browse files
dougbdougb
authored
Renamed stringLengthChecks {min|max}Value to {min|max}Length (#20)
1 parent 3529d66 commit d52c2d1

3 files changed

Lines changed: 66 additions & 66 deletions

File tree

‎README.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -272,14 +272,14 @@ Takes 2 - 4 parameters, described below. If the value in the column doesn't fall
272272

273273
#### `stringLengthCheck`
274274

275-
Takes 2 to 4 parameters, described in the table below. If the length of the string in the column doesn't fall within the range specified by (`minValue`, `maxValue`), both inclusive, the check will fail.
276-
At least one of `minValue` or `maxValue` must be specified. The data type of `column` must be String.
275+
Takes 2 to 4 parameters, described in the table below. If the length of the string in the column doesn't fall within the range specified by (`minLength`, `maxLength`), both inclusive, the check will fail.
276+
At least one of `minLength` or `maxLength` must be specified. The data type of `column` must be String.
277277

278278
| Arg | Type | Description |
279279
|-----|------|-------------|
280280
| `column` | String | Table column to be checked. The DataType of the column must be a String
281-
| `minValue` | Integer | Lower bound of the length of the string, inclusive.
282-
| `maxValue` | Integer | Upper bound of the length of the string, inclusive.
281+
| `minLength` | Integer | Lower bound of the length of the string, inclusive.
282+
| `maxLength` | Integer | Upper bound of the length of the string, inclusive.
283283
| `threshold` | String | See above description of threshold.
284284

285285
#### `rowCount`
@@ -370,8 +370,8 @@ tables:
370370
# stringLengthCheck - checks if the length of the string in the column falls within the specified range, counts number of rows in which the length of the string is outside the specified range.
371371
- type: stringLengthCheck
372372
column: occupation
373-
minValue: 1
374-
maxValue: 5
373+
minLength: 1
374+
maxLength: 5
375375
```
376376

377377
## Working with OOZIE Workflows

‎src/main/scala/com/target/data_validator/validator/StringLengthCheck.scala‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,17 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructType}
1313

1414
case class StringLengthCheck(
1515
column: String,
16-
minValue: Option[Json],
17-
maxValue: Option[Json],
16+
minLength: Option[Json],
17+
maxLength: Option[Json],
1818
threshold: Option[String]
1919
) extends RowBased {
2020

2121
override def substituteVariables(dict: VarSubstitution): ValidatorBase = {
2222

2323
val ret = StringLengthCheck(
2424
getVarSub(column, "column", dict),
25-
minValue.map(getVarSubJson(_, "minValue", dict)),
26-
maxValue.map(getVarSubJson(_, "maxValue", dict)),
25+
minLength.map(getVarSubJson(_, "minLength", dict)),
26+
maxLength.map(getVarSubJson(_, "maxLength", dict)),
2727
threshold.map(getVarSub(_, "threshold", dict))
2828
)
2929
getEvents.foreach(ret.addEvent)
@@ -41,10 +41,10 @@ case class StringLengthCheck(
4141

4242
val colExp = Length(UnresolvedAttribute(column))
4343

44-
val minValueExpression = cmpExpr(colExp, minValue, LessThan)
45-
val maxValueExpression = cmpExpr(colExp, maxValue, GreaterThan)
44+
val minLengthExpression = cmpExpr(colExp, minLength, LessThan)
45+
val maxLengthExpression = cmpExpr(colExp, maxLength, GreaterThan)
4646

47-
val ret = (minValueExpression, maxValueExpression) match {
47+
val ret = (minLengthExpression, maxLengthExpression) match {
4848
case (Some(x), None) => x
4949
case (None, Some(y)) => y
5050
case (Some(x), Some(y)) => Or(x, y)
@@ -59,7 +59,7 @@ case class StringLengthCheck(
5959
if (values.forall(_.isNumber)) {
6060
values.flatMap(_.asNumber) match {
6161
case mv :: xv :: Nil if mv.toDouble > xv.toDouble =>
62-
addEvent(ValidatorError(s"min: ${minValue.get} must be less than or equal to max: ${maxValue.get}"))
62+
addEvent(ValidatorError(s"min: ${minLength.get} must be less than or equal to max: ${maxLength.get}"))
6363
case _ =>
6464
}
6565
} else if (values.forall(_.isString)) {
@@ -77,9 +77,9 @@ case class StringLengthCheck(
7777
override def configCheck(df: DataFrame): Boolean = {
7878

7979
// Verify if at least one of min or max is specified.
80-
val values = (minValue::maxValue::Nil).flatten
80+
val values = (minLength::maxLength::Nil).flatten
8181
if (values.isEmpty) {
82-
addEvent(ValidatorError("Must define minValue or maxValue or both."))
82+
addEvent(ValidatorError("Must define minLength or maxLength or both."))
8383
}
8484

8585
// Verify that min is less than max
@@ -103,8 +103,8 @@ case class StringLengthCheck(
103103
("type", Json.fromString("stringLengthCheck")),
104104
("column", Json.fromString(column))
105105
) ++
106-
minValue.map(mv => ("minValue", mv)) ++
107-
maxValue.map(mv => ("maxValue", mv)) ++
106+
minLength.map(mv => ("minLength", mv)) ++
107+
maxLength.map(mv => ("maxLength", mv)) ++
108108
Seq(
109109
("events", getEvents.asJson)
110110
)
@@ -115,16 +115,16 @@ case class StringLengthCheck(
115115
object StringLengthCheck extends LazyLogging {
116116
def fromJson(c: HCursor): Either[DecodingFailure, ValidatorBase] = {
117117
val column = c.downField("column").as[String].right.get
118-
val minValueJ = c.downField("minValue").as[Json].right.toOption
119-
val maxValueJ = c.downField("maxValue").as[Json].right.toOption
118+
val minLengthJ = c.downField("minLength").as[Json].right.toOption
119+
val maxLengthJ = c.downField("maxLength").as[Json].right.toOption
120120
val threshold = c.downField("threshold").as[String].right.toOption
121121

122122
logger.debug(s"column: $column")
123-
logger.debug(s"minValue: $minValueJ type: ${minValueJ.getClass.getCanonicalName}")
124-
logger.debug(s"maxValue: $maxValueJ type: ${maxValueJ.getClass.getCanonicalName}")
123+
logger.debug(s"minLength: $minLengthJ type: ${minLengthJ.getClass.getCanonicalName}")
124+
logger.debug(s"maxLength: $maxLengthJ type: ${maxLengthJ.getClass.getCanonicalName}")
125125
logger.debug(s"threshold: $threshold type: ${threshold.getClass.getCanonicalName}")
126126

127127
c.focus.foreach {f => logger.info(s"StringLengthCheckJson: ${f.spaces2}")}
128-
scala.util.Right(StringLengthCheck(column, minValueJ, maxValueJ, threshold))
128+
scala.util.Right(StringLengthCheck(column, minLengthJ, maxLengthJ, threshold))
129129
}
130130
}

‎src/test/scala/com/target/data_validator/validator/StringLengthCheckSpec.scala‎

Lines changed: 43 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
3737
val df = mkDataFrame(spark, defData)
3838
val sut = StringLengthCheck("item", None, None, None)
3939
assert(sut.configCheck(df))
40-
assert(sut.getEvents contains ValidatorError("Must define minValue or maxValue or both."))
40+
assert(sut.getEvents contains ValidatorError("Must define minLength or maxLength or both."))
4141
assert(sut.failed)
4242
}
4343

@@ -80,14 +80,14 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
8080
assert(sut.failed)
8181
}
8282

83-
it("minValue less than maxValue fails configCheck") {
84-
val maxValue = Math.abs(Random.nextInt(1000)) //scalastyle:ignore
85-
val minValue = maxValue + 10
86-
val sut = StringLengthCheck("item", Some(Json.fromInt(minValue)), Some(Json.fromInt(maxValue)), None)
83+
it("minLength less than maxLength fails configCheck") {
84+
val maxLength = Math.abs(Random.nextInt(1000)) //scalastyle:ignore
85+
val minLength = maxLength + 10
86+
val sut = StringLengthCheck("item", Some(Json.fromInt(minLength)), Some(Json.fromInt(maxLength)), None)
8787
val df = mkDataFrame(spark, defData)
8888
assert(sut.configCheck(df))
8989
assert(sut.failed)
90-
assert(sut.getEvents contains ValidatorError(s"min: $minValue must be less than or equal to max: $maxValue"))
90+
assert(sut.getEvents contains ValidatorError(s"min: $minLength must be less than or equal to max: $maxLength"))
9191
}
9292

9393
}
@@ -104,72 +104,72 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
104104
it("variable column name is good.") {
105105
val dict = new VarSubstitution
106106
dict.addString("column", "item")
107-
val minValue = Json.fromDouble(0.0)
108-
val sut = StringLengthCheck("$column", minValue, None, None)
109-
assert(sut.substituteVariables(dict) == StringLengthCheck("item", minValue, None, None))
107+
val minLength = Json.fromDouble(0.0)
108+
val sut = StringLengthCheck("$column", minLength, None, None)
109+
assert(sut.substituteVariables(dict) == StringLengthCheck("item", minLength, None, None))
110110
assert(!sut.failed)
111111
}
112112

113-
it("substitutes minValue") {
113+
it("substitutes minLength") {
114114
val dict = new VarSubstitution
115-
dict.addString("minValue", "0")
116-
val sut = StringLengthCheck("item", Some(Json.fromString("${minValue}")), None, None)
115+
dict.addString("minLength", "0")
116+
val sut = StringLengthCheck("item", Some(Json.fromString("${minLength}")), None, None)
117117
assert(sut.substituteVariables(dict) == StringLengthCheck("item", Some(Json.fromInt(0)), None, None))
118118
assert(!sut.failed)
119119
}
120120

121-
it("substitutes maxValue") {
121+
it("substitutes maxLength") {
122122
val dict = new VarSubstitution
123-
dict.addString("maxValue", "10")
124-
val sut = StringLengthCheck("item", None, Some(Json.fromString("${maxValue}")), None)
123+
dict.addString("maxLength", "10")
124+
val sut = StringLengthCheck("item", None, Some(Json.fromString("${maxLength}")), None)
125125
assert(sut.substituteVariables(dict) ==
126126
StringLengthCheck("item", None, Some(Json.fromInt(10)), None)) // scalastyle:ignore
127127
assert(!sut.failed)
128128
}
129129

130-
it("substitutes minValue and maxValue") {
130+
it("substitutes minLength and maxLength") {
131131
val dict = new VarSubstitution
132-
dict.addString("minValue", "1")
133-
dict.addString("maxValue", "10")
134-
val sut = StringLengthCheck("item", Some(Json.fromString("${minValue}")),
135-
Some(Json.fromString("${maxValue}")), None)
132+
dict.addString("minLength", "1")
133+
dict.addString("maxLength", "10")
134+
val sut = StringLengthCheck("item", Some(Json.fromString("${minLength}")),
135+
Some(Json.fromString("${maxLength}")), None)
136136
assert(sut.substituteVariables(dict) == StringLengthCheck("item", Some(Json.fromInt(1)),
137-
Some(Json.fromInt(10)), None)) // scalastyle: ignore
137+
Some(Json.fromInt(10)), None)) // scalastyle:ignore
138138
assert(!sut.failed)
139139
}
140140

141141
it("substitutes threshold") {
142142
val dict = new VarSubstitution
143143
// scalastyle:off
144144
val threshold = Json.fromInt(100)
145-
val minValue = Some(Json.fromInt(1))
146-
val maxValue = Some(Json.fromInt(10))
145+
val minLength = Some(Json.fromInt(1))
146+
val maxLength = Some(Json.fromInt(10))
147147
// scalastyle:on
148148
dict.add("threshold", threshold)
149-
val sut = StringLengthCheck("item", minValue, maxValue, Some("${threshold}"))
150-
assert(sut.substituteVariables(dict) == StringLengthCheck("item", minValue, maxValue,
149+
val sut = StringLengthCheck("item", minLength, maxLength, Some("${threshold}"))
150+
assert(sut.substituteVariables(dict) == StringLengthCheck("item", minLength, maxLength,
151151
Some("100"))) // scalastyle: ignore
152152
assert(!sut.failed)
153153
}
154154
}
155155

156156
describe("colTest") {
157157

158-
it("minValue") {
158+
it("minLength") {
159159
val dict = new VarSubstitution
160160
val sut = StringLengthCheck("item", Some(Json.fromInt(2)), None, None)
161161
assert(sut.colTest(schema, dict).sql ==
162162
LessThan(Length(UnresolvedAttribute("item")), Literal.create(2, IntegerType)).sql)
163163
}
164164

165-
it("maxValue") {
165+
it("maxLength") {
166166
val dict = new VarSubstitution
167167
val sut = StringLengthCheck("item", None, Some(Json.fromInt(2)), None)
168168
assert(sut.colTest(schema, dict).sql ==
169169
GreaterThan(Length(UnresolvedAttribute("item")), Literal.create(2, IntegerType)).sql)
170170
}
171171

172-
it("minValue and maxValue") {
172+
it("minLength and maxLength") {
173173
val dict = new VarSubstitution
174174
val sut = StringLengthCheck("item", Some(Json.fromInt(1)), Some(Json.fromInt(10)), None) // scalastyle:ignore
175175
assert(sut.colTest(schema, dict).sql ==
@@ -187,8 +187,8 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
187187
"""---
188188
|- type: stringLengthCheck
189189
| column: item
190-
| minValue: 0
191-
| maxValue: 100
190+
| minLength: 0
191+
| maxLength: 100
192192
|
193193
""".stripMargin
194194
val json = io.circe.yaml.parser.parse(yaml).right.getOrElse(Json.Null)
@@ -208,7 +208,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
208208
"""---
209209
|- type: stringLengthCheck
210210
| column: item
211-
| minValue: 0
211+
| minLength: 0
212212
|
213213
""".stripMargin
214214
val json = io.circe.yaml.parser.parse(yaml).right.getOrElse(Json.Null)
@@ -228,7 +228,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
228228
"""---
229229
|- type: stringLengthCheck
230230
| column: item
231-
| maxValue: 100
231+
| maxLength: 100
232232
|
233233
""".stripMargin
234234
val json = io.circe.yaml.parser.parse(yaml).right.getOrElse(Json.Null)
@@ -308,7 +308,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
308308
"StringLengthCheck failed! item = and ((length('item) < 5) || (length('item) > 6))"))
309309
}
310310

311-
it("String length check fails for minValue = maxValue and numErrorsToReport:3") {
311+
it("String length check fails for minLength = maxLength and numErrorsToReport:3") {
312312
val dict = new VarSubstitution
313313
val df = mkDataFrame(spark, defData)
314314
val sut = StringLengthCheck("item", Some(Json.fromInt(5)), Some(Json.fromInt(5)), None) // scalastyle:ignore
@@ -333,7 +333,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
333333
"StringLengthCheck failed! item = Item23 and ((length('item) < 5) || (length('item) > 5))"))
334334
}
335335

336-
it("String length check fails for only minValue specified and numErrorsToReport:2") {
336+
it("String length check fails for only minLength specified and numErrorsToReport:2") {
337337
val dict = new VarSubstitution
338338
val df = mkDataFrame(spark, defData)
339339
val sut = StringLengthCheck("item", Some(Json.fromInt(5)), None, None) // scalastyle:ignore
@@ -354,7 +354,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
354354
"StringLengthCheck failed! item = and (length('item) < 5)"))
355355
}
356356

357-
it("String length check fails for only maxValue specified and numErrorsToReport:2") {
357+
it("String length check fails for only maxLength specified and numErrorsToReport:2") {
358358
val dict = new VarSubstitution
359359
val df = mkDataFrame(spark, defData)
360360
val sut = StringLengthCheck("item", None, Some(Json.fromInt(2)), None) // scalastyle:ignore
@@ -375,7 +375,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
375375
"StringLengthCheck failed! item = Item23 and (length('item) > 2)"))
376376
}
377377

378-
it("String length check passes for minValue and maxValue specified") {
378+
it("String length check passes for minLength and maxLength specified") {
379379
val dict = new VarSubstitution
380380
val df = mkDataFrame(spark, defData)
381381
val sut = StringLengthCheck("item", Some(Json.fromInt(0)), Some(Json.fromInt(6)), None) // scalastyle:ignore
@@ -388,7 +388,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
388388
ValidatorCheckEvent(failure = false, "StringLengthCheck on column 'item'", 4, 0)) // scalastyle:ignore
389389
}
390390

391-
it("String length check passes for only minValue specified") {
391+
it("String length check passes for only minLength specified") {
392392
val dict = new VarSubstitution
393393
val df = mkDataFrame(spark, defData)
394394
val sut = StringLengthCheck("item", Some(Json.fromInt(0)), None, None) // scalastyle:ignore
@@ -401,7 +401,7 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
401401
ValidatorCheckEvent(failure = false, "StringLengthCheck on column 'item'", 4, 0)) // scalastyle:ignore
402402
}
403403

404-
it("String length check passes for only maxValue specified") {
404+
it("String length check passes for only maxLength specified") {
405405
val dict = new VarSubstitution
406406
val df = mkDataFrame(spark, defData)
407407
val sut = StringLengthCheck("item", None, Some(Json.fromInt(6)), None) // scalastyle:ignore
@@ -415,14 +415,14 @@ class StringLengthCheckSpec extends FunSpec with Matchers with TestingSparkSessi
415415
}
416416

417417
it("toJson works") {
418-
val minValue = Random.nextInt(1000) // scalastyle:ignore
419-
val minJson = Json.fromInt(minValue)
420-
val maxJson = Json.fromInt(minValue + Random.nextInt(1000) + 1) // scalastyle:ignore
418+
val minLength = Random.nextInt(1000) // scalastyle:ignore
419+
val minJson = Json.fromInt(minLength)
420+
val maxJson = Json.fromInt(minLength + Random.nextInt(1000) + 1) // scalastyle:ignore
421421
val sut = StringLengthCheck("item", Some(minJson), Some(maxJson), None) // scalastyle:ignore
422422
assert(sut.toJson == Json.obj(("type", Json.fromString("stringLengthCheck")),
423423
("column", Json.fromString("item")),
424-
("minValue", minJson),
425-
("maxValue", maxJson),
424+
("minLength", minJson),
425+
("maxLength", maxJson),
426426
("events", Json.arr())
427427
))
428428
}

0 commit comments

Comments
 (0)