deequ_blog_example.scala

val numTitles = fetchNumTitles();
val maxExpectedPhoneRatio = calcMaxExpectedPhoneRatio();

val result = VerificationSuite()
  .onData(data)
  .addCheck(Check(CheckLevel.Error, "Completeness + uniqueness on main fields")
    .areComplete(Seq("customerId", "title", "impressionStart", "impressionEnd", "deviceType", "priority"))
    .areUnique(Seq("customerId", "countryResidence",  "deviceType", "title"))
    .hasApproxCountDistinct("", value => numTitles >= value)
    .hasHistogramValues("deviceType", distribution => maxExpectedPhoneRatio >= distribution("phone").ratio))

  .addCheck(new Check(CheckLevel.Error, "Check consistency of range-based data")
    .isNonNegative("count")
    .isLessThan("impressionStart", "impressionEnd")
    .isContainedIn("priority", Array("hi", "lo")))

  .addCheck(new Check(CheckLevel.Warning, "Check correlation")
    .isLessThan("impressionStart", "impressionEnd")
    .hasCorrelation("countryResidence", "cityResidence", corr => corr >= .99)
    .hasCorrelation("countryResidence", "zipCode", corr => corr >= .99))
  .run()