use of com.thinkbiganalytics.spark.datavalidator.DataValidatorResult in project kylo by Teradata.
the class ValidationStage method apply.
@Nonnull
@Override
public TransformResult apply(@Nullable final TransformResult result) {
Preconditions.checkNotNull(result);
// Validate the data set
final DataValidatorResult validatorResult = validator.validate(result.getDataSet(), getPolicyMap(result.getDataSet().schema()));
// Add the validation result to the transform result
final List<List<TransformValidationResult>> rows = validatorResult.getCleansedRowResultRDD().map(new ListTransformValidationResults()).collect();
result.setValidationResults(rows);
// Add the profile to the transform result
final List<OutputRow> profile = (result.getProfile() != null) ? new ArrayList<>(result.getProfile()) : new ArrayList<OutputRow>();
profile.addAll(validator.getProfileStats(validatorResult));
result.setProfile(profile);
return result;
}
Aggregations