Examples with Results org.apache.gobblin.compaction.verify.DataCompletenessVerifier.Results used on opensource projects

Example 1 with Results

use of org.apache.gobblin.compaction.verify.DataCompletenessVerifier.Results in project incubator-gobblin by apache.

the class MRCompactor method verifyDataCompleteness.

private void verifyDataCompleteness() {
    List<Pattern> blacklist = DatasetFilterUtils.getPatternList(this.state, COMPACTION_COMPLETENESS_VERIFICATION_BLACKLIST);
    List<Pattern> whitelist = DatasetFilterUtils.getPatternList(this.state, COMPACTION_COMPLETENESS_VERIFICATION_WHITELIST);
    int numDatasetsVerifiedTogether = getNumDatasetsVerifiedTogether();
    List<Dataset> datasetsToBeVerified = Lists.newArrayList();
    for (Dataset dataset : this.datasets) {
        if (dataset.state() != UNVERIFIED) {
            continue;
        }
        if (shouldVerifyCompletenessForDataset(dataset, blacklist, whitelist)) {
            datasetsToBeVerified.add(dataset);
            if (datasetsToBeVerified.size() >= numDatasetsVerifiedTogether) {
                ListenableFuture<Results> future = this.verifier.get().verify(datasetsToBeVerified);
                addCallback(datasetsToBeVerified, future);
                datasetsToBeVerified = Lists.newArrayList();
            }
        } else {
            dataset.setState(VERIFIED);
        }
    }
    if (!datasetsToBeVerified.isEmpty()) {
        ListenableFuture<Results> future = this.verifier.get().verify(datasetsToBeVerified);
        addCallback(datasetsToBeVerified, future);
    }
}