Examples with Predicate org.apache.gobblin.source.extractor.watermark.Predicate used on opensource projects

Example 6 with Predicate

use of org.apache.gobblin.source.extractor.watermark.Predicate in project incubator-gobblin by apache.

the class QueryBasedExtractor method getLatestWatermark.

/**
 * if snapshot extract, get latest watermark else return work unit high watermark
 *
 * @param watermark column
 * @param low watermark value
 * @param high watermark value
 * @param column format
 * @return letst watermark
 * @throws IOException
 */
private long getLatestWatermark(String watermarkColumn, WatermarkType watermarkType, long lwmValue, long hwmValue) throws HighWatermarkException, IOException {
    if (!Boolean.valueOf(this.workUnitState.getProp(ConfigurationKeys.SOURCE_QUERYBASED_SKIP_HIGH_WATERMARK_CALC))) {
        log.info("Getting high watermark");
        List<Predicate> list = new ArrayList<>();
        WatermarkPredicate watermark = new WatermarkPredicate(watermarkColumn, watermarkType);
        String lwmOperator = partition.isLowWatermarkInclusive() ? ">=" : ">";
        String hwmOperator = (partition.isLastPartition() || partition.isHighWatermarkInclusive()) ? "<=" : "<";
        Predicate lwmPredicate = watermark.getPredicate(this, lwmValue, lwmOperator, Predicate.PredicateType.LWM);
        Predicate hwmPredicate = watermark.getPredicate(this, hwmValue, hwmOperator, Predicate.PredicateType.HWM);
        if (lwmPredicate != null) {
            list.add(lwmPredicate);
        }
        if (hwmPredicate != null) {
            list.add(hwmPredicate);
        }
        return this.getMaxWatermark(this.schema, this.entity, watermarkColumn, list, watermark.getWatermarkSourceFormat(this));
    }
    return hwmValue;
}