Examples with Watermark org.apache.gobblin.source.extractor.Watermark used on opensource projects

Example 1 with Watermark

use of org.apache.gobblin.source.extractor.Watermark in project incubator-gobblin by apache.

the class ReplicaHadoopFsEndPoint method getWatermark.

@Override
public synchronized Optional<ComparableWatermark> getWatermark() {
    if (this.watermarkInitialized) {
        return this.cachedWatermark;
    }
    this.watermarkInitialized = true;
    try {
        Path metaData = new Path(rc.getPath(), WATERMARK_FILE);
        FileSystem fs = FileSystem.get(rc.getFsURI(), new Configuration());
        if (fs.exists(metaData)) {
            try (FSDataInputStream fin = fs.open(metaData)) {
                InputStreamReader reader = new InputStreamReader(fin, Charsets.UTF_8);
                String content = CharStreams.toString(reader);
                Watermark w = WatermarkMetadataUtil.deserialize(content);
                if (w instanceof ComparableWatermark) {
                    this.cachedWatermark = Optional.of((ComparableWatermark) w);
                }
            }
            return this.cachedWatermark;
        }
        // for replica, can not use the file time stamp as that is different with original source time stamp
        return this.cachedWatermark;
    } catch (IOException e) {
        log.warn("Can not find " + WATERMARK_FILE + " for replica " + this);
        return this.cachedWatermark;
    } catch (WatermarkMetadataUtil.WatermarkMetadataMulFormatException e) {
        log.warn("Can not create watermark from " + WATERMARK_FILE + " for replica " + this);
        return this.cachedWatermark;
    }
}