Examples with LongWatermark org.apache.gobblin.source.extractor.extract.LongWatermark used on opensource projects

Example 31 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class FineGrainedWatermarkTrackerBenchmark method scheduledDelayedAcks.

@Benchmark
@Group("scheduledDelayed")
public void scheduledDelayedAcks(Control control, TrackerState trackerState) throws Exception {
    if (!control.stopMeasurement) {
        final AcknowledgableWatermark wmark = new AcknowledgableWatermark(new DefaultCheckpointableWatermark("0", new LongWatermark(trackerState._index)));
        trackerState._index++;
        int delay = trackerState._random.nextInt(10);
        trackerState._executorService.schedule(new Runnable() {

            @Override
            public void run() {
                wmark.ack();
            }
        }, delay, TimeUnit.MILLISECONDS);
    }
}

Example 32 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class FineGrainedWatermarkTrackerBenchmark method scheduledNoRandomDelayedAcks.

@Benchmark
@Group("scheduledNoRandom")
public void scheduledNoRandomDelayedAcks(Control control, TrackerState trackerState) throws Exception {
    if (!control.stopMeasurement) {
        final AcknowledgableWatermark wmark = new AcknowledgableWatermark(new DefaultCheckpointableWatermark("0", new LongWatermark(trackerState._index)));
        trackerState._index++;
        int delay = 10;
        trackerState._executorService.schedule(new Runnable() {

            @Override
            public void run() {
                wmark.ack();
            }
        }, delay, TimeUnit.MILLISECONDS);
    }
}

Example 33 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class FineGrainedWatermarkTrackerBenchmark method trackImmediateAcks.

@Benchmark
@Group("trackImmediate")
public void trackImmediateAcks(Control control, TrackerState trackerState) throws Exception {
    if (!control.stopMeasurement) {
        AcknowledgableWatermark wmark = new AcknowledgableWatermark(new DefaultCheckpointableWatermark("0", new LongWatermark(trackerState._index)));
        trackerState._watermarkTracker.track(wmark);
        trackerState._index++;
        wmark.ack();
    }
}

Example 34 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class SequentialTestSource method getWorkunits.

@Override
public List<WorkUnit> getWorkunits(SourceState state) {
    configureIfNeeded(ConfigFactory.parseProperties(state.getProperties()));
    final List<WorkUnitState> previousWorkUnitStates = state.getPreviousWorkUnitStates();
    if (!previousWorkUnitStates.isEmpty()) {
        List<WorkUnit> newWorkUnits = Lists.newArrayListWithCapacity(previousWorkUnitStates.size());
        int i = 0;
        for (WorkUnitState workUnitState : previousWorkUnitStates) {
            WorkUnit workUnit;
            if (workUnitState.getWorkingState().equals(WorkUnitState.WorkingState.COMMITTED)) {
                LongWatermark watermark = workUnitState.getActualHighWatermark(LongWatermark.class);
                LongWatermark expectedWatermark = new LongWatermark(watermark.getValue() + numRecordsPerExtract);
                WatermarkInterval watermarkInterval = new WatermarkInterval(watermark, expectedWatermark);
                workUnit = WorkUnit.create(newExtract(tableType, namespace, table), watermarkInterval);
                log.debug("Will be setting watermark interval to " + watermarkInterval.toJson());
                workUnit.setProp(WORK_UNIT_INDEX, workUnitState.getWorkunit().getProp(WORK_UNIT_INDEX));
            } else {
                // retry
                LongWatermark watermark = workUnitState.getWorkunit().getLowWatermark(LongWatermark.class);
                LongWatermark expectedWatermark = new LongWatermark(watermark.getValue() + numRecordsPerExtract);
                WatermarkInterval watermarkInterval = new WatermarkInterval(watermark, expectedWatermark);
                workUnit = WorkUnit.create(newExtract(tableType, namespace, table), watermarkInterval);
                log.debug("Will be setting watermark interval to " + watermarkInterval.toJson());
                workUnit.setProp(WORK_UNIT_INDEX, workUnitState.getWorkunit().getProp(WORK_UNIT_INDEX));
            }
            newWorkUnits.add(workUnit);
        }
        return newWorkUnits;
    } else {
        return initialWorkUnits();
    }
}

Example 35 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class SourceHadoopFsEndPoint method getWatermark.

@Override
public synchronized Optional<ComparableWatermark> getWatermark() {
    if (this.initialized) {
        return this.cachedWatermark;
    }
    try {
        long curTs = -1;
        FileSystem fs = FileSystem.get(rc.getFsURI(), new Configuration());
        Collection<Path> validPaths = ReplicationDataValidPathPicker.getValidPaths(this);
        for (Path p : validPaths) {
            this.allFileStatus.addAll(FileListUtils.listFilesRecursively(fs, p, super.getPathFilter(), super.isApplyFilterToDirectories()));
        }
        for (FileStatus f : this.allFileStatus) {
            if (f.getModificationTime() > curTs) {
                curTs = f.getModificationTime();
            }
        }
        ComparableWatermark result = new LongWatermark(curTs);
        this.cachedWatermark = Optional.of(result);
        if (this.cachedWatermark.isPresent()) {
            this.initialized = true;
        }
        return this.cachedWatermark;
    } catch (IOException e) {
        log.error("Error while retrieve the watermark for " + this);
        return this.cachedWatermark;
    }
}