Examples with LongWatermark org.apache.gobblin.source.extractor.extract.LongWatermark used on opensource projects

Example 1 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class FineGrainedWatermarkTrackerTest method testConcurrentWatermarkTracker.

/**
 * A concurrent test, attempts fired in a single thread, but acks come in from multiple threads,
 * out of order.
 */
@Test
public static void testConcurrentWatermarkTracker() throws IOException, InterruptedException {
    Random random = new Random();
    ScheduledExecutorService ackingService = new ScheduledThreadPoolExecutor(100, ExecutorsUtils.defaultThreadFactory());
    for (int j = 0; j < 100; ++j) {
        FineGrainedWatermarkTracker tracker = new FineGrainedWatermarkTracker(ConfigFactory.empty());
        tracker.start();
        int numWatermarks = 1 + random.nextInt(1000);
        AcknowledgableWatermark[] acknowledgableWatermarks = new AcknowledgableWatermark[numWatermarks];
        SortedSet<Integer> holes = new TreeSet<>();
        final AtomicInteger numAcks = new AtomicInteger(0);
        for (int i = 0; i < numWatermarks; ++i) {
            CheckpointableWatermark checkpointableWatermark = new DefaultCheckpointableWatermark("default", new LongWatermark(i));
            final AcknowledgableWatermark ackable = new AcknowledgableWatermark(checkpointableWatermark);
            tracker.track(ackable);
            acknowledgableWatermarks[i] = ackable;
            // ack or not
            boolean ack = random.nextBoolean();
            if (ack) {
                numAcks.incrementAndGet();
                long sleepTime = random.nextInt(100);
                ackingService.schedule(new Callable<Object>() {

                    @Override
                    public Object call() throws Exception {
                        ackable.ack();
                        numAcks.decrementAndGet();
                        return null;
                    }
                }, sleepTime, TimeUnit.MILLISECONDS);
            } else {
                holes.add(i);
            }
        }
        while (numAcks.get() != 0) {
            log.info("Waiting for " + numAcks.get() + " acks");
            Thread.sleep(100);
        }
        verifyCommitables(tracker, holes, numWatermarks - 1);
        tracker.close();
    }
}

Example 2 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class FineGrainedWatermarkTrackerTest method testSweep.

/**
 * Tests that sweep is sweeping the correct number of entries.
 */
@Test
public static void testSweep() {
    Random random = new Random();
    for (int j = 0; j < 1000; ++j) {
        FineGrainedWatermarkTracker tracker = new FineGrainedWatermarkTracker(ConfigFactory.empty());
        tracker.setAutoStart(false);
        int numWatermarks = 1 + random.nextInt(1000);
        AcknowledgableWatermark[] acknowledgableWatermarks = new AcknowledgableWatermark[numWatermarks];
        for (int i = 0; i < numWatermarks; ++i) {
            CheckpointableWatermark checkpointableWatermark = new DefaultCheckpointableWatermark("default", new LongWatermark(i));
            AcknowledgableWatermark ackable = new AcknowledgableWatermark(checkpointableWatermark);
            acknowledgableWatermarks[i] = ackable;
            tracker.track(ackable);
        }
        int numMissingAcks = random.nextInt(numWatermarks);
        SortedSet<Integer> holes = new TreeSet<>();
        for (int i = 0; i < numMissingAcks; ++i) {
            holes.add(random.nextInt(numWatermarks));
        }
        for (int i = 0; i < numWatermarks; ++i) {
            if (!holes.contains(i)) {
                acknowledgableWatermarks[i].ack();
            }
        }
        verifyCommitables(tracker, holes, numWatermarks - 1);
        int swept = tracker.sweep();
        if (holes.isEmpty()) {
            Assert.assertEquals(swept, numWatermarks - 1);
        } else {
            if (holes.contains(0)) {
                Assert.assertEquals(swept, 0);
            } else {
                Assert.assertEquals(swept, holes.first() - 1);
            }
        }
        verifyCommitables(tracker, holes, numWatermarks - 1);
    }
}

Example 3 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class MultiWriterWatermarkManagerTest method testFlakyWatermarkStorage.

/**
 * Test that in the presence of intermittent commit successes and failures, we continue to make progress
 */
@Test
public void testFlakyWatermarkStorage() throws IOException, InterruptedException {
    final int failEvery = 2;
    WatermarkStorage mockWatermarkStorage = new WatermarkStorage() {

        private int watermarkInstance = 0;

        private List<CheckpointableWatermark> checkpointed = new ArrayList<>();

        @Override
        public void commitWatermarks(java.lang.Iterable<CheckpointableWatermark> watermarks) throws IOException {
            ++watermarkInstance;
            if (watermarkInstance % failEvery == 0) {
                throw new IOException("Failed to write");
            } else {
                checkpointed.clear();
                for (CheckpointableWatermark watermark : watermarks) {
                    checkpointed.add(watermark);
                }
            }
        }

        @Override
        public Map<String, CheckpointableWatermark> getCommittedWatermarks(Class<? extends CheckpointableWatermark> watermarkClass, Iterable<String> sourcePartitions) throws IOException {
            return null;
        }
    };
    WatermarkAwareWriter mockWatermarkWriter = new WatermarkAwareWriter() {

        private long watermark = 0;

        @Override
        public boolean isWatermarkCapable() {
            return true;
        }

        @Override
        public void writeEnvelope(RecordEnvelope recordEnvelope) throws IOException {
            throw new UnsupportedOperationException();
        }

        @Override
        public Map<String, CheckpointableWatermark> getCommittableWatermark() {
            watermark++;
            return Collections.singletonMap("default", (CheckpointableWatermark) new DefaultCheckpointableWatermark("default", new LongWatermark(watermark)));
        }

        @Override
        public Map<String, CheckpointableWatermark> getUnacknowledgedWatermark() {
            return null;
        }

        @Override
        public void write(Object record) throws IOException {
            throw new UnsupportedOperationException();
        }

        @Override
        public void commit() throws IOException {
            throw new UnsupportedOperationException();
        }

        @Override
        public void cleanup() throws IOException {
            throw new UnsupportedOperationException();
        }

        @Override
        public long recordsWritten() {
            return 0;
        }

        @Override
        public long bytesWritten() throws IOException {
            return 0;
        }

        @Override
        public void close() throws IOException {
        }
    };
    MultiWriterWatermarkManager watermarkManager = new MultiWriterWatermarkManager(mockWatermarkStorage, 1000, Optional.<Logger>absent());
    watermarkManager.registerWriter(mockWatermarkWriter);
    try {
        watermarkManager.start();
    } catch (Exception e) {
        Assert.fail("Should not throw exception", e);
    }
    Thread.sleep(2000);
    watermarkManager.close();
    MultiWriterWatermarkManager.CommitStatus commitStatus = watermarkManager.getCommitStatus();
    System.out.println(commitStatus);
    MultiWriterWatermarkManager.RetrievalStatus retrievalStatus = watermarkManager.getRetrievalStatus();
    Assert.assertTrue(retrievalStatus.getLastWatermarkRetrievalAttemptTimestampMillis() > 0);
    Assert.assertTrue(retrievalStatus.getLastWatermarkRetrievalSuccessTimestampMillis() > 0);
    Assert.assertTrue(retrievalStatus.getLastWatermarkRetrievalFailureTimestampMillis() == 0);
    System.out.println(retrievalStatus);
}

Example 4 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class FineGrainedWatermarkTrackerBenchmark method trackWithDelayedAcks.

@Benchmark
@Group("trackDelayed")
public void trackWithDelayedAcks(Control control, TrackerState trackerState) throws Exception {
    if (!control.stopMeasurement) {
        final AcknowledgableWatermark wmark = new AcknowledgableWatermark(new DefaultCheckpointableWatermark("0", new LongWatermark(trackerState._index)));
        trackerState._watermarkTracker.track(wmark);
        trackerState._index++;
        int delay = trackerState._random.nextInt(10);
        trackerState._executorService.schedule(new Runnable() {

            @Override
            public void run() {
                wmark.ack();
            }
        }, delay, TimeUnit.MILLISECONDS);
    }
}

Example 5 with LongWatermark

use of org.apache.gobblin.source.extractor.extract.LongWatermark in project incubator-gobblin by apache.

the class SequentialTestSource method initialWorkUnits.

private List<WorkUnit> initialWorkUnits() {
    List<WorkUnit> workUnits = Lists.newArrayList();
    for (int i = 0; i < num_parallelism; i++) {
        WorkUnit workUnit = WorkUnit.create(newExtract(Extract.TableType.APPEND_ONLY, namespace, table));
        LongWatermark lowWatermark = new LongWatermark(i * numRecordsPerExtract + 1);
        LongWatermark expectedHighWatermark = new LongWatermark((i + 1) * numRecordsPerExtract);
        workUnit.setWatermarkInterval(new WatermarkInterval(lowWatermark, expectedHighWatermark));
        workUnit.setProp(WORK_UNIT_INDEX, i);
        workUnits.add(workUnit);
    }
    return workUnits;
}