Examples with Watermarks cz.o2.proxima.time.Watermarks used on opensource projects

Example 1 with Watermarks

use of cz.o2.proxima.time.Watermarks in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReaderIT method writePoisonedPills.

/**
 * Write poisoned pills (element with timestamp = {@link Watermarks#MAX_WATERMARK}) to all
 * partitions.
 *
 * @param numPartitions Number of partitions in topic.
 * @return Completion latch.
 */
private CountDownLatch writePoisonedPills(int numPartitions) {
    final OnlineAttributeWriter writer = Optionals.get(operator.getWriter(fooDescriptor));
    // We assume test uses default partitioner.
    final KeyPartitioner keyPartitioner = new KeyPartitioner();
    final Set<Integer> poisonedPartitions = new HashSet<>();
    final CountDownLatch done = new CountDownLatch(numPartitions);
    for (int i = 0; poisonedPartitions.size() < numPartitions; i++) {
        final StreamElement poisonedPill = StreamElement.upsert(entity, fooDescriptor, UUID.randomUUID().toString(), String.format("poisoned-pill-%d", i), fooDescriptor.getName(), Watermarks.MAX_WATERMARK, "value".getBytes(StandardCharsets.UTF_8));
        final int partition = (keyPartitioner.getPartitionId(poisonedPill) & Integer.MAX_VALUE) % numPartitions;
        if (poisonedPartitions.add(partition)) {
            writer.write(poisonedPill, ((success, error) -> {
                if (success) {
                    done.countDown();
                }
            }));
        }
    }
    return done;
}

Example 2 with Watermarks

use of cz.o2.proxima.time.Watermarks in project proxima-platform by O2-Czech-Republic.

the class AbstractBulkFileSystemAttributeWriterTest method testWriteLate.

@Test(timeout = 10000)
public synchronized void testWriteLate() throws Exception {
    CountDownLatch latch = new CountDownLatch(2);
    long now = 1500000000000L;
    StreamElement[] elements = { // not late
    StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key1", "attr", now + 200, new byte[] { 1 }), // not late
    StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key2", "wildcard.1", now, new byte[] { 1, 2 }), // not late, flushes previous two
    StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key3", "wildcard.1", now + params.getRollPeriod() + 1, new byte[] { 1, 2, 3 }), // late, not flushed
    StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key4", "wildcard.1", now + params.getRollPeriod() / 2 + 1, new byte[] { 1, 2, 3, 4 }), // not late, flush
    StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key5", "wildcard.1", now + 3 * params.getRollPeriod(), new byte[] { 1, 2, 3, 4, 5 }) };
    List<Long> watermarks = new ArrayList<>(Arrays.asList(now, now, now + 2 * params.getRollPeriod() + params.getAllowedLateness() + 1, now + 2 * params.getRollPeriod() + params.getAllowedLateness() + 1, now + 3 * params.getRollPeriod() + params.getAllowedLateness()));
    Set<Long> committed = Collections.synchronizedSet(new HashSet<>());
    Arrays.stream(elements).forEach(e -> write(e, watermarks.remove(0), (succ, exc) -> {
        assertTrue("Exception " + exc, succ);
        assertNull(exc);
        committed.add(e.getStamp());
        latch.countDown();
    }));
    long watermark = now + 4 * params.getRollPeriod() + 1 + params.getAllowedLateness();
    while (true) {
        writer.bulk().updateWatermark(watermark);
        if (latch.await(50, TimeUnit.MILLISECONDS)) {
            break;
        }
    }
    assertEquals("Written: " + written.keySet(), 3, written.size());
    validate(written.get(now + params.getRollPeriod()), elements[0], elements[1]);
    validate(written.get(now + params.getRollPeriod() + 1), elements[2], elements[3]);
    validate(written.get(now + 4 * params.getRollPeriod()), elements[4]);
    assertEquals("Expected three paths, got " + flushedPaths, 3, flushedPaths.size());
    assertEquals(2, committed.size());
    assertEquals(Sets.newHashSet(now, now + 3 * params.getRollPeriod()), committed);
}

Example 3 with Watermarks

use of cz.o2.proxima.time.Watermarks in project proxima-platform by O2-Czech-Republic.

the class AbstractBulkFileSystemAttributeWriterTest method testFlushingOnOutOfOrder.

@Test(timeout = 10000)
public synchronized void testFlushingOnOutOfOrder() throws Exception {
    CountDownLatch latch = new CountDownLatch(1);
    long now = 1500000000000L;
    StreamElement[] elements = { StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", "attr", now + 200, new byte[] { 1 }), StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key", "wildcard.1", now, new byte[] { 1, 2 }), StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key", "wildcard.1", now + params.getRollPeriod(), new byte[] { 1, 2, 3 }), StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key", "wildcard.1", now + params.getRollPeriod() / 2, new byte[] { 1, 2, 3, 4 }), StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key", "wildcard.1", now + 3 * params.getRollPeriod(), new byte[] { 1, 2, 3, 4, 5 }) };
    List<Long> watermarks = new ArrayList<>(Arrays.asList(now, now, now + params.getRollPeriod() / 2, now + params.getRollPeriod() / 2, now + 3 * params.getRollPeriod() - params.getRollPeriod() / 2));
    Arrays.stream(elements).forEach(e -> write(e, watermarks.remove(0), (succ, exc) -> {
        assertTrue("Exception " + exc, succ);
        assertNull(exc);
        assertEquals(now + params.getRollPeriod() / 2, e.getStamp());
        latch.countDown();
    }));
    latch.await();
    assertEquals(2, written.size());
    validate(written.get(now + params.getRollPeriod()), elements[0], elements[1], elements[3]);
    validate(written.get(now + 2 * params.getRollPeriod()), elements[2]);
    assertEquals("Expected two paths, got " + flushedPaths, 2, flushedPaths.size());
    assertTrue("Invalid range for " + Iterables.get(flushedPaths, 0).toString(), params.getNamingConvention().isInRange(Iterables.get(flushedPaths, 0).toString(), now, now + 1));
    assertTrue(Iterables.get(flushedPaths, 0).contains("/path/2017/07/"));
    assertTrue(params.getNamingConvention().isInRange(Iterables.get(flushedPaths, 1).toString(), now + params.getRollPeriod(), now + params.getRollPeriod() + 1));
    assertTrue(Iterables.get(flushedPaths, 1).contains("/path/2017/07/"));
}

Example 4 with Watermarks

use of cz.o2.proxima.time.Watermarks in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveWithCustomWatemarkEstimator.

@Test(timeout = 1000)
public void testObserveWithCustomWatemarkEstimator() throws InterruptedException {
    long now = System.currentTimeMillis() - 1000;
    int numElements = 10;
    CommitLogReader reader = ListCommitLog.of(data(numElements), new TestWatermarkEstimator(now), direct.getContext());
    CountDownLatch latch = new CountDownLatch(1);
    List<Long> watermarks = new ArrayList<>();
    ObserveHandle handle = reader.observe(null, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.confirm();
            watermarks.add(context.getWatermark());
            return true;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }
    });
    latch.await();
    assertEquals(numElements, watermarks.size());
    List<Long> expected = IntStream.range(0, numElements).mapToObj(i -> now + i).collect(Collectors.toList());
    assertEquals(expected, watermarks);
}

Example 5 with Watermarks

use of cz.o2.proxima.time.Watermarks in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableWatermark.

@Test
public void testObserveNonExternalizableWatermark() throws InterruptedException {
    int numElements = 10;
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(numElements), direct.getContext());
    CountDownLatch first = new CountDownLatch(1);
    List<Long> watermarks = new ArrayList<>();
    ObserveHandle handle = reader.observeBulk(null, new CommitLogObserver() {

        int received = 0;

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            watermarks.add(context.getWatermark());
            if (++received == numElements) {
                context.confirm();
            }
            return true;
        }

        @Override
        public void onCompleted() {
            first.countDown();
        }
    });
    first.await();
    assertEquals(numElements, watermarks.size());
    long min = watermarks.get(0);
    for (int i = 1; i < numElements; i++) {
        assertEquals(min, (long) watermarks.get(i));
    }
}