Examples with ObserveHandle cz.o2.proxima.direct.batch.ObserveHandle used on opensource projects

Example 1 with ObserveHandle

use of cz.o2.proxima.direct.batch.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class HadoopStorageTest method testObserveCancel.

@Test(timeout = 5000L)
public void testObserveCancel() throws InterruptedException {
    Map<String, Object> cfg = cfg(HadoopDataAccessor.HADOOP_ROLL_INTERVAL, -1);
    HadoopDataAccessor accessor = new HadoopDataAccessor(TestUtils.createTestFamily(entity, uri, cfg));
    CountDownLatch latch = new CountDownLatch(1);
    writeOneElement(accessor, (success, error) -> {
        assertTrue(success);
        assertNull(error);
        latch.countDown();
    }).updateWatermark(Long.MAX_VALUE);
    latch.await();
    BatchLogReader reader = accessor.getBatchLogReader(direct.getContext()).orElse(null);
    assertNotNull(reader);
    List<Partition> partitions = reader.getPartitions();
    assertEquals(1, partitions.size());
    CountDownLatch cancelledLatch = new CountDownLatch(1);
    AtomicReference<ObserveHandle> handle = new AtomicReference<>();
    handle.set(reader.observe(partitions, Collections.singletonList(attribute), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            handle.get().close();
            return true;
        }

        @Override
        public void onCompleted() {
            fail("onCompleted should not have been called");
        }

        @Override
        public void onCancelled() {
            cancelledLatch.countDown();
        }

        @Override
        public boolean onError(Throwable error) {
            onCancelled();
            return true;
        }
    }));
    cancelledLatch.await();
}

Example 2 with ObserveHandle

use of cz.o2.proxima.direct.batch.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class HBaseLogReader method observe.

@Override
public ObserveHandle observe(List<Partition> partitions, List<AttributeDescriptor<?>> attributes, BatchLogObserver observer) {
    TerminationContext terminationContext = new TerminationContext(observer);
    observeInternal(partitions, attributes, observer, terminationContext);
    return terminationContext.asObserveHandle();
}

Example 3 with ObserveHandle

use of cz.o2.proxima.direct.batch.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class HadoopBatchLogReader method observe.

@Override
public ObserveHandle observe(List<Partition> partitions, List<AttributeDescriptor<?>> attributes, BatchLogObserver observer) {
    TerminationContext terminationContext = new TerminationContext(observer);
    observeInternal(partitions, attributes, observer, terminationContext);
    return terminationContext.asObserveHandle();
}

Example 4 with ObserveHandle

use of cz.o2.proxima.direct.batch.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testBulkObserveSuccess.

@Test(timeout = 10000)
public void testBulkObserveSuccess() throws InterruptedException {
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(3)));
    LocalKafkaWriter writer = accessor.newWriter();
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final AtomicInteger restarts = new AtomicInteger();
    final AtomicReference<Throwable> exc = new AtomicReference<>();
    final AtomicReference<StreamElement> input = new AtomicReference<>();
    final CountDownLatch latch = new CountDownLatch(2);
    final StreamElement update = StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 });
    final ObserveHandle handle = reader.observeBulk("test", Position.NEWEST, new CommitLogObserver() {

        @Override
        public void onRepartition(OnRepartitionContext context) {
            restarts.incrementAndGet();
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            input.set(ingest);
            context.confirm();
            latch.countDown();
            return true;
        }

        @Override
        public void onCompleted() {
            fail("This should not be called");
        }

        @Override
        public boolean onError(Throwable error) {
            exc.set(error);
            throw new RuntimeException(error);
        }
    });
    writer.write(update, (succ, e) -> {
        assertTrue(succ);
        latch.countDown();
    });
    latch.await();
    assertNull(exc.get());
    assertTrue(restarts.get() > 0);
    assertArrayEquals(update.getValue(), input.get().getValue());
    assertEquals(3, handle.getCommittedOffsets().size());
    assertEquals(handle.getCommittedOffsets().toString(), 1L, (long) (Long) handle.getCommittedOffsets().stream().mapToLong(o -> ((TopicOffset) o).getOffset()).sum());
}

Example 5 with ObserveHandle

use of cz.o2.proxima.direct.batch.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testBulkObserveOffsets2.

@Test(timeout = 10000)
public void testBulkObserveOffsets2() throws InterruptedException {
    final Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(3)));
    final LocalKafkaWriter writer = accessor.newWriter();
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final List<KafkaStreamElement> input = new ArrayList<>();
    final AtomicReference<CountDownLatch> latch = new AtomicReference<>(new CountDownLatch(3));
    final StreamElement update = StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 });
    final CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            input.add((KafkaStreamElement) ingest);
            latch.get().countDown();
            // terminate after reading first record
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    };
    final List<Offset> offsets;
    try (final ObserveHandle handle = reader.observeBulkPartitions(reader.getPartitions(), Position.NEWEST, observer)) {
        // write two elements
        for (int i = 0; i < 2; i++) {
            writer.write(update, (succ, e) -> {
                assertTrue(succ);
                latch.get().countDown();
            });
        }
        latch.get().await();
        latch.set(new CountDownLatch(1));
        offsets = handle.getCurrentOffsets();
    }
    // restart from old offset
    reader.observeBulkOffsets(Lists.newArrayList(offsets), observer);
    latch.get().await();
    assertEquals(2, input.size());
    assertEquals(0, input.get(0).getOffset());
    assertEquals(0, input.get(1).getOffset());
}