Examples with BatchLogReader cz.o2.proxima.direct.batch.BatchLogReader used on opensource projects

Example 1 with BatchLogReader

use of cz.o2.proxima.direct.batch.BatchLogReader in project proxima-platform by O2-Czech-Republic.

the class HadoopStorageTest method testObserveCancel.

@Test(timeout = 5000L)
public void testObserveCancel() throws InterruptedException {
    Map<String, Object> cfg = cfg(HadoopDataAccessor.HADOOP_ROLL_INTERVAL, -1);
    HadoopDataAccessor accessor = new HadoopDataAccessor(TestUtils.createTestFamily(entity, uri, cfg));
    CountDownLatch latch = new CountDownLatch(1);
    writeOneElement(accessor, (success, error) -> {
        assertTrue(success);
        assertNull(error);
        latch.countDown();
    }).updateWatermark(Long.MAX_VALUE);
    latch.await();
    BatchLogReader reader = accessor.getBatchLogReader(direct.getContext()).orElse(null);
    assertNotNull(reader);
    List<Partition> partitions = reader.getPartitions();
    assertEquals(1, partitions.size());
    CountDownLatch cancelledLatch = new CountDownLatch(1);
    AtomicReference<ObserveHandle> handle = new AtomicReference<>();
    handle.set(reader.observe(partitions, Collections.singletonList(attribute), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            handle.get().close();
            return true;
        }

        @Override
        public void onCompleted() {
            fail("onCompleted should not have been called");
        }

        @Override
        public void onCancelled() {
            cancelledLatch.countDown();
        }

        @Override
        public boolean onError(Throwable error) {
            onCancelled();
            return true;
        }
    }));
    cancelledLatch.await();
}

Example 2 with BatchLogReader

use of cz.o2.proxima.direct.batch.BatchLogReader in project proxima-platform by O2-Czech-Republic.

the class HadoopStorageTest method testWriteElementNotYetFlushed.

@Test(timeout = 5_000L)
public void testWriteElementNotYetFlushed() throws InterruptedException {
    Map<String, Object> cfg = cfg(HadoopDataAccessor.HADOOP_ROLL_INTERVAL, 1000);
    HadoopDataAccessor accessor = new HadoopDataAccessor(TestUtils.createTestFamily(entity, uri, cfg));
    CountDownLatch latch = new CountDownLatch(1);
    BulkAttributeWriter writer = writeOneElement(accessor, ((success, error) -> {
        if (error != null) {
            log.error("Failed to flush write", error);
        }
        assertTrue("Error in flush " + error, success);
        assertNull(error);
        latch.countDown();
    }));
    assertTrue(root.exists());
    List<File> files = listRecursively(root);
    assertEquals("Expected single file in " + files, 1, files.size());
    assertTrue(Iterables.getOnlyElement(files).getAbsolutePath().contains("_tmp"));
    BatchLogReader reader = accessor.getBatchLogReader(direct.getContext()).orElse(null);
    assertNotNull(reader);
    List<Partition> partitions = reader.getPartitions();
    assertTrue("Expected empty partitions, got " + partitions, partitions.isEmpty());
    // advance watermark to flush
    writer.updateWatermark(Long.MAX_VALUE);
    latch.await();
    partitions = reader.getPartitions();
    assertEquals(1, partitions.size());
    BlockingQueue<StreamElement> queue = new SynchronousQueue<>();
    reader.observe(partitions, Collections.singletonList(attribute), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            ExceptionUtils.unchecked(() -> queue.put(element));
            return true;
        }
    });
    StreamElement element = queue.take();
    assertNotNull(element);
}

Example 3 with BatchLogReader

use of cz.o2.proxima.direct.batch.BatchLogReader in project proxima-platform by O2-Czech-Republic.

the class HadoopStorageTest method testWriteElementJson.

@Test(timeout = 5000L)
public void testWriteElementJson() throws InterruptedException {
    Map<String, Object> cfg = cfg(HadoopDataAccessor.HADOOP_ROLL_INTERVAL, -1, "hadoop.format", "json");
    HadoopDataAccessor accessor = new HadoopDataAccessor(TestUtils.createTestFamily(entity, uri, cfg));
    CountDownLatch latch = new CountDownLatch(1);
    BulkAttributeWriter writer = writeOneElement(accessor, ((success, error) -> {
        assertTrue(success);
        assertNull(error);
        latch.countDown();
    }));
    writer.updateWatermark(Long.MAX_VALUE);
    latch.await();
    assertTrue(root.exists());
    List<File> files = listRecursively(root);
    assertEquals("Expected single file in " + files, 1, files.size());
    assertFalse(Iterables.getOnlyElement(files).getAbsolutePath().contains("_tmp"));
    BatchLogReader reader = accessor.getBatchLogReader(direct.getContext()).orElse(null);
    assertNotNull(reader);
    List<Partition> partitions = reader.getPartitions();
    assertEquals(1, partitions.size());
    BlockingQueue<StreamElement> queue = new SynchronousQueue<>();
    reader.observe(partitions, Collections.singletonList(attribute), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            ExceptionUtils.unchecked(() -> queue.put(element));
            return true;
        }
    });
    StreamElement element = queue.take();
    assertNotNull(element);
}

Example 4 with BatchLogReader

use of cz.o2.proxima.direct.batch.BatchLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataAccessorWrapper method createStreamFromUpdates.

@Override
public PCollection<StreamElement> createStreamFromUpdates(Pipeline pipeline, List<AttributeDescriptor<?>> attrs, long startStamp, long endStamp, long limit) {
    BatchLogReader reader = direct.getBatchLogReader(context).orElseThrow(() -> new IllegalArgumentException("Cannot create batch reader from " + direct));
    final PCollection<StreamElement> ret;
    ret = pipeline.apply("ReadBatchUnbounded:" + uri, BatchLogRead.of(attrs, Long.MAX_VALUE, factory, reader, startStamp, endStamp));
    return ret.setCoder(StreamElementCoder.of(factory)).setTypeDescriptor(TypeDescriptor.of(StreamElement.class));
}

Example 5 with BatchLogReader

use of cz.o2.proxima.direct.batch.BatchLogReader in project proxima-platform by O2-Czech-Republic.

the class BatchLogSourceFunctionTest method testRunAndClose.

@Test
void testRunAndClose() throws Exception {
    final Repository repository = Repository.ofTest(ConfigFactory.parseString(MODEL));
    final AttributeDescriptor<?> attribute = repository.getEntity("test").getAttribute("data");
    final BatchLogSourceFunction<StreamElement> sourceFunction = new BatchLogSourceFunction<StreamElement>(repository.asFactory(), Collections.singletonList(attribute), ResultExtractor.identity()) {

        @Override
        BatchLogReader createLogReader(List<AttributeDescriptor<?>> attributeDescriptors) {
            final DirectDataOperator direct = repository.getOrCreateOperator(DirectDataOperator.class);
            final ListBatchReader reader = ListBatchReader.ofPartitioned(direct.getContext());
            return OffsetTrackingBatchLogReader.of(reader);
        }
    };
    final AbstractStreamOperatorTestHarness<StreamElement> testHarness = createTestHarness(sourceFunction, 1, 0);
    testHarness.initializeEmptyState();
    testHarness.open();
    final CheckedThread runThread = new CheckedThread("run") {

        @Override
        public void go() throws Exception {
            sourceFunction.run(new TestSourceContext<StreamElement>() {

                @Override
                public void collect(StreamElement element) {
                // No-op.
                }
            });
        }
    };
    runThread.start();
    sourceFunction.awaitRunning();
    sourceFunction.cancel();
    testHarness.close();
    // Make sure run thread finishes normally.
    runThread.sync();
}