Examples with CommitLogReader cz.o2.proxima.direct.commitlog.CommitLogReader used on opensource projects

Example 46 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableUnnamedPauseContinueNoCommit.

@Test(timeout = 10000)
public void testObserveNonExternalizableUnnamedPauseContinueNoCommit() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown(), v -> v.getValue()[0] < 5));
    latch.await();
    assertEquals(6, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    CountDownLatch nextLatch = new CountDownLatch(1);
    reader.observeBulkOffsets(handle.getCurrentOffsets(), toList(data, b -> nextLatch.countDown()));
    nextLatch.await();
    assertEquals(10, data.size());
}

Example 47 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class TransformationRunner method runTransformation.

/**
 * Run given transformation in local JVM.
 *
 * @param direct the operator to run transformations with
 * @param name name of the transformation
 * @param desc the transformation to run
 * @param onReplicated callback to be called before write to replicated target
 * @return {@link ObserveHandle} of the transformation
 */
public static ObserveHandle runTransformation(DirectDataOperator direct, String name, TransformationDescriptor desc, Consumer<StreamElement> onReplicated) {
    final CommitLogObserver observer;
    if (desc.getTransformation().isContextual()) {
        observer = new TransformationObserver.Contextual(direct, name, desc.getTransformation().as(DirectElementWiseTransform.class), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    } else {
        observer = new TransformationObserver.NonContextual(direct, name, desc.getTransformation().asElementWiseTransform(), desc.getOutputTransactionMode() == OutputTransactionMode.ENABLED, desc.getFilter()) {

            @Override
            protected void onReplicated(StreamElement element) {
                onReplicated.accept(element);
            }
        };
    }
    CommitLogReader reader = desc.getAttributes().stream().flatMap(attr -> findFamilyDescriptorForAttribute(direct, attr)).findAny().flatMap(DirectAttributeFamilyDescriptor::getCommitLogReader).orElseThrow(() -> new IllegalStateException("No commit log reader for attributes of transformation " + desc));
    log.debug("Starting to observe reader {} with observer {} as {}", reader, observer, name);
    return reader.observe(name, observer);
}

Example 48 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectUnboundedSource method createReader.

@Override
public UnboundedReader<StreamElement> createReader(PipelineOptions po, Checkpoint cmt) {
    Offset offset = cmt == null ? null : cmt.getOffset();
    long readerLimit = cmt == null ? limit : cmt.getLimit();
    CommitLogReader reader = reader();
    log.info("Created reader reading from {} with offset {} and limit {}", reader.getUri(), offset, readerLimit);
    return BeamCommitLogReader.unbounded(this, name, reader, position, eventTime, readerLimit, partition, offset);
}

Example 49 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class DirectDataAccessorWrapper method createStream.

@Override
public PCollection<StreamElement> createStream(String name, Pipeline pipeline, Position position, boolean stopAtCurrent, boolean eventTime, long limit) {
    CommitLogReader reader = direct.getCommitLogReader(context).orElseThrow(() -> new IllegalArgumentException("Cannot create commit log from " + direct));
    final PCollection<StreamElement> ret;
    if (stopAtCurrent) {
        // bounded
        // FIXME: this should be converted to SDF
        // we need to support CommitLogReader#fetchOffsets() for that
        // see https://github.com/O2-Czech-Republic/proxima-platform/issues/191
        // once that is resolved, we can proceed
        ret = pipeline.apply("ReadBounded:" + uri, Read.from(DirectBoundedSource.of(factory, name, reader, position, limit)));
    } else {
        // unbounded
        ret = pipeline.apply("ReadUnbounded:" + uri, CommitLogRead.of(name, position, limit, factory, reader));
    }
    return ret.setCoder(StreamElementCoder.of(factory)).setTypeDescriptor(TypeDescriptor.of(StreamElement.class));
}

Example 50 with CommitLogReader

use of cz.o2.proxima.direct.commitlog.CommitLogReader in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testBulkObserveWithException.

@Test(timeout = 10000)
public void testBulkObserveWithException() throws InterruptedException {
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(3)));
    LocalKafkaWriter writer = accessor.newWriter();
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(context()));
    final AtomicInteger restarts = new AtomicInteger();
    final AtomicReference<Throwable> exc = new AtomicReference<>();
    final CountDownLatch latch = new CountDownLatch(2);
    final StreamElement update = StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), System.currentTimeMillis(), new byte[] { 1, 2 });
    final ObserveHandle handle = reader.observeBulk("test", Position.NEWEST, new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            restarts.incrementAndGet();
            throw new RuntimeException("FAIL!");
        }

        @Override
        public void onCompleted() {
            fail("This should not be called");
        }

        @Override
        public boolean onError(Throwable error) {
            exc.set(error);
            latch.countDown();
            throw new RuntimeException(error);
        }
    });
    writer.write(update, (succ, e) -> {
        assertTrue(succ);
        latch.countDown();
    });
    latch.await();
    assertEquals("FAIL!", exc.get().getMessage());
    assertEquals(1, restarts.get());
    assertEquals(3, handle.getCommittedOffsets().size());
    List<Long> startedOffsets = handle.getCurrentOffsets().stream().map(o -> ((TopicOffset) o).getOffset()).filter(o -> o >= 0).collect(Collectors.toList());
    assertEquals(Collections.singletonList(0L), startedOffsets);
}