Examples with Position cz.o2.proxima.storage.commitlog.Position used on opensource projects

Example 1 with Position

use of cz.o2.proxima.storage.commitlog.Position in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testOnlineObserveWithRebalanceResetsOffsetCommitter.

@Test(timeout = 100000)
public void testOnlineObserveWithRebalanceResetsOffsetCommitter() throws InterruptedException {
    int numWrites = 5;
    Accessor accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, cfg(Pair.of(LocalKafkaCommitLogDescriptor.CFG_NUM_PARTITIONS, 3), // poll single record to commit it in atomic way
    Pair.of(KafkaAccessor.MAX_POLL_RECORDS, 1))));
    final CountDownLatch latch = new CountDownLatch(numWrites);
    AtomicInteger consumed = new AtomicInteger();
    List<OnNextContext> unconfirmed = Collections.synchronizedList(new ArrayList<>());
    CommitLogObserver observer = new CommitLogObserver() {

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            switch(consumed.getAndIncrement()) {
                case 0:
                    // we must confirm the first message to create a committed position
                    context.confirm();
                    break;
                case 2:
                    throw new RuntimeException("Failing first consumer!");
                default:
                    unconfirmed.add(context);
                    break;
            }
            if (consumed.get() == numWrites) {
                unconfirmed.forEach(OnNextContext::confirm);
            }
            latch.countDown();
            return true;
        }

        @Override
        public void onCompleted() {
        }

        @Override
        public boolean onError(Throwable error) {
            return true;
        }
    };
    testOnlineObserveWithRebalanceResetsOffsetCommitterWithObserver(observer, accessor, numWrites);
    latch.await();
    assertEquals("Invalid committed offests: " + accessor.committedOffsets, 3, accessor.committedOffsets.values().stream().mapToInt(AtomicInteger::get).sum());
}

Example 2 with Position

use of cz.o2.proxima.storage.commitlog.Position in project proxima-platform by O2-Czech-Republic.

the class PubSubReader method observeBulk.

private ObserveHandle observeBulk(@Nullable String name, Position position, boolean stopAtCurrent, long minWatermark, CommitLogObserver observer) {
    validateNotStopAtCurrent(stopAtCurrent);
    validatePosition(position);
    AtomicReference<List<AckReplyConsumer>> unconfirmed = new AtomicReference<>(new ArrayList<>());
    Object lock = new Object();
    Object listLock = new Object();
    AtomicLong globalOffset = new AtomicLong();
    String consumerName = asConsumerName(name);
    AtomicLong committedWatermark = new AtomicLong(minWatermark);
    PubSubPartition partition = new PubSubPartition(consumerName);
    return consume(consumerName, (e, w, c) -> {
        final long confirmUntil;
        synchronized (listLock) {
            List<AckReplyConsumer> list = unconfirmed.get();
            list.add(c);
            confirmUntil = list.size() + globalOffset.get();
        }
        OffsetCommitter committer = createBulkCommitter(listLock, confirmUntil, globalOffset, unconfirmed, w, committedWatermark);
        // ensure explicit synchronization here
        synchronized (lock) {
            try {
                Offset offset = new PubSubOffset(consumerName, w.getWatermark());
                if (!observer.onNext(e, asOnNextContext(committer, offset))) {
                    observer.onCompleted();
                    return false;
                }
                return true;
            } catch (Exception ex) {
                log.error("Error calling on next", ex);
                committer.fail(ex);
                throw new RuntimeException(ex);
            }
        }
    }, observer::onError, () -> observer.onRepartition(asRepartitionContext(Arrays.asList(partition))), () -> observer.onRepartition(asRepartitionContext(Arrays.asList(partition))), observer::onCancelled, committedWatermark);
}

Example 3 with Position

use of cz.o2.proxima.storage.commitlog.Position in project proxima-platform by O2-Czech-Republic.

the class ListCommitLog method observe.

@Override
public ObserveHandle observe(@Nullable String name, Position position, CommitLogObserver observer) {
    String consumerName = name == null ? UUID.randomUUID().toString() : name;
    Consumer consumer = CONSUMERS.get(uuid).computeIfAbsent(consumerName, k -> new Consumer(uuid, consumerName, watermarkEstimator));
    ListObserveHandle handle = new ListObserveHandle(uuid, consumerName);
    pushTo((element, offset) -> {
        if (handle.isClosed()) {
            return false;
        }
        final CommitLogObserver.OffsetCommitter committer = (succ, exc) -> {
            if (exc != null) {
                observer.onError(exc);
            }
        };
        final boolean acceptable;
        OnNextContext context = null;
        synchronized (consumer) {
            acceptable = (externalizableOffsets || !consumer.getAckedOffsets().contains(offset) && !consumer.getInflightOffsets().contains(offset));
            if (acceptable) {
                context = consumer.asOnNextContext(committer, offset);
            }
        }
        if (acceptable) {
            return observer.onNext(element, context);
        }
        return true;
    }, externalizableOffsets ? () -> true : allMatchOffset(consumer::isAcked), observer::onCompleted, observer::onCancelled);
    return handle;
}

Example 4 with Position

use of cz.o2.proxima.storage.commitlog.Position in project proxima-platform by O2-Czech-Republic.

the class DirectUnboundedSource method createReader.

@Override
public UnboundedReader<StreamElement> createReader(PipelineOptions po, Checkpoint cmt) {
    Offset offset = cmt == null ? null : cmt.getOffset();
    long readerLimit = cmt == null ? limit : cmt.getLimit();
    CommitLogReader reader = reader();
    log.info("Created reader reading from {} with offset {} and limit {}", reader.getUri(), offset, readerLimit);
    return BeamCommitLogReader.unbounded(this, name, reader, position, eventTime, readerLimit, partition, offset);
}

Example 5 with Position

use of cz.o2.proxima.storage.commitlog.Position in project proxima-platform by O2-Czech-Republic.

the class DirectDataAccessorWrapper method createStream.

@Override
public PCollection<StreamElement> createStream(String name, Pipeline pipeline, Position position, boolean stopAtCurrent, boolean eventTime, long limit) {
    CommitLogReader reader = direct.getCommitLogReader(context).orElseThrow(() -> new IllegalArgumentException("Cannot create commit log from " + direct));
    final PCollection<StreamElement> ret;
    if (stopAtCurrent) {
        // bounded
        // FIXME: this should be converted to SDF
        // we need to support CommitLogReader#fetchOffsets() for that
        // see https://github.com/O2-Czech-Republic/proxima-platform/issues/191
        // once that is resolved, we can proceed
        ret = pipeline.apply("ReadBounded:" + uri, Read.from(DirectBoundedSource.of(factory, name, reader, position, limit)));
    } else {
        // unbounded
        ret = pipeline.apply("ReadUnbounded:" + uri, CommitLogRead.of(name, position, limit, factory, reader));
    }
    return ret.setCoder(StreamElementCoder.of(factory)).setTypeDescriptor(TypeDescriptor.of(StreamElement.class));
}