Search in sources :

Example 46 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveExternalizableUnnamedPauseContinue.

@Test(timeout = 10000)
public void testObserveExternalizableUnnamedPauseContinue() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.of(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown(), v -> v.getValue()[0] < 5));
    latch.await();
    assertEquals(6, data.size());
    assertTrue(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    CountDownLatch nextLatch = new CountDownLatch(1);
    reader.observeBulkOffsets(handle.getCurrentOffsets(), toList(data, b -> nextLatch.countDown()));
    nextLatch.await();
    assertEquals(11, data.size());
}
Also used : IntStream(java.util.stream.IntStream) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) HashMap(java.util.HashMap) Function(java.util.function.Function) LogObserverUtils.toList(cz.o2.proxima.direct.commitlog.LogObserverUtils.toList) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) WatermarkEstimator(cz.o2.proxima.time.WatermarkEstimator) ConfigFactory(com.typesafe.config.ConfigFactory) SerializationException(cz.o2.proxima.scheme.SerializationException) TypeReference(com.fasterxml.jackson.core.type.TypeReference) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ExecutorService(java.util.concurrent.ExecutorService) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) Repository(cz.o2.proxima.repository.Repository) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) AttributeDescriptor(cz.o2.proxima.repository.AttributeDescriptor) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) Test(org.junit.Test) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Offset(cz.o2.proxima.direct.commitlog.Offset) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) DirectDataOperator(cz.o2.proxima.direct.core.DirectDataOperator) Assert(org.junit.Assert) Collections(java.util.Collections) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.junit.Test)

Example 47 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveExternalizableUnnamed.

@Test(timeout = 10000)
public void testObserveExternalizableUnnamed() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.of(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown()));
    latch.await();
    assertEquals(10, data.size());
    assertTrue(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
}
Also used : IntStream(java.util.stream.IntStream) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) HashMap(java.util.HashMap) Function(java.util.function.Function) LogObserverUtils.toList(cz.o2.proxima.direct.commitlog.LogObserverUtils.toList) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) WatermarkEstimator(cz.o2.proxima.time.WatermarkEstimator) ConfigFactory(com.typesafe.config.ConfigFactory) SerializationException(cz.o2.proxima.scheme.SerializationException) TypeReference(com.fasterxml.jackson.core.type.TypeReference) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ExecutorService(java.util.concurrent.ExecutorService) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) Repository(cz.o2.proxima.repository.Repository) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) AttributeDescriptor(cz.o2.proxima.repository.AttributeDescriptor) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) Test(org.junit.Test) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Offset(cz.o2.proxima.direct.commitlog.Offset) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) DirectDataOperator(cz.o2.proxima.direct.core.DirectDataOperator) Assert(org.junit.Assert) Collections(java.util.Collections) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.junit.Test)

Example 48 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableUnnamedPauseContinue.

@Test(timeout = 10000)
public void testObserveNonExternalizableUnnamedPauseContinue() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown(), v -> v.getValue()[0] < 5));
    latch.await();
    assertEquals(6, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    CountDownLatch nextLatch = new CountDownLatch(1);
    reader.observeBulkOffsets(handle.getCurrentOffsets(), toList(data, b -> nextLatch.countDown()));
    nextLatch.await();
    assertEquals(10, data.size());
}
Also used : IntStream(java.util.stream.IntStream) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) HashMap(java.util.HashMap) Function(java.util.function.Function) LogObserverUtils.toList(cz.o2.proxima.direct.commitlog.LogObserverUtils.toList) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) WatermarkEstimator(cz.o2.proxima.time.WatermarkEstimator) ConfigFactory(com.typesafe.config.ConfigFactory) SerializationException(cz.o2.proxima.scheme.SerializationException) TypeReference(com.fasterxml.jackson.core.type.TypeReference) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ExecutorService(java.util.concurrent.ExecutorService) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) Repository(cz.o2.proxima.repository.Repository) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) AttributeDescriptor(cz.o2.proxima.repository.AttributeDescriptor) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) Test(org.junit.Test) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Offset(cz.o2.proxima.direct.commitlog.Offset) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) DirectDataOperator(cz.o2.proxima.direct.core.DirectDataOperator) Assert(org.junit.Assert) Collections(java.util.Collections) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.junit.Test)

Example 49 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveNonExternalizableUnnamed.

@Test(timeout = 10000)
public void testObserveNonExternalizableUnnamed() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    ObserveHandle handle = reader.observe(null, toList(data, b -> latch.countDown()));
    latch.await();
    assertEquals(10, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    ListObserveHandle listObserveHandle = (ListObserveHandle) handle;
    ListCommitLog.Consumer consumer = listObserveHandle.getConsumer();
    assertTrue(consumer.getInflightOffsets().isEmpty());
}
Also used : IntStream(java.util.stream.IntStream) EntityDescriptor(cz.o2.proxima.repository.EntityDescriptor) HashMap(java.util.HashMap) Function(java.util.function.Function) LogObserverUtils.toList(cz.o2.proxima.direct.commitlog.LogObserverUtils.toList) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) WatermarkEstimator(cz.o2.proxima.time.WatermarkEstimator) ConfigFactory(com.typesafe.config.ConfigFactory) SerializationException(cz.o2.proxima.scheme.SerializationException) TypeReference(com.fasterxml.jackson.core.type.TypeReference) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ExecutorService(java.util.concurrent.ExecutorService) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) Repository(cz.o2.proxima.repository.Repository) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) AttributeDescriptor(cz.o2.proxima.repository.AttributeDescriptor) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) Test(org.junit.Test) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Offset(cz.o2.proxima.direct.commitlog.Offset) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) DirectDataOperator(cz.o2.proxima.direct.core.DirectDataOperator) Assert(org.junit.Assert) Collections(java.util.Collections) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) CountDownLatch(java.util.concurrent.CountDownLatch) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) Test(org.junit.Test)

Example 50 with ObserveHandle

use of cz.o2.proxima.direct.commitlog.ObserveHandle in project proxima-platform by O2-Czech-Republic.

the class ListCommitLogTest method testObserveBulkNonExternalizableUnnamed.

@Test(timeout = 10000)
public void testObserveBulkNonExternalizableUnnamed() throws InterruptedException {
    CommitLogReader reader = ListCommitLog.ofNonExternalizable(data(10), direct.getContext());
    List<StreamElement> data = new ArrayList<>();
    CountDownLatch first = new CountDownLatch(1);
    CountDownLatch second = new CountDownLatch(1);
    ObserveHandle handle = reader.observeBulk(null, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            context.nack();
            return false;
        }

        @Override
        public void onCancelled() {
            first.countDown();
        }
    });
    first.await();
    List<Offset> offsets = handle.getCurrentOffsets();
    handle = reader.observeBulkOffsets(offsets, toList(data, b -> second.countDown()));
    second.await();
    assertEquals(10, data.size());
    assertFalse(handle.getCommittedOffsets().isEmpty());
    assertFalse(handle.getCurrentOffsets().isEmpty());
    ListObserveHandle listObserveHandle = (ListObserveHandle) handle;
    ListCommitLog.Consumer consumer = listObserveHandle.getConsumer();
    assertTrue(consumer.getInflightOffsets().isEmpty());
}
Also used : ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) ObserveHandle(cz.o2.proxima.direct.commitlog.ObserveHandle) CommitLogReader(cz.o2.proxima.direct.commitlog.CommitLogReader) ArrayList(java.util.ArrayList) StreamElement(cz.o2.proxima.storage.StreamElement) CountDownLatch(java.util.concurrent.CountDownLatch) Offset(cz.o2.proxima.direct.commitlog.Offset) CommitLogObserver(cz.o2.proxima.direct.commitlog.CommitLogObserver) ListObserveHandle(cz.o2.proxima.direct.storage.ListCommitLog.ListObserveHandle) Test(org.junit.Test)

Aggregations

StreamElement (cz.o2.proxima.storage.StreamElement)58 ObserveHandle (cz.o2.proxima.direct.commitlog.ObserveHandle)56 Test (org.junit.Test)52 CommitLogObserver (cz.o2.proxima.direct.commitlog.CommitLogObserver)47 CommitLogReader (cz.o2.proxima.direct.commitlog.CommitLogReader)45 CountDownLatch (java.util.concurrent.CountDownLatch)43 ArrayList (java.util.ArrayList)38 Offset (cz.o2.proxima.direct.commitlog.Offset)32 AtomicReference (java.util.concurrent.atomic.AtomicReference)30 List (java.util.List)29 Partition (cz.o2.proxima.storage.Partition)26 HashMap (java.util.HashMap)26 DirectDataOperator (cz.o2.proxima.direct.core.DirectDataOperator)25 AttributeDescriptor (cz.o2.proxima.repository.AttributeDescriptor)24 UUID (java.util.UUID)24 WatermarkEstimator (cz.o2.proxima.time.WatermarkEstimator)23 Collections (java.util.Collections)23 Collectors (java.util.stream.Collectors)23 Repository (cz.o2.proxima.repository.Repository)22 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)22