Examples with InMemStorage cz.o2.proxima.direct.storage.InMemStorage used on opensource projects

Example 1 with InMemStorage

use of cz.o2.proxima.direct.storage.InMemStorage in project proxima-platform by O2-Czech-Republic.

the class GroovyEnvTest method testReduceValueStateByKeyWithSameStamp.

@Test
public void testReduceValueStateByKeyWithSameStamp() throws Exception {
    int prefixLen = wildcard.toAttributePrefix().length();
    final Script compiled = compile("env.batch.wildcard.batchUpdates()" + ".flatMap({ [1, 2].collect({ i -> " + "new Tuple(it.key, i + Integer.valueOf(it.attribute[" + prefixLen + "])) }) })" + ".reduceValueStateByKey(" + " { it[0] }, { it[1] }, " + "{ 0 }, {s, v -> v - s}, {s, v -> v})" + ".map({ it.second })" + ".withTimestamp()" + ".collect()");
    // the InMemStorage is not append storage, so we need
    // to append additional suffix to the attribute name with ID of write
    // operation (1..5). That is ignored during value extraction in
    // reduceValueStateByKey
    long now = System.currentTimeMillis();
    write(StreamElement.upsert(batch, wildcard, "uuid1", "key1", wildcard.toAttributePrefix() + "11", now, new byte[] {}));
    write(StreamElement.upsert(batch, wildcard, "uuid2", "key1", wildcard.toAttributePrefix() + "02", now + 1, new byte[] {}));
    write(StreamElement.upsert(batch, wildcard, "uuid3", "key2", wildcard.toAttributePrefix() + "13", now + 2, new byte[] {}));
    write(StreamElement.upsert(batch, wildcard, "uuid4", "key1", wildcard.toAttributePrefix() + "14", now + 3, new byte[] {}));
    write(StreamElement.upsert(batch, wildcard, "uuid5", "key1", wildcard.toAttributePrefix() + "15", now + 4, new byte[] {}));
    @SuppressWarnings({ "unchecked", "rawtypes" }) List<Pair<Integer, Long>> result = (List) compiled.run();
    assertUnorderedEquals(Arrays.asList(Pair.of(2, now), Pair.of(1, now), Pair.of(-2, now + 1), Pair.of(1, now + 1), Pair.of(2, now + 2), Pair.of(1, now + 2), Pair.of(0, now + 3), Pair.of(1, now + 3), Pair.of(-1, now + 4), Pair.of(1, now + 4)), result);
}

Example 2 with InMemStorage

use of cz.o2.proxima.direct.storage.InMemStorage in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObservePartitions.

@Test(timeout = 10000)
public void testObservePartitions() throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///inmemstoragetest")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    AtomicReference<CountDownLatch> latch = new AtomicReference<>();
    ObserveHandle handle = reader.observePartitions(reader.getPartitions(), new CommitLogObserver() {

        @Override
        public void onRepartition(OnRepartitionContext context) {
            assertEquals(1, context.partitions().size());
            latch.set(new CountDownLatch(1));
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            assertEquals(0, context.getPartition().getId());
            assertEquals("key", ingest.getKey());
            context.confirm();
            latch.get().countDown();
            return false;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    assertTrue(ObserveHandleUtils.isAtHead(handle, reader));
    writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key", data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 }), (succ, exc) -> {
    });
    latch.get().await();
}

Example 3 with InMemStorage

use of cz.o2.proxima.direct.storage.InMemStorage in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testReadWriteSequentialIds.

@Test(timeout = 10000)
public void testReadWriteSequentialIds() throws InterruptedException {
    InMemStorage storage = new InMemStorage();
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(URI.create("inmem:///inmemstoragetest")));
    CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    List<StreamElement> result = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    CommitLogObserver observer = LogObserverUtils.toList(result, Assert::assertTrue, elem -> {
        latch.countDown();
        return false;
    });
    reader.observe("test", observer);
    writer.online().write(StreamElement.upsert(entity, data, 1L, "key", data.getName(), System.currentTimeMillis(), new byte[] { 1, 2, 3 }), (succ, exc) -> {
    });
    latch.await();
    assertEquals(1, result.size());
    assertTrue(result.get(0).hasSequentialId());
    assertEquals(1L, result.get(0).getSequentialId());
}

Example 4 with InMemStorage

use of cz.o2.proxima.direct.storage.InMemStorage in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveError.

@Test(timeout = 1000L)
public void testObserveError() throws InterruptedException {
    final URI uri = URI.create("inmem:///inmemstoragetest");
    final InMemStorage storage = new InMemStorage();
    final DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(uri));
    final CommitLogReader reader = Optionals.get(accessor.getCommitLogReader(direct.getContext()));
    final AttributeWriterBase writer = Optionals.get(accessor.getWriter(direct.getContext()));
    final CountDownLatch failingObserverErrorReceived = new CountDownLatch(1);
    final AtomicInteger failingObserverMessages = new AtomicInteger(0);
    reader.observe("failing-observer", new CommitLogObserver() {

        @Override
        public void onCompleted() {
            throw new UnsupportedOperationException("This should never happen.");
        }

        @Override
        public boolean onError(Throwable error) {
            failingObserverErrorReceived.countDown();
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            failingObserverMessages.incrementAndGet();
            throw new RuntimeException("Test exception.");
        }
    });
    final int numElements = 100;
    final CountDownLatch successObserverAllReceived = new CountDownLatch(numElements);
    reader.observe("success-observer", new CommitLogObserver() {

        @Override
        public void onCompleted() {
            throw new UnsupportedOperationException("This should never happen.");
        }

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            successObserverAllReceived.countDown();
            return true;
        }
    });
    for (int i = 0; i < numElements; i++) {
        writer.online().write(StreamElement.upsert(entity, data, UUID.randomUUID().toString(), "key_" + i, data.getName(), System.currentTimeMillis(), new byte[] { 2 }), (success, error) -> {
        });
    }
    failingObserverErrorReceived.await();
    successObserverAllReceived.await();
    assertEquals(1, failingObserverMessages.get());
}

Example 5 with InMemStorage

use of cz.o2.proxima.direct.storage.InMemStorage in project proxima-platform by O2-Czech-Republic.

the class InMemStorageTest method testObserveWithEndOfTime.

@Test
public void testObserveWithEndOfTime() throws InterruptedException {
    URI uri = URI.create("inmem:///inmemstoragetest");
    InMemStorage storage = new InMemStorage();
    InMemStorage.setWatermarkEstimatorFactory(uri, (stamp, name, offset) -> new WatermarkEstimator() {

        {
            Preconditions.checkArgument(offset != null);
        }

        @Override
        public long getWatermark() {
            return Watermarks.MAX_WATERMARK - InMemStorage.getBoundedOutOfOrderness();
        }

        @Override
        public void update(StreamElement element) {
        }

        @Override
        public void setMinWatermark(long minWatermark) {
        }
    });
    DataAccessor accessor = storage.createAccessor(direct, createFamilyDescriptor(uri));
    CommitLogReader reader = accessor.getCommitLogReader(direct.getContext()).orElseThrow(() -> new IllegalStateException("Missing commit log reader"));
    CountDownLatch completed = new CountDownLatch(1);
    reader.observe("observer", new CommitLogObserver() {

        @Override
        public void onCompleted() {
            completed.countDown();
        }

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            return false;
        }
    });
    assertTrue(completed.await(1, TimeUnit.SECONDS));
}