Examples with StreamElement cz.o2.proxima.storage.StreamElement used on opensource projects

Example 1 with StreamElement

use of cz.o2.proxima.storage.StreamElement in project proxima-platform by O2-Czech-Republic.

the class HBaseLogReaderTest method testObserveLast.

@Test(timeout = 30000)
public void testObserveLast() throws InterruptedException, IOException {
    long now = 1500000000000L;
    write("secon", "dummy", "secon", now);
    write("second", "dummy", "second", now);
    write("third", "dummy", "third", now);
    List<Partition> partitions = reader.getPartitions();
    List<String> keys = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    reader.observe(partitions.subList(2, 3), Lists.newArrayList(attr), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            assertEquals(now, element.getStamp());
            keys.add(element.getKey());
            return true;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }
    });
    latch.await();
    assertEquals(Lists.newArrayList("second", "third"), keys);
}

Example 2 with StreamElement

use of cz.o2.proxima.storage.StreamElement in project proxima-platform by O2-Czech-Republic.

the class HBaseLogReaderTest method testObserve.

@Test(timeout = 30000)
public void testObserve() throws InterruptedException, IOException {
    long now = 1500000000000L;
    write("a", "dummy", "a", now);
    write("firs", "wildcard.1", "firs", now);
    write("fir", "dummy", "fir", now);
    write("first", "dummy", "first", now);
    List<Partition> partitions = reader.getPartitions();
    List<String> keys = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    reader.observe(partitions.subList(0, 1), Lists.newArrayList(attr), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            assertEquals(now, element.getStamp());
            keys.add(element.getKey());
            return true;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }
    });
    latch.await();
    assertEquals(Arrays.asList("a", "fir"), keys);
}

Example 3 with StreamElement

use of cz.o2.proxima.storage.StreamElement in project proxima-platform by O2-Czech-Republic.

the class HBaseLogReaderTest method testObserveMultiple.

@Test(timeout = 30000)
public void testObserveMultiple() throws IOException, InterruptedException {
    long now = 1500000000000L;
    write("a", "dummy", "a", now);
    write("firs", "wildcard.1", "firs", now);
    write("fir", "dummy", "fir", now);
    write("first", "dummy", "first", now);
    List<Partition> partitions = reader.getPartitions();
    List<String> keys = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    assertEquals(3, partitions.size());
    reader.observe(partitions.subList(0, 1), Lists.newArrayList(attr, wildcard), new BatchLogObserver() {

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public boolean onNext(StreamElement element) {
            assertEquals(now, element.getStamp());
            keys.add(element.getKey());
            return true;
        }

        @Override
        public boolean onError(Throwable error) {
            throw new RuntimeException(error);
        }
    });
    latch.await();
    assertEquals(Lists.newArrayList("a", "fir", "firs"), keys);
}

Example 4 with StreamElement

use of cz.o2.proxima.storage.StreamElement in project proxima-platform by O2-Czech-Republic.

the class HBaseWriterTest method testWriteGetObserve.

@Test
public void testWriteGetObserve() throws InterruptedException {
    URI uri = URI.create("hbase://localhost:2181/users?family=u");
    HBaseWriter writer = new HBaseWriter(uri, cluster.getConfiguration(), Collections.emptyMap());
    HBaseLogReader reader = new HBaseLogReader(uri, cluster.getConfiguration(), entity, Executors::newCachedThreadPool);
    RandomHBaseReader randomReader = new RandomHBaseReader(uri, cluster.getConfiguration(), Collections.emptyMap(), entity);
    StreamElement write = StreamElement.upsert(entity, wildcard, UUID.randomUUID().toString(), "key", wildcard.toAttributePrefix() + "1", System.currentTimeMillis(), new byte[] { 1, 2, 3 });
    CountDownLatch writeLatch = new CountDownLatch(1);
    writer.write(write, (succ, exc) -> writeLatch.countDown());
    writeLatch.await();
    List<StreamElement> el = new ArrayList<>();
    CountDownLatch observeLatch = new CountDownLatch(1);
    reader.observe(reader.getPartitions(), Collections.singletonList(wildcard), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            el.add(element);
            return true;
        }

        @Override
        public void onCompleted() {
            observeLatch.countDown();
        }
    });
    observeLatch.await();
    assertEquals(1, el.size());
    assertEquals(write.getKey(), el.get(0).getKey());
    assertEquals(write.getAttribute(), el.get(0).getAttribute());
    assertArrayEquals(write.getValue(), el.get(0).getValue());
    Optional<? extends KeyValue<?>> kv = randomReader.get("key", wildcard.toAttributePrefix() + "1", wildcard, Long.MAX_VALUE);
    assertTrue(kv.isPresent());
    assertEquals(write.getKey(), el.get(0).getKey());
    assertEquals(write.getAttribute(), el.get(0).getAttribute());
    assertArrayEquals(write.getValue(), el.get(0).getValue());
}

Example 5 with StreamElement

use of cz.o2.proxima.storage.StreamElement in project proxima-platform by O2-Czech-Republic.

the class HadoopStorageTest method testObserveCancel.

@Test(timeout = 5000L)
public void testObserveCancel() throws InterruptedException {
    Map<String, Object> cfg = cfg(HadoopDataAccessor.HADOOP_ROLL_INTERVAL, -1);
    HadoopDataAccessor accessor = new HadoopDataAccessor(TestUtils.createTestFamily(entity, uri, cfg));
    CountDownLatch latch = new CountDownLatch(1);
    writeOneElement(accessor, (success, error) -> {
        assertTrue(success);
        assertNull(error);
        latch.countDown();
    }).updateWatermark(Long.MAX_VALUE);
    latch.await();
    BatchLogReader reader = accessor.getBatchLogReader(direct.getContext()).orElse(null);
    assertNotNull(reader);
    List<Partition> partitions = reader.getPartitions();
    assertEquals(1, partitions.size());
    CountDownLatch cancelledLatch = new CountDownLatch(1);
    AtomicReference<ObserveHandle> handle = new AtomicReference<>();
    handle.set(reader.observe(partitions, Collections.singletonList(attribute), new BatchLogObserver() {

        @Override
        public boolean onNext(StreamElement element) {
            handle.get().close();
            return true;
        }

        @Override
        public void onCompleted() {
            fail("onCompleted should not have been called");
        }

        @Override
        public void onCancelled() {
            cancelledLatch.countDown();
        }

        @Override
        public boolean onError(Throwable error) {
            onCancelled();
            return true;
        }
    }));
    cancelledLatch.await();
}