Examples with OnlineAttributeWriter cz.o2.proxima.direct.core.OnlineAttributeWriter used on opensource projects

Example 1 with OnlineAttributeWriter

use of cz.o2.proxima.direct.core.OnlineAttributeWriter in project proxima-platform by O2-Czech-Republic.

the class LocalKafkaCommitLogDescriptorTest method testWriteNull.

@Test
public void testWriteNull() {
    LocalKafkaCommitLogDescriptor.Accessor accessor;
    accessor = kafka.createAccessor(direct, createTestFamily(entity, storageUri, partitionsCfg(2)));
    OnlineAttributeWriter writer = Optionals.get(accessor.getWriter(context())).online();
    long now = 1234567890000L;
    KafkaConsumer<Object, Object> consumer = accessor.createConsumerFactory().create();
    writer.write(StreamElement.upsert(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), now, new byte[] { 1 }), (succ, exc) -> {
    });
    writer.write(StreamElement.delete(entity, attr, UUID.randomUUID().toString(), "key", attr.getName(), now + 1000), (succ, exc) -> {
    });
    ConsumerRecords<Object, Object> polled = consumer.poll(Duration.ofMillis(100));
    assertEquals(2, polled.count());
    int matched = 0;
    for (ConsumerRecord<Object, Object> r : polled) {
        if (r.timestamp() == now) {
            assertEquals(1, ((byte[]) r.value()).length);
            matched++;
        } else if (r.timestamp() == now + 1000) {
            assertNull(r.value());
            matched++;
        }
    }
    assertEquals(2, matched);
}

Example 2 with OnlineAttributeWriter

use of cz.o2.proxima.direct.core.OnlineAttributeWriter in project proxima-platform by O2-Czech-Republic.

the class KafkaLogReaderIT method writePoisonedPills.

/**
 * Write poisoned pills (element with timestamp = {@link Watermarks#MAX_WATERMARK}) to all
 * partitions.
 *
 * @param numPartitions Number of partitions in topic.
 * @return Completion latch.
 */
private CountDownLatch writePoisonedPills(int numPartitions) {
    final OnlineAttributeWriter writer = Optionals.get(operator.getWriter(fooDescriptor));
    // We assume test uses default partitioner.
    final KeyPartitioner keyPartitioner = new KeyPartitioner();
    final Set<Integer> poisonedPartitions = new HashSet<>();
    final CountDownLatch done = new CountDownLatch(numPartitions);
    for (int i = 0; poisonedPartitions.size() < numPartitions; i++) {
        final StreamElement poisonedPill = StreamElement.upsert(entity, fooDescriptor, UUID.randomUUID().toString(), String.format("poisoned-pill-%d", i), fooDescriptor.getName(), Watermarks.MAX_WATERMARK, "value".getBytes(StandardCharsets.UTF_8));
        final int partition = (keyPartitioner.getPartitionId(poisonedPill) & Integer.MAX_VALUE) % numPartitions;
        if (poisonedPartitions.add(partition)) {
            writer.write(poisonedPill, ((success, error) -> {
                if (success) {
                    done.countDown();
                }
            }));
        }
    }
    return done;
}

Example 3 with OnlineAttributeWriter

use of cz.o2.proxima.direct.core.OnlineAttributeWriter in project proxima-platform by O2-Czech-Republic.

the class ModelTest method testReadWrite.

@Test
public void testReadWrite() throws InterruptedException {
    Regular<BaseEvent> desc = model.getEvent().getDataDescriptor();
    Optional<OnlineAttributeWriter> writer = model.directOperator().getWriter(desc);
    writer.get().write(desc.upsert(UUID.randomUUID().toString(), System.currentTimeMillis(), BaseEvent.newBuilder().setUserName("user").build()), (succ, exc) -> {
    });
    Optional<CommitLogReader> reader = model.directOperator().getCommitLogReader(desc);
    List<String> users = new ArrayList<>();
    CountDownLatch latch = new CountDownLatch(1);
    reader.get().observeBulk("consumer", Position.OLDEST, true, new CommitLogObserver() {

        @Override
        public boolean onError(Throwable error) {
            return false;
        }

        @Override
        public void onCompleted() {
            latch.countDown();
        }

        @Override
        public boolean onNext(StreamElement ingest, OnNextContext context) {
            Optional<BaseEvent> baseEvent = desc.valueOf(ingest);
            users.add(baseEvent.get().getUserName());
            return true;
        }
    });
    latch.await();
    assertEquals("user", users.get(0));
}

Example 4 with OnlineAttributeWriter

use of cz.o2.proxima.direct.core.OnlineAttributeWriter in project proxima-platform by O2-Czech-Republic.

the class DirectBatchUnboundedSourceTest method testOffsetWithNoAdvance.

@Test
public void testOffsetWithNoAdvance() throws Exception {
    PipelineOptions opts = PipelineOptionsFactory.create();
    Repository repo = Repository.ofTest(ConfigFactory.load("test-reference.conf").resolve());
    DirectBatchUnboundedSource source = DirectBatchUnboundedSource.of(repo.asFactory(), Optionals.get(direct.getBatchLogReader(armed)), Collections.singletonList(repo.getEntity("gateway").getAttribute("armed")), Long.MIN_VALUE, Long.MAX_VALUE, Collections.emptyMap());
    List<? extends UnboundedSource<StreamElement, Checkpoint>> split = source.split(1, opts);
    assertEquals(1, split.size());
    OnlineAttributeWriter writer = Optionals.get(direct.getWriter(armed));
    long now = System.currentTimeMillis();
    writer.write(StreamElement.upsert(gateway, armed, UUID.randomUUID().toString(), "key", armed.getName(), now, new byte[] {}), (succ, exc) -> {
    });
    UnboundedReader<StreamElement> reader = split.get(0).createReader(opts, null);
    while (!reader.advance()) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    Checkpoint checkpoint = (Checkpoint) reader.getCheckpointMark();
    assertNotNull(checkpoint);
    assertEquals(1, checkpoint.getSkipFromFirst());
    reader = split.get(0).createReader(opts, checkpoint);
    checkpoint = (Checkpoint) reader.getCheckpointMark();
    TestUtils.assertSerializable(source);
    TestUtils.assertHashCodeAndEquals(source, SerializableUtils.clone(source));
    assertEquals(checkpoint, reader.getCheckpointMark());
    TestUtils.assertSerializable(checkpoint);
    TestUtils.assertHashCodeAndEquals(checkpoint, SerializableUtils.clone(checkpoint));
}

Example 5 with OnlineAttributeWriter

use of cz.o2.proxima.direct.core.OnlineAttributeWriter in project proxima-platform by O2-Czech-Republic.

the class DirectUnboundedSourceTest method testCheckpointWithNoAdvance.

@Test
public void testCheckpointWithNoAdvance() throws Exception {
    OnlineAttributeWriter writer = Optionals.get(direct.getWriter(armed));
    CommitLogReader commitLogReader = Optionals.get(direct.getCommitLogReader(armed));
    DirectUnboundedSource source = DirectUnboundedSource.of(repo.asFactory(), "name", commitLogReader, Position.OLDEST, true, Long.MAX_VALUE);
    long now = System.currentTimeMillis();
    writer.write(StreamElement.upsert(gateway, armed, UUID.randomUUID().toString(), "key", armed.getName(), now, new byte[] {}), (succ, exc) -> {
    });
    PipelineOptions opts = PipelineOptionsFactory.create();
    UnboundedSource<StreamElement, Checkpoint> split = source.split(1, opts).get(0);
    UnboundedReader<StreamElement> reader = split.createReader(opts, null);
    boolean start = reader.start();
    while (!start && !reader.advance()) {
        TimeUnit.MILLISECONDS.sleep(10);
    }
    Checkpoint mark = (Checkpoint) reader.getCheckpointMark();
    assertNotNull(mark);
    reader = split.createReader(opts, mark);
    TestUtils.assertSerializable(source);
    TestUtils.assertHashCodeAndEquals(source, SerializableUtils.clone(source));
    assertEquals(mark, reader.getCheckpointMark());
    TestUtils.assertSerializable(mark);
    TestUtils.assertHashCodeAndEquals(mark, SerializableUtils.clone(mark));
}