Examples with SequencedMessage com.google.cloud.pubsublite.proto.SequencedMessage used on opensource projects

Example 1 with SequencedMessage

use of com.google.cloud.pubsublite.proto.SequencedMessage in project beam by apache.

the class RowHandlerTest method fullMessageToRow.

@Test
public void fullMessageToRow() {
    RowHandler rowHandler = new RowHandler(FULL_READ_SCHEMA);
    Instant event = Instant.now();
    Instant publish = Instant.now();
    PubSubMessage userMessage = PubSubMessage.newBuilder().setKey(ByteString.copyFromUtf8("val1")).setData(ByteString.copyFromUtf8("val2")).setEventTime(Timestamps.fromMillis(event.getMillis())).putAttributes("key1", AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8("attr1")).addValues(ByteString.copyFromUtf8("attr2")).build()).putAttributes("key2", AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8("attr3")).build()).build();
    SequencedMessage sequencedMessage = SequencedMessage.newBuilder().setMessage(userMessage).setPublishTime(Timestamps.fromMillis(publish.getMillis())).build();
    Row expected = Row.withSchema(FULL_READ_SCHEMA).withFieldValue(RowHandler.MESSAGE_KEY_FIELD, "val1".getBytes(UTF_8)).withFieldValue(RowHandler.PAYLOAD_FIELD, "val2".getBytes(UTF_8)).withFieldValue(RowHandler.EVENT_TIMESTAMP_FIELD, event).withFieldValue(RowHandler.PUBLISH_TIMESTAMP_FIELD, publish).withFieldValue(RowHandler.ATTRIBUTES_FIELD, ImmutableList.of(Row.withSchema(RowHandler.ATTRIBUTES_ENTRY_SCHEMA).attachValues("key1", ImmutableList.of("attr1".getBytes(UTF_8), "attr2".getBytes(UTF_8))), Row.withSchema(RowHandler.ATTRIBUTES_ENTRY_SCHEMA).attachValues("key2", ImmutableList.of("attr3".getBytes(UTF_8))))).build();
    assertEquals(expected, rowHandler.messageToRow(sequencedMessage));
}

Example 2 with SequencedMessage

use of com.google.cloud.pubsublite.proto.SequencedMessage in project beam by apache.

the class UuidDeduplicationTransformTest method dedupesBasedOnReturnedUuid.

@Test
public void dedupesBasedOnReturnedUuid() {
    byte[] bytes = { (byte) 0x123, (byte) 0x456 };
    // These messages have different uuids, so they would both appear in the output collection if
    // the extractor is not respected.
    SequencedMessage message1 = newMessage();
    SequencedMessage message2 = newMessage();
    TestStream<SequencedMessage> messageStream = TestStream.create(ProtoCoder.of(SequencedMessage.class)).advanceWatermarkTo(START).addElements(message1, message2).advanceWatermarkToInfinity();
    PCollection<SequencedMessage> results = pipeline.apply(messageStream).apply(new UuidDeduplicationTransform(UuidDeduplicationOptions.newBuilder().setUuidExtractor(message -> Uuid.of(ByteString.copyFrom(bytes))).build()));
    PAssert.that(results).satisfies(messages -> {
        Preconditions.checkArgument(Iterables.size(messages) == 1);
        return null;
    });
    pipeline.run();
}

Example 3 with SequencedMessage

use of com.google.cloud.pubsublite.proto.SequencedMessage in project beam by apache.

the class SubscriptionPartitionProcessorImplTest method successfulClaimThenTimeout.

@Test
public void successfulClaimThenTimeout() throws Exception {
    when(tracker.tryClaim(any())).thenReturn(true);
    SettableApiFuture<Void> runDone = SettableApiFuture.create();
    SystemExecutors.getFuturesExecutor().execute(() -> {
        assertEquals(ProcessContinuation.resume(), processor.runFor(Duration.standardSeconds(3)));
        runDone.set(null);
    });
    SequencedMessage message1 = messageWithOffset(1);
    SequencedMessage message3 = messageWithOffset(3);
    leakedConsumer.accept(ImmutableList.of(message1, message3));
    runDone.get();
    InOrder order = inOrder(tracker, receiver, subscriber);
    order.verify(tracker).tryClaim(OffsetByteProgress.of(Offset.of(3), message1.getSizeBytes() + message3.getSizeBytes()));
    order.verify(receiver).outputWithTimestamp(message1, new Instant(Timestamps.toMillis(message1.getPublishTime())));
    order.verify(receiver).outputWithTimestamp(message3, new Instant(Timestamps.toMillis(message3.getPublishTime())));
    order.verify(subscriber).allowFlow(FlowControlRequest.newBuilder().setAllowedMessages(2).setAllowedBytes(message1.getSizeBytes() + message3.getSizeBytes()).build());
    assertEquals(processor.lastClaimed().get(), Offset.of(3));
}

Example 4 with SequencedMessage

use of com.google.cloud.pubsublite.proto.SequencedMessage in project beam by apache.

the class ReadWriteIT method testReadWrite.

@Test
public void testReadWrite() throws Exception {
    pipeline.getOptions().as(StreamingOptions.class).setStreaming(true);
    pipeline.getOptions().as(TestPipelineOptions.class).setBlockOnRun(false);
    TopicPath topic = createTopic(getProject(pipeline.getOptions()));
    SubscriptionPath subscription = null;
    Exception lastException = null;
    for (int i = 0; i < 30; ++i) {
        // Sleep for topic creation to propagate.
        Thread.sleep(1000);
        try {
            subscription = createSubscription(topic);
            break;
        } catch (Exception e) {
            lastException = e;
            LOG.info("Retrying exception on subscription creation.", e);
        }
    }
    if (subscription == null) {
        throw lastException;
    }
    // Publish some messages
    writeMessages(topic, pipeline);
    // Read some messages. They should be deduplicated by the time we see them, so there should be
    // exactly numMessages, one for every index in [0,MESSAGE_COUNT).
    PCollection<SequencedMessage> messages = readMessages(subscription, pipeline);
    PCollection<Integer> ids = messages.apply(MapElements.via(extractIds()));
    ids.apply("PubsubSignalTest", signal.signalSuccessWhen(BigEndianIntegerCoder.of(), testIds()));
    Supplier<Void> start = signal.waitForStart(Duration.standardMinutes(5));
    pipeline.apply(signal.signalStart());
    PipelineResult job = pipeline.run();
    start.get();
    LOG.info("Running!");
    signal.waitForSuccess(Duration.standardMinutes(5));
    // A runner may not support cancel
    try {
        job.cancel();
    } catch (UnsupportedOperationException exc) {
    // noop
    }
}

Example 5 with SequencedMessage

use of com.google.cloud.pubsublite.proto.SequencedMessage in project beam by apache.

the class UuidDeduplicationTransformTest method unrelatedUuidsProxied.

@Test
public void unrelatedUuidsProxied() {
    SequencedMessage message1 = newMessage();
    SequencedMessage message2 = newMessage();
    TestStream<SequencedMessage> messageStream = TestStream.create(ProtoCoder.of(SequencedMessage.class)).advanceWatermarkTo(START).addElements(message1).advanceWatermarkTo(START.plus(Deduplicate.DEFAULT_DURATION.dividedBy(2))).addElements(message2).advanceWatermarkToInfinity();
    PCollection<SequencedMessage> results = pipeline.apply(messageStream).apply(new UuidDeduplicationTransform(UuidDeduplicationOptions.newBuilder().build()));
    PAssert.that(results).containsInAnyOrder(message1, message2);
    pipeline.run();
}