Examples with OutgoingMessage org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage used on opensource projects

Example 1 with OutgoingMessage

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage in project beam by apache.

the class PubsubUnboundedSinkTest method sendMoreThanOneBatchByByteSize.

@Test
public void sendMoreThanOneBatchByByteSize() throws IOException {
    List<OutgoingMessage> outgoing = new ArrayList<>();
    List<String> data = new ArrayList<>();
    int batchSize = 100;
    int batchBytes = 10;
    int n = 0;
    while (n < batchBytes * 10) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < batchBytes; i++) {
            sb.append(String.valueOf(n));
        }
        String str = sb.toString();
        outgoing.add(new OutgoingMessage(str.getBytes(), ImmutableMap.<String, String>of(), TIMESTAMP, getRecordId(str)));
        data.add(str);
        n += str.length();
    }
    try (PubsubTestClientFactory factory = PubsubTestClient.createFactoryForPublish(TOPIC, outgoing, ImmutableList.<OutgoingMessage>of())) {
        PubsubUnboundedSink sink = new PubsubUnboundedSink(factory, StaticValueProvider.of(TOPIC), TIMESTAMP_ATTRIBUTE, ID_ATTRIBUTE, NUM_SHARDS, batchSize, batchBytes, Duration.standardSeconds(2), RecordIdMethod.DETERMINISTIC);
        p.apply(Create.of(data)).apply(ParDo.of(new Stamp())).apply(sink);
        p.run();
    }
// The PubsubTestClientFactory will assert fail on close if the actual published
// message does not match the expected publish message.
}

Example 2 with OutgoingMessage

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage in project beam by apache.

the class PubsubUnboundedSinkTest method saneCoder.

@Test
public void saneCoder() throws Exception {
    OutgoingMessage message = new OutgoingMessage(DATA.getBytes(), ImmutableMap.<String, String>of(), TIMESTAMP, getRecordId(DATA));
    CoderProperties.coderDecodeEncodeEqual(PubsubUnboundedSink.CODER, message);
    CoderProperties.coderSerializable(PubsubUnboundedSink.CODER);
}

Example 3 with OutgoingMessage

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage in project beam by apache.

the class PubsubJsonClientTest method publishOneMessage.

@Test
public void publishOneMessage() throws IOException {
    String expectedTopic = TOPIC.getPath();
    PubsubMessage expectedPubsubMessage = new PubsubMessage().encodeData(DATA.getBytes()).setAttributes(ImmutableMap.<String, String>builder().put(TIMESTAMP_ATTRIBUTE, String.valueOf(MESSAGE_TIME)).put(ID_ATTRIBUTE, RECORD_ID).put("k", "v").build());
    PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage));
    PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID));
    Mockito.when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())).thenReturn(expectedResponse);
    Map<String, String> attrs = new HashMap<>();
    attrs.put("k", "v");
    OutgoingMessage actualMessage = new OutgoingMessage(DATA.getBytes(), attrs, MESSAGE_TIME, RECORD_ID);
    int n = client.publish(TOPIC, ImmutableList.of(actualMessage));
    assertEquals(1, n);
}

Example 4 with OutgoingMessage

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage in project beam by apache.

the class PubsubUnboundedSinkTest method sendOneMessage.

@Test
public void sendOneMessage() throws IOException {
    List<OutgoingMessage> outgoing = ImmutableList.of(new OutgoingMessage(DATA.getBytes(), ATTRIBUTES, TIMESTAMP, getRecordId(DATA)));
    int batchSize = 1;
    int batchBytes = 1;
    try (PubsubTestClientFactory factory = PubsubTestClient.createFactoryForPublish(TOPIC, outgoing, ImmutableList.<OutgoingMessage>of())) {
        PubsubUnboundedSink sink = new PubsubUnboundedSink(factory, StaticValueProvider.of(TOPIC), TIMESTAMP_ATTRIBUTE, ID_ATTRIBUTE, NUM_SHARDS, batchSize, batchBytes, Duration.standardSeconds(2), RecordIdMethod.DETERMINISTIC);
        p.apply(Create.of(ImmutableList.of(DATA))).apply(ParDo.of(new Stamp(ATTRIBUTES))).apply(sink);
        p.run();
    }
// The PubsubTestClientFactory will assert fail on close if the actual published
// message does not match the expected publish message.
}

Example 5 with OutgoingMessage

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage in project beam by apache.

the class PubsubUnboundedSinkTest method sendMoreThanOneBatchByNumMessages.

@Test
public void sendMoreThanOneBatchByNumMessages() throws IOException {
    List<OutgoingMessage> outgoing = new ArrayList<>();
    List<String> data = new ArrayList<>();
    int batchSize = 2;
    int batchBytes = 1000;
    for (int i = 0; i < batchSize * 10; i++) {
        String str = String.valueOf(i);
        outgoing.add(new OutgoingMessage(str.getBytes(), ImmutableMap.<String, String>of(), TIMESTAMP, getRecordId(str)));
        data.add(str);
    }
    try (PubsubTestClientFactory factory = PubsubTestClient.createFactoryForPublish(TOPIC, outgoing, ImmutableList.<OutgoingMessage>of())) {
        PubsubUnboundedSink sink = new PubsubUnboundedSink(factory, StaticValueProvider.of(TOPIC), TIMESTAMP_ATTRIBUTE, ID_ATTRIBUTE, NUM_SHARDS, batchSize, batchBytes, Duration.standardSeconds(2), RecordIdMethod.DETERMINISTIC);
        p.apply(Create.of(data)).apply(ParDo.of(new Stamp())).apply(sink);
        p.run();
    }
// The PubsubTestClientFactory will assert fail on close if the actual published
// message does not match the expected publish message.
}