Search in sources :

Example 1 with SubscriptionPath

use of com.google.cloud.pubsublite.SubscriptionPath in project beam by apache.

the class ReadWriteIT method createSubscription.

private SubscriptionPath createSubscription(TopicPath topic) throws Exception {
    SubscriptionPath toReturn = SubscriptionPath.newBuilder().setProject(topic.project()).setLocation(ZONE).setName(SubscriptionName.of(randomName())).build();
    Subscription.Builder subscription = Subscription.newBuilder().setName(toReturn.toString());
    subscription.getDeliveryConfigBuilder().setDeliveryRequirement(DeliveryRequirement.DELIVER_IMMEDIATELY);
    subscription.setTopic(topic.toString());
    cleanupActions.addLast(() -> {
        try (AdminClient client = newAdminClient()) {
            client.deleteSubscription(toReturn).get();
        } catch (Throwable t) {
            LOG.error("Failed to clean up subscription.", t);
        }
    });
    try (AdminClient client = newAdminClient()) {
        client.createSubscription(subscription.build(), BacklogLocation.BEGINNING).get();
    }
    return toReturn;
}
Also used : SubscriptionPath(com.google.cloud.pubsublite.SubscriptionPath) Subscription(com.google.cloud.pubsublite.proto.Subscription) AdminClient(com.google.cloud.pubsublite.AdminClient)

Example 2 with SubscriptionPath

use of com.google.cloud.pubsublite.SubscriptionPath in project beam by apache.

the class ReadWriteIT method testReadWrite.

@Test
public void testReadWrite() throws Exception {
    pipeline.getOptions().as(StreamingOptions.class).setStreaming(true);
    pipeline.getOptions().as(TestPipelineOptions.class).setBlockOnRun(false);
    TopicPath topic = createTopic(getProject(pipeline.getOptions()));
    SubscriptionPath subscription = null;
    Exception lastException = null;
    for (int i = 0; i < 30; ++i) {
        // Sleep for topic creation to propagate.
        Thread.sleep(1000);
        try {
            subscription = createSubscription(topic);
            break;
        } catch (Exception e) {
            lastException = e;
            LOG.info("Retrying exception on subscription creation.", e);
        }
    }
    if (subscription == null) {
        throw lastException;
    }
    // Publish some messages
    writeMessages(topic, pipeline);
    // Read some messages. They should be deduplicated by the time we see them, so there should be
    // exactly numMessages, one for every index in [0,MESSAGE_COUNT).
    PCollection<SequencedMessage> messages = readMessages(subscription, pipeline);
    PCollection<Integer> ids = messages.apply(MapElements.via(extractIds()));
    ids.apply("PubsubSignalTest", signal.signalSuccessWhen(BigEndianIntegerCoder.of(), testIds()));
    Supplier<Void> start = signal.waitForStart(Duration.standardMinutes(5));
    pipeline.apply(signal.signalStart());
    PipelineResult job = pipeline.run();
    start.get();
    LOG.info("Running!");
    signal.waitForSuccess(Duration.standardMinutes(5));
    // A runner may not support cancel
    try {
        job.cancel();
    } catch (UnsupportedOperationException exc) {
    // noop
    }
}
Also used : TopicPath(com.google.cloud.pubsublite.TopicPath) SubscriptionPath(com.google.cloud.pubsublite.SubscriptionPath) StreamingOptions(org.apache.beam.sdk.options.StreamingOptions) PipelineResult(org.apache.beam.sdk.PipelineResult) TestPipelineOptions(org.apache.beam.sdk.testing.TestPipelineOptions) SequencedMessage(com.google.cloud.pubsublite.proto.SequencedMessage) Test(org.junit.Test)

Aggregations

SubscriptionPath (com.google.cloud.pubsublite.SubscriptionPath)2 AdminClient (com.google.cloud.pubsublite.AdminClient)1 TopicPath (com.google.cloud.pubsublite.TopicPath)1 SequencedMessage (com.google.cloud.pubsublite.proto.SequencedMessage)1 Subscription (com.google.cloud.pubsublite.proto.Subscription)1 PipelineResult (org.apache.beam.sdk.PipelineResult)1 StreamingOptions (org.apache.beam.sdk.options.StreamingOptions)1 TestPipelineOptions (org.apache.beam.sdk.testing.TestPipelineOptions)1 Test (org.junit.Test)1