Examples with TopicPath org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath used on opensource projects

Example 11 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class PubsubUnboundedSourceTest method noSubscriptionNoSplitGeneratesSubscription.

@Test
public void noSubscriptionNoSplitGeneratesSubscription() throws Exception {
    TopicPath topicPath = PubsubClient.topicPathFromName("my_project", "my_topic");
    factory = PubsubTestClient.createFactoryForCreateSubscription();
    PubsubUnboundedSource source = new PubsubUnboundedSource(factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), StaticValueProvider.of(topicPath), null, /* subscription */
    null, /* timestampLabel */
    null, /* idLabel */
    false);
    assertThat(source.getSubscription(), nullValue());
    assertThat(source.getSubscription(), nullValue());
    PipelineOptions options = PipelineOptionsFactory.create();
    PubsubSource actualSource = new PubsubSource(source);
    PubsubReader reader = actualSource.createReader(options, null);
    SubscriptionPath createdSubscription = reader.subscription;
    assertThat(createdSubscription, not(nullValue()));
    PubsubCheckpoint checkpoint = reader.getCheckpointMark();
    assertThat(checkpoint.subscriptionPath, equalTo(createdSubscription.getPath()));
    checkpoint.finalizeCheckpoint();
    PubsubCheckpoint deserCheckpoint = CoderUtils.clone(actualSource.getCheckpointMarkCoder(), checkpoint);
    assertThat(checkpoint.subscriptionPath, not(nullValue()));
    assertThat(checkpoint.subscriptionPath, equalTo(deserCheckpoint.subscriptionPath));
    PubsubReader readerFromOriginal = actualSource.createReader(options, checkpoint);
    PubsubReader readerFromDeser = actualSource.createReader(options, deserCheckpoint);
    assertThat(readerFromOriginal.subscription, equalTo(createdSubscription));
    assertThat(readerFromDeser.subscription, equalTo(createdSubscription));
}

Example 12 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class PubsubHelper method createTopic.

/**
 * Create a topic from short name. Delete it if it already exists. Ensure the topic will be
 * deleted on cleanup. Return full topic name.
 */
public TopicPath createTopic(String shortTopic) throws IOException {
    TopicPath topic = PubsubClient.topicPathFromName(project, shortTopic);
    while (true) {
        try {
            NexmarkUtils.console("create topic %s", topic);
            pubsubClient.createTopic(topic);
            createdTopics.add(topic);
            return topic;
        } catch (GoogleJsonResponseException ex) {
            NexmarkUtils.console("attempting to cleanup topic %s", topic);
            pubsubClient.deleteTopic(topic);
            try {
                if (!BackOffUtils.next(sleeper, backOff)) {
                    NexmarkUtils.console("too many retries for creating topic %s", topic);
                    throw ex;
                }
            } catch (InterruptedException in) {
                throw new IOException(in);
            }
        }
    }
}

Example 13 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class PubsubHelper method cleanup.

/**
 * Delete all the subscriptions and topics we created.
 */
public void cleanup() {
    for (SubscriptionPath subscription : createdSubscriptions) {
        try {
            NexmarkUtils.console("delete subscription %s", subscription);
            pubsubClient.deleteSubscription(subscription);
        } catch (IOException ex) {
            NexmarkUtils.console("could not delete subscription %s", subscription);
        }
    }
    for (TopicPath topic : createdTopics) {
        try {
            NexmarkUtils.console("delete topic %s", topic);
            pubsubClient.deleteTopic(topic);
        } catch (IOException ex) {
            NexmarkUtils.console("could not delete topic %s", topic);
        }
    }
}

Example 14 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class PubsubHelper method createSubscription.

/**
 * Create subscription from short name. Ensure the subscription will be deleted on cleanup. Return
 * full subscription name.
 */
public SubscriptionPath createSubscription(String shortTopic, String shortSubscription) throws IOException {
    TopicPath topic = PubsubClient.topicPathFromName(project, shortTopic);
    SubscriptionPath subscription = PubsubClient.subscriptionPathFromName(project, shortSubscription);
    while (true) {
        try {
            NexmarkUtils.console("create subscription %s", subscription);
            pubsubClient.createSubscription(topic, subscription, 60);
            createdSubscriptions.add(subscription);
            return subscription;
        } catch (GoogleJsonResponseException ex) {
            NexmarkUtils.console("attempting to cleanup subscription %s", subscription);
            pubsubClient.deleteSubscription(subscription);
            try {
                if (!BackOffUtils.next(sleeper, backOff)) {
                    NexmarkUtils.console("too many retries for creating subscription %s", subscription);
                    throw ex;
                }
            } catch (InterruptedException in) {
                throw new IOException(in);
            }
        }
    }
}

Example 15 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class PubSubWritePayloadTranslationTest method testTranslateSinkWithTopicOverridden.

@Test
public void testTranslateSinkWithTopicOverridden() throws Exception {
    ValueProvider<TopicPath> runtimeProvider = pipeline.newProvider(TOPIC);
    PubsubUnboundedSink pubsubUnboundedSinkSink = new PubsubUnboundedSink(null, runtimeProvider, TIMESTAMP_ATTRIBUTE, ID_ATTRIBUTE, 0, 0, 0, Duration.ZERO, null);
    PubsubSink pubsubSink = new PubsubSink(pubsubUnboundedSinkSink);
    PCollection<byte[]> input = pipeline.apply(Create.of(new byte[0]));
    PDone output = input.apply(pubsubSink);
    AppliedPTransform<?, ?, PubsubSink> appliedPTransform = AppliedPTransform.of("sink", PValues.expandInput(input), PValues.expandOutput(output), pubsubSink, ResourceHints.create(), pipeline);
    SdkComponents components = SdkComponents.create();
    components.registerEnvironment(Environments.createDockerEnvironment("java"));
    RunnerApi.FunctionSpec spec = sinkTranslator.translate(appliedPTransform, components);
    assertEquals(PTransformTranslation.PUBSUB_WRITE, spec.getUrn());
    PubSubWritePayload payload = PubSubWritePayload.parseFrom(spec.getPayload());
    assertEquals(((NestedValueProvider) runtimeProvider).propertyName(), payload.getTopicRuntimeOverridden());
    assertTrue(payload.getTopic().isEmpty());
    assertEquals(TIMESTAMP_ATTRIBUTE, payload.getTimestampAttribute());
    assertEquals(ID_ATTRIBUTE, payload.getIdAttribute());
}