Examples with TopicPath org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath used on opensource projects

Example 6 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class TestPubsubSignal method initializePubsub.

private void initializePubsub(Description description) throws IOException {
    topicAdmin = TopicAdminClient.create(TopicAdminSettings.newBuilder().setCredentialsProvider(pipelineOptions::getGcpCredential).setEndpoint(pubsubEndpoint).build());
    subscriptionAdmin = SubscriptionAdminClient.create(SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pipelineOptions::getGcpCredential).setEndpoint(pubsubEndpoint).build());
    // Example topic name:
    // integ-test-TestClassName-testMethodName-2018-12-11-23-32-333-<random-long>-result
    TopicPath resultTopicPathTmp = PubsubClient.topicPathFromName(pipelineOptions.getProject(), createTopicName(description, RESULT_TOPIC_NAME));
    TopicPath startTopicPathTmp = PubsubClient.topicPathFromName(pipelineOptions.getProject(), createTopicName(description, START_TOPIC_NAME));
    topicAdmin.createTopic(resultTopicPathTmp.getPath());
    topicAdmin.createTopic(startTopicPathTmp.getPath());
    // Set these after successful creation; this signals that they need teardown
    resultTopicPath = resultTopicPathTmp;
    startTopicPath = startTopicPathTmp;
}

Example 7 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class PubsubUnboundedSource method createRandomSubscription.

private SubscriptionPath createRandomSubscription(PipelineOptions options) {
    TopicPath topicPath = topic.get();
    ProjectPath projectPath;
    if (project != null) {
        projectPath = project.get();
    } else {
        String projectId = options.as(GcpOptions.class).getProject();
        checkState(projectId != null, "Cannot create subscription to topic %s because pipeline option 'project' not specified", topicPath);
        projectPath = PubsubClient.projectPathFromId(options.as(GcpOptions.class).getProject());
    }
    try {
        try (PubsubClient pubsubClient = pubsubFactory.newClient(timestampAttribute, idAttribute, options.as(PubsubOptions.class))) {
            SubscriptionPath subscriptionPath = pubsubClient.createRandomSubscription(projectPath, topicPath, DEAULT_ACK_TIMEOUT_SEC);
            LOG.warn("Created subscription {} to topic {}." + " Note this subscription WILL NOT be deleted when the pipeline terminates", subscriptionPath, topic);
            return subscriptionPath;
        }
    } catch (Exception e) {
        throw new RuntimeException(String.format("Failed to create subscription to topic %s on project %s: %s", topicPath, projectPath, e.getMessage()), e);
    }
}

Example 8 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class TestPubsub method initializePubsub.

private void initializePubsub(Description description) throws IOException {
    if (isLocalhost) {
        channel = ManagedChannelBuilder.forTarget(pubsubEndpoint).usePlaintext().build();
    } else {
        channel = ManagedChannelBuilder.forTarget(pubsubEndpoint).useTransportSecurity().build();
    }
    channelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel));
    topicAdmin = TopicAdminClient.create(TopicAdminSettings.newBuilder().setCredentialsProvider(pipelineOptions::getGcpCredential).setTransportChannelProvider(channelProvider).setEndpoint(pubsubEndpoint).build());
    subscriptionAdmin = SubscriptionAdminClient.create(SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pipelineOptions::getGcpCredential).setTransportChannelProvider(channelProvider).setEndpoint(pubsubEndpoint).build());
    TopicPath eventsTopicPathTmp = PubsubClient.topicPathFromName(pipelineOptions.getProject(), createTopicName(description, EVENTS_TOPIC_NAME));
    topicAdmin.createTopic(eventsTopicPathTmp.getPath());
    // Set this after successful creation; it signals that the topic needs teardown
    eventsTopicPath = eventsTopicPathTmp;
    String subscriptionName = topicPath().getName() + "_beam_" + ThreadLocalRandom.current().nextLong();
    SubscriptionPath subscriptionPathTmp = new SubscriptionPath(String.format("projects/%s/subscriptions/%s", pipelineOptions.getProject(), subscriptionName));
    subscriptionAdmin.createSubscription(subscriptionPathTmp.getPath(), topicPath().getPath(), PushConfig.getDefaultInstance(), DEFAULT_ACK_DEADLINE_SECONDS);
    subscriptionPath = subscriptionPathTmp;
}

Example 9 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class TestPubsub method publish.

/**
 * Publish messages to {@link #topicPath()}.
 */
public void publish(List<PubsubMessage> messages) {
    Preconditions.checkNotNull(eventsTopicPath);
    Publisher eventPublisher;
    try {
        eventPublisher = Publisher.newBuilder(eventsTopicPath.getPath()).setCredentialsProvider(pipelineOptions::getGcpCredential).setChannelProvider(channelProvider).setEndpoint(pubsubEndpoint).build();
    } catch (IOException e) {
        throw new RuntimeException("Error creating event publisher", e);
    }
    List<ApiFuture<String>> futures = messages.stream().map((message) -> {
        com.google.pubsub.v1.PubsubMessage.Builder builder = com.google.pubsub.v1.PubsubMessage.newBuilder().setData(ByteString.copyFrom(message.getPayload())).putAllAttributes(message.getAttributeMap());
        return eventPublisher.publish(builder.build());
    }).collect(Collectors.toList());
    try {
        ApiFutures.allAsList(futures).get();
    } catch (ExecutionException e) {
        throw new RuntimeException("Error publishing a test message", e);
    } catch (InterruptedException e) {
        throw new RuntimeException("Interrupted while waiting for messages to publish", e);
    }
    eventPublisher.shutdown();
}

Example 10 with TopicPath

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath in project beam by apache.

the class PubsubUnboundedSourceTest method noSubscriptionSplitGeneratesSubscription.

@Test
public void noSubscriptionSplitGeneratesSubscription() throws Exception {
    TopicPath topicPath = PubsubClient.topicPathFromName("my_project", "my_topic");
    factory = PubsubTestClient.createFactoryForCreateSubscription();
    PubsubUnboundedSource source = new PubsubUnboundedSource(factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), StaticValueProvider.of(topicPath), null, /* subscription */
    null, /* timestampLabel */
    null, /* idLabel */
    false);
    assertThat(source.getSubscription(), nullValue());
    assertThat(source.getSubscription(), nullValue());
    PipelineOptions options = PipelineOptionsFactory.create();
    List<PubsubSource> splits = new PubsubSource(source).split(3, options);
    // We have at least one returned split
    assertThat(splits, hasSize(greaterThan(0)));
    for (PubsubSource split : splits) {
        // Each split is equal
        assertThat(split, equalTo(splits.get(0)));
    }
    assertThat(splits.get(0).subscriptionPath, not(nullValue()));
}