Examples with PipelineOptions org.apache.beam.sdk.options.PipelineOptions used on opensource projects

Example 91 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class PubsubUnboundedSourceTest method noSubscriptionNoSplitGeneratesSubscription.

@Test
public void noSubscriptionNoSplitGeneratesSubscription() throws Exception {
    TopicPath topicPath = PubsubClient.topicPathFromName("my_project", "my_topic");
    factory = PubsubTestClient.createFactoryForCreateSubscription();
    PubsubUnboundedSource source = new PubsubUnboundedSource(factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), StaticValueProvider.of(topicPath), null, /* subscription */
    null, /* timestampLabel */
    null, /* idLabel */
    false);
    assertThat(source.getSubscription(), nullValue());
    assertThat(source.getSubscription(), nullValue());
    PipelineOptions options = PipelineOptionsFactory.create();
    PubsubSource actualSource = new PubsubSource(source);
    PubsubReader reader = actualSource.createReader(options, null);
    SubscriptionPath createdSubscription = reader.subscription;
    assertThat(createdSubscription, not(nullValue()));
    PubsubCheckpoint checkpoint = reader.getCheckpointMark();
    assertThat(checkpoint.subscriptionPath, equalTo(createdSubscription.getPath()));
    checkpoint.finalizeCheckpoint();
    PubsubCheckpoint deserCheckpoint = CoderUtils.clone(actualSource.getCheckpointMarkCoder(), checkpoint);
    assertThat(checkpoint.subscriptionPath, not(nullValue()));
    assertThat(checkpoint.subscriptionPath, equalTo(deserCheckpoint.subscriptionPath));
    PubsubReader readerFromOriginal = actualSource.createReader(options, checkpoint);
    PubsubReader readerFromDeser = actualSource.createReader(options, deserCheckpoint);
    assertThat(readerFromOriginal.subscription, equalTo(createdSubscription));
    assertThat(readerFromDeser.subscription, equalTo(createdSubscription));
}

Example 92 with PipelineOptions

use of org.apache.beam.sdk.options.PipelineOptions in project beam by apache.

the class PubsubUnboundedSourceTest method noSubscriptionSplitGeneratesSubscription.

@Test
public void noSubscriptionSplitGeneratesSubscription() throws Exception {
    TopicPath topicPath = PubsubClient.topicPathFromName("my_project", "my_topic");
    factory = PubsubTestClient.createFactoryForCreateSubscription();
    PubsubUnboundedSource source = new PubsubUnboundedSource(factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), StaticValueProvider.of(topicPath), null, /* subscription */
    null, /* timestampLabel */
    null, /* idLabel */
    false);
    assertThat(source.getSubscription(), nullValue());
    assertThat(source.getSubscription(), nullValue());
    PipelineOptions options = PipelineOptionsFactory.create();
    List<PubsubSource> splits = (new PubsubSource(source)).split(3, options);
    // We have at least one returned split
    assertThat(splits, hasSize(greaterThan(0)));
    for (PubsubSource split : splits) {
        // Each split is equal
        assertThat(split, equalTo(splits.get(0)));
    }
    assertThat(splits.get(0).subscriptionPath, not(nullValue()));
}