Examples with PortablePipelineOptions org.apache.beam.sdk.options.PortablePipelineOptions used on opensource projects

Example 11 with PortablePipelineOptions

use of org.apache.beam.sdk.options.PortablePipelineOptions in project beam by apache.

the class EnvironmentsTest method createEnvironmentProcessFromEnvironmentOptions.

@Test
public void createEnvironmentProcessFromEnvironmentOptions() {
    PortablePipelineOptions options = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    options.setDefaultEnvironmentType(Environments.ENVIRONMENT_PROCESS);
    options.setEnvironmentOptions(ImmutableList.of("process_command=run.sh", "process_variables=k1=v1,k2=v2"));
    assertThat(Environments.createOrGetDefaultEnvironment(options), is(Environment.newBuilder().setUrn(BeamUrns.getUrn(StandardEnvironments.Environments.PROCESS)).setPayload(ProcessPayload.newBuilder().setCommand("run.sh").putEnv("k1", "v1").putEnv("k2", "v2").build().toByteString()).addAllCapabilities(Environments.getJavaCapabilities()).build()));
}

Example 12 with PortablePipelineOptions

use of org.apache.beam.sdk.options.PortablePipelineOptions in project beam by apache.

the class EnvironmentsTest method createEnvironmentDockerFromEnvironmentConfig.

@Test
public void createEnvironmentDockerFromEnvironmentConfig() throws IOException {
    PortablePipelineOptions options = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    options.setDefaultEnvironmentType(Environments.ENVIRONMENT_DOCKER);
    options.setDefaultEnvironmentConfig("java");
    assertThat(Environments.createOrGetDefaultEnvironment(options), is(Environment.newBuilder().setUrn(BeamUrns.getUrn(StandardEnvironments.Environments.DOCKER)).setPayload(DockerPayload.newBuilder().setContainerImage("java").build().toByteString()).addAllCapabilities(Environments.getJavaCapabilities()).build()));
}

Example 13 with PortablePipelineOptions

use of org.apache.beam.sdk.options.PortablePipelineOptions in project beam by apache.

the class EnvironmentsTest method createEnvironmentExternalFromEnvironmentOptions.

@Test
public void createEnvironmentExternalFromEnvironmentOptions() {
    PortablePipelineOptions options = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    options.setDefaultEnvironmentType(Environments.ENVIRONMENT_EXTERNAL);
    options.setEnvironmentOptions(ImmutableList.of("external_service_address=foo"));
    assertThat(Environments.createOrGetDefaultEnvironment(options), is(Environment.newBuilder().setUrn(BeamUrns.getUrn(StandardEnvironments.Environments.EXTERNAL)).setPayload(RunnerApi.ExternalPayload.newBuilder().setEndpoint(Endpoints.ApiServiceDescriptor.newBuilder().setUrl("foo").build()).build().toByteString()).addAllCapabilities(Environments.getJavaCapabilities()).build()));
}

Example 14 with PortablePipelineOptions

use of org.apache.beam.sdk.options.PortablePipelineOptions in project beam by apache.

the class EnvironmentsTest method environmentConfigAndEnvironmentOptionsAreMutuallyExclusive.

@Test
public void environmentConfigAndEnvironmentOptionsAreMutuallyExclusive() {
    PortablePipelineOptions options = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    options.setDefaultEnvironmentType(Environments.ENVIRONMENT_DOCKER);
    options.setDefaultEnvironmentConfig("foo");
    options.setEnvironmentOptions(ImmutableList.of("bar"));
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage("Pipeline options defaultEnvironmentConfig and environmentOptions are mutually exclusive.");
    Environments.createOrGetDefaultEnvironment(options);
}

Example 15 with PortablePipelineOptions

use of org.apache.beam.sdk.options.PortablePipelineOptions in project beam by apache.

the class DefaultJobBundleFactoryTest method expiresEnvironment.

@Test
public void expiresEnvironment() throws Exception {
    ServerFactory serverFactory = ServerFactory.createDefault();
    Environment environmentA = Environment.newBuilder().setUrn("env:urn:a").build();
    EnvironmentFactory envFactoryA = mock(EnvironmentFactory.class);
    when(envFactoryA.createEnvironment(eq(environmentA), any())).thenReturn(remoteEnvironment);
    EnvironmentFactory.Provider environmentProviderFactoryA = mock(EnvironmentFactory.Provider.class);
    when(environmentProviderFactoryA.createEnvironmentFactory(any(), any(), any(), any(), any(), any())).thenReturn(envFactoryA);
    when(environmentProviderFactoryA.getServerFactory()).thenReturn(serverFactory);
    Map<String, Provider> environmentFactoryProviderMap = ImmutableMap.of(environmentA.getUrn(), environmentProviderFactoryA);
    PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    portableOptions.setEnvironmentExpirationMillis(1);
    Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions);
    try (DefaultJobBundleFactory bundleFactory = new DefaultJobBundleFactory(JobInfo.create("testJob", "testJob", "token", pipelineOptions), environmentFactoryProviderMap, stageIdGenerator, serverInfo)) {
        OutputReceiverFactory orf = mock(OutputReceiverFactory.class);
        StateRequestHandler srh = mock(StateRequestHandler.class);
        when(srh.getCacheTokens()).thenReturn(Collections.emptyList());
        StageBundleFactory sbf = bundleFactory.forStage(getExecutableStage(environmentA));
        // allow environment to expire
        Thread.sleep(10);
        sbf.getBundle(orf, srh, BundleProgressHandler.ignored()).close();
        // allow environment to expire
        Thread.sleep(10);
        sbf.getBundle(orf, srh, BundleProgressHandler.ignored()).close();
    }
    verify(envFactoryA, Mockito.times(3)).createEnvironment(eq(environmentA), any());
    verify(remoteEnvironment, Mockito.times(3)).close();
}