Examples with Environment org.apache.beam.model.pipeline.v1.RunnerApi.Environment used on opensource projects

Example 21 with Environment

use of org.apache.beam.model.pipeline.v1.RunnerApi.Environment in project beam by apache.

the class Environments method createOrGetDefaultEnvironment.

public static Environment createOrGetDefaultEnvironment(PortablePipelineOptions options) {
    verifyEnvironmentOptions(options);
    String type = options.getDefaultEnvironmentType();
    String config = options.getDefaultEnvironmentConfig();
    Environment defaultEnvironment;
    if (Strings.isNullOrEmpty(type)) {
        defaultEnvironment = JAVA_SDK_HARNESS_ENVIRONMENT;
    } else {
        switch(type) {
            case ENVIRONMENT_EMBEDDED:
                defaultEnvironment = createEmbeddedEnvironment(config);
                break;
            case ENVIRONMENT_EXTERNAL:
            case ENVIRONMENT_LOOPBACK:
                defaultEnvironment = createExternalEnvironment(getExternalServiceAddress(options));
                break;
            case ENVIRONMENT_PROCESS:
                defaultEnvironment = createProcessEnvironment(options);
                break;
            case ENVIRONMENT_DOCKER:
            default:
                defaultEnvironment = createDockerEnvironment(getDockerContainerImage(options));
        }
    }
    return defaultEnvironment.toBuilder().addAllDependencies(getDeferredArtifacts(options)).addAllCapabilities(getJavaCapabilities()).build();
}

Example 22 with Environment

use of org.apache.beam.model.pipeline.v1.RunnerApi.Environment in project beam by apache.

the class GreedyStageFuser method getStageEnvironment.

private static Environment getStageEnvironment(QueryablePipeline pipeline, Set<PTransformNode> initialNodes) {
    Supplier<IllegalArgumentException> missingEnv = () -> new IllegalArgumentException(String.format("%s must be populated on all %s in a %s", Environment.class.getSimpleName(), PTransformNode.class.getSimpleName(), GreedyStageFuser.class.getSimpleName()));
    Environment env = pipeline.getEnvironment(initialNodes.iterator().next()).orElseThrow(missingEnv);
    initialNodes.forEach(transformNode -> checkArgument(env.equals(pipeline.getEnvironment(transformNode).orElseThrow(missingEnv)), "All %s in a %s must be the same. Got %s and %s", Environment.class.getSimpleName(), ExecutableStage.class.getSimpleName(), env, pipeline.getEnvironment(transformNode).get()));
    return env;
}

Example 23 with Environment

use of org.apache.beam.model.pipeline.v1.RunnerApi.Environment in project beam by apache.

the class ReadTranslationTest method testToFromProtoUnbounded.

@Test
public void testToFromProtoUnbounded() throws Exception {
    assumeThat(source, instanceOf(UnboundedSource.class));
    UnboundedSource<?, ?> unboundedSource = (UnboundedSource<?, ?>) this.source;
    SplittableParDo.PrimitiveUnboundedRead<?> unboundedRead = new SplittableParDo.PrimitiveUnboundedRead<>(Read.from(unboundedSource));
    // No environment set for unbounded sources
    ReadPayload payload = ReadTranslation.toProto(unboundedRead);
    assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.Enum.UNBOUNDED));
    UnboundedSource<?, ?> deserializedSource = ReadTranslation.unboundedSourceFromProto(payload);
    assertThat(deserializedSource, equalTo(source));
}

Example 24 with Environment

use of org.apache.beam.model.pipeline.v1.RunnerApi.Environment in project beam by apache.

the class RehydratedComponentsTest method testEnvironment.

@Test
public void testEnvironment() {
    SdkComponents sdkComponents = SdkComponents.create();
    sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java"));
    Environment env = Environments.createDockerEnvironment("java_test");
    String id = sdkComponents.registerEnvironment(env);
    RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(sdkComponents.toComponents());
    Environment rehydratedEnv = rehydratedComponents.getEnvironment(id);
    assertThat(rehydratedEnv, equalTo(env));
    assertThat(rehydratedComponents.getEnvironment(id), theInstance(rehydratedEnv));
}

Example 25 with Environment

use of org.apache.beam.model.pipeline.v1.RunnerApi.Environment in project beam by apache.

the class EnvironmentsTest method getEnvironmentPTransform.

@Test
public void getEnvironmentPTransform() throws IOException {
    Pipeline p = Pipeline.create();
    SdkComponents components = SdkComponents.create();
    Environment env = Environments.createDockerEnvironment("java");
    components.registerEnvironment(env);
    ParDoPayload payload = ParDoTranslation.translateParDo(ParDo.of(new DoFn<String, String>() {

        @ProcessElement
        public void process(ProcessContext ctxt) {
        }
    }).withOutputTags(new TupleTag<>(), TupleTagList.empty()), PCollection.createPrimitiveOutputInternal(p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, StringUtf8Coder.of()), DoFnSchemaInformation.create(), Pipeline.create(), components);
    RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(components.toComponents());
    PTransform ptransform = PTransform.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN).setPayload(payload.toByteString()).build()).setEnvironmentId(components.getOnlyEnvironmentId()).build();
    Environment env1 = Environments.getEnvironment(ptransform, rehydratedComponents).get();
    assertThat(env1, equalTo(components.toComponents().getEnvironmentsOrThrow(ptransform.getEnvironmentId())));
}