Examples with Struct org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct used on opensource projects

Example 11 with Struct

use of org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct in project curiostack by curioswitch.

the class MessageMarshallerTest method anyFields.

@Test
public void anyFields() throws Exception {
    TestAllTypes content = TestAllTypes.newBuilder().setOptionalInt32(1234).build();
    TestAny message = TestAny.newBuilder().setAnyValue(Any.pack(content)).build();
    assertMatchesUpstream(message, TestAllTypes.getDefaultInstance());
    TestAny messageWithDefaultAnyValue = TestAny.newBuilder().setAnyValue(Any.getDefaultInstance()).build();
    assertMatchesUpstream(messageWithDefaultAnyValue);
    // Well-known types have a special formatting when embedded in Any.
    // 
    // 1. Any in Any.
    Any anyMessage = Any.pack(Any.pack(content));
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    // 2. Wrappers in Any.
    anyMessage = Any.pack(Int32Value.newBuilder().setValue(12345).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(UInt32Value.newBuilder().setValue(12345).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(Int64Value.newBuilder().setValue(12345).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(UInt64Value.newBuilder().setValue(12345).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(FloatValue.newBuilder().setValue(12345).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(DoubleValue.newBuilder().setValue(12345).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(BoolValue.newBuilder().setValue(true).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(StringValue.newBuilder().setValue("Hello").build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    anyMessage = Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(new byte[] { 1, 2 })).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    // 3. Timestamp in Any.
    anyMessage = Any.pack(Timestamps.parse("1969-12-31T23:59:59Z"));
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    // 4. Duration in Any
    anyMessage = Any.pack(Durations.parse("12345.10s"));
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    // 5. FieldMask in Any
    anyMessage = Any.pack(FieldMaskUtil.fromString("foo.bar,baz"));
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    // 6. Struct in Any
    Struct.Builder structBuilder = Struct.newBuilder();
    structBuilder.putFields("number", Value.newBuilder().setNumberValue(1.125).build());
    anyMessage = Any.pack(structBuilder.build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    // 7. Value (number type) in Any
    Value.Builder valueBuilder = Value.newBuilder();
    valueBuilder.setNumberValue(1);
    anyMessage = Any.pack(valueBuilder.build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
    // 8. Value (null type) in Any
    anyMessage = Any.pack(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build());
    assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
}

Example 12 with Struct

use of org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct in project curiostack by curioswitch.

the class MessageMarshallerTest method anyInMaps.

@Test
public void anyInMaps() throws Exception {
    TestAny.Builder testAny = TestAny.newBuilder();
    testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build()));
    testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build()));
    testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z")));
    testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s")));
    testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz")));
    Value numberValue = Value.newBuilder().setNumberValue(1.125).build();
    Struct.Builder struct = Struct.newBuilder();
    struct.putFields("number", numberValue);
    testAny.putAnyMap("struct", Any.pack(struct.build()));
    Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build();
    testAny.putAnyMap("list_value", Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build()));
    testAny.putAnyMap("number_value", Any.pack(numberValue));
    testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue)));
    testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance()));
    testAny.putAnyMap("default", Any.getDefaultInstance());
    assertMatchesUpstream(testAny.build(), TestAllTypes.getDefaultInstance());
}

Example 13 with Struct

use of org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct in project flink by apache.

the class BeamPythonFunctionRunner method open.

// ------------------------------------------------------------------------
@Override
public void open(PythonConfig config) throws Exception {
    this.bundleStarted = false;
    this.resultBuffer = new LinkedBlockingQueue<>();
    this.reusableResultTuple = new Tuple2<>();
    // The creation of stageBundleFactory depends on the initialized environment manager.
    environmentManager.open();
    PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    if (jobOptions.containsKey(PythonOptions.STATE_CACHE_SIZE.key())) {
        portableOptions.as(ExperimentalOptions.class).setExperiments(Collections.singletonList(ExperimentalOptions.STATE_CACHE_SIZE + "=" + jobOptions.get(PythonOptions.STATE_CACHE_SIZE.key())));
    }
    Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions);
    if (memoryManager != null && config.isUsingManagedMemory()) {
        Preconditions.checkArgument(managedMemoryFraction > 0 && managedMemoryFraction <= 1.0, "The configured managed memory fraction for Python worker process must be within (0, 1], was: %s. " + "It may be because the consumer type \"Python\" was missing or set to 0 for the config option \"taskmanager.memory.managed.consumer-weights\"." + managedMemoryFraction);
        final LongFunctionWithException<PythonSharedResources, Exception> initializer = (size) -> new PythonSharedResources(createJobBundleFactory(pipelineOptions), createPythonExecutionEnvironment(size));
        sharedResources = memoryManager.getSharedMemoryResourceForManagedMemory(MANAGED_MEMORY_RESOURCE_ID, initializer, managedMemoryFraction);
        LOG.info("Obtained shared Python process of size {} bytes", sharedResources.getSize());
        sharedResources.getResourceHandle().addPythonEnvironmentManager(environmentManager);
        JobBundleFactory jobBundleFactory = sharedResources.getResourceHandle().getJobBundleFactory();
        RunnerApi.Environment environment = sharedResources.getResourceHandle().getEnvironment();
        stageBundleFactory = createStageBundleFactory(jobBundleFactory, environment);
    } else {
        // there is no way to access the MemoryManager for the batch job of old planner,
        // fallback to the way that spawning a Python process for each Python operator
        jobBundleFactory = createJobBundleFactory(pipelineOptions);
        stageBundleFactory = createStageBundleFactory(jobBundleFactory, createPythonExecutionEnvironment(-1));
    }
    progressHandler = getProgressHandler(flinkMetricContainer);
}

Example 14 with Struct

use of org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct in project beam by apache.

the class FlinkPipelineRunner method main.

/**
 * Main method to be called only as the entry point to an executable jar with structure as defined
 * in {@link PortablePipelineJarUtils}.
 */
public static void main(String[] args) throws Exception {
    // Register standard file systems.
    FileSystems.setDefaultPipelineOptions(PipelineOptionsFactory.create());
    FlinkPipelineRunnerConfiguration configuration = parseArgs(args);
    String baseJobName = configuration.baseJobName == null ? PortablePipelineJarUtils.getDefaultJobName() : configuration.baseJobName;
    Preconditions.checkArgument(baseJobName != null, "No default job name found. Job name must be set using --base-job-name.");
    Pipeline pipeline = PortablePipelineJarUtils.getPipelineFromClasspath(baseJobName);
    Struct originalOptions = PortablePipelineJarUtils.getPipelineOptionsFromClasspath(baseJobName);
    // The retrieval token is only required by the legacy artifact service, which the Flink runner
    // no longer uses.
    String retrievalToken = ArtifactApi.CommitManifestResponse.Constants.NO_ARTIFACTS_STAGED_TOKEN.getValueDescriptor().getOptions().getExtension(RunnerApi.beamConstant);
    FlinkPipelineOptions flinkOptions = PipelineOptionsTranslation.fromProto(originalOptions).as(FlinkPipelineOptions.class);
    String invocationId = String.format("%s_%s", flinkOptions.getJobName(), UUID.randomUUID().toString());
    FlinkPipelineRunner runner = new FlinkPipelineRunner(flinkOptions, configuration.flinkConfDir, detectClassPathResourcesToStage(FlinkPipelineRunner.class.getClassLoader(), flinkOptions));
    JobInfo jobInfo = JobInfo.create(invocationId, flinkOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(flinkOptions));
    try {
        runner.run(pipeline, jobInfo);
    } catch (Exception e) {
        throw new RuntimeException(String.format("Job %s failed.", invocationId), e);
    }
    LOG.info("Job {} finished successfully.", invocationId);
}

Example 15 with Struct

use of org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.Struct in project beam by apache.

the class DefaultJobBundleFactoryTest method expiresEnvironment.

@Test
public void expiresEnvironment() throws Exception {
    ServerFactory serverFactory = ServerFactory.createDefault();
    Environment environmentA = Environment.newBuilder().setUrn("env:urn:a").build();
    EnvironmentFactory envFactoryA = mock(EnvironmentFactory.class);
    when(envFactoryA.createEnvironment(eq(environmentA), any())).thenReturn(remoteEnvironment);
    EnvironmentFactory.Provider environmentProviderFactoryA = mock(EnvironmentFactory.Provider.class);
    when(environmentProviderFactoryA.createEnvironmentFactory(any(), any(), any(), any(), any(), any())).thenReturn(envFactoryA);
    when(environmentProviderFactoryA.getServerFactory()).thenReturn(serverFactory);
    Map<String, Provider> environmentFactoryProviderMap = ImmutableMap.of(environmentA.getUrn(), environmentProviderFactoryA);
    PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    portableOptions.setEnvironmentExpirationMillis(1);
    Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions);
    try (DefaultJobBundleFactory bundleFactory = new DefaultJobBundleFactory(JobInfo.create("testJob", "testJob", "token", pipelineOptions), environmentFactoryProviderMap, stageIdGenerator, serverInfo)) {
        OutputReceiverFactory orf = mock(OutputReceiverFactory.class);
        StateRequestHandler srh = mock(StateRequestHandler.class);
        when(srh.getCacheTokens()).thenReturn(Collections.emptyList());
        StageBundleFactory sbf = bundleFactory.forStage(getExecutableStage(environmentA));
        // allow environment to expire
        Thread.sleep(10);
        sbf.getBundle(orf, srh, BundleProgressHandler.ignored()).close();
        // allow environment to expire
        Thread.sleep(10);
        sbf.getBundle(orf, srh, BundleProgressHandler.ignored()).close();
    }
    verify(envFactoryA, Mockito.times(3)).createEnvironment(eq(environmentA), any());
    verify(remoteEnvironment, Mockito.times(3)).close();
}