use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct in project curiostack by curioswitch.
the class MessageMarshallerTest method anyFields.
@Test
public void anyFields() throws Exception {
TestAllTypes content = TestAllTypes.newBuilder().setOptionalInt32(1234).build();
TestAny message = TestAny.newBuilder().setAnyValue(Any.pack(content)).build();
assertMatchesUpstream(message, TestAllTypes.getDefaultInstance());
TestAny messageWithDefaultAnyValue = TestAny.newBuilder().setAnyValue(Any.getDefaultInstance()).build();
assertMatchesUpstream(messageWithDefaultAnyValue);
// Well-known types have a special formatting when embedded in Any.
//
// 1. Any in Any.
Any anyMessage = Any.pack(Any.pack(content));
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
// 2. Wrappers in Any.
anyMessage = Any.pack(Int32Value.newBuilder().setValue(12345).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(UInt32Value.newBuilder().setValue(12345).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(Int64Value.newBuilder().setValue(12345).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(UInt64Value.newBuilder().setValue(12345).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(FloatValue.newBuilder().setValue(12345).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(DoubleValue.newBuilder().setValue(12345).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(BoolValue.newBuilder().setValue(true).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(StringValue.newBuilder().setValue("Hello").build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
anyMessage = Any.pack(BytesValue.newBuilder().setValue(ByteString.copyFrom(new byte[] { 1, 2 })).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
// 3. Timestamp in Any.
anyMessage = Any.pack(Timestamps.parse("1969-12-31T23:59:59Z"));
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
// 4. Duration in Any
anyMessage = Any.pack(Durations.parse("12345.10s"));
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
// 5. FieldMask in Any
anyMessage = Any.pack(FieldMaskUtil.fromString("foo.bar,baz"));
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
// 6. Struct in Any
Struct.Builder structBuilder = Struct.newBuilder();
structBuilder.putFields("number", Value.newBuilder().setNumberValue(1.125).build());
anyMessage = Any.pack(structBuilder.build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
// 7. Value (number type) in Any
Value.Builder valueBuilder = Value.newBuilder();
valueBuilder.setNumberValue(1);
anyMessage = Any.pack(valueBuilder.build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
// 8. Value (null type) in Any
anyMessage = Any.pack(Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build());
assertMatchesUpstream(anyMessage, TestAllTypes.getDefaultInstance());
}
use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct in project curiostack by curioswitch.
the class MessageMarshallerTest method anyInMaps.
@Test
public void anyInMaps() throws Exception {
TestAny.Builder testAny = TestAny.newBuilder();
testAny.putAnyMap("int32_wrapper", Any.pack(Int32Value.newBuilder().setValue(123).build()));
testAny.putAnyMap("int64_wrapper", Any.pack(Int64Value.newBuilder().setValue(456).build()));
testAny.putAnyMap("timestamp", Any.pack(Timestamps.parse("1969-12-31T23:59:59Z")));
testAny.putAnyMap("duration", Any.pack(Durations.parse("12345.1s")));
testAny.putAnyMap("field_mask", Any.pack(FieldMaskUtil.fromString("foo.bar,baz")));
Value numberValue = Value.newBuilder().setNumberValue(1.125).build();
Struct.Builder struct = Struct.newBuilder();
struct.putFields("number", numberValue);
testAny.putAnyMap("struct", Any.pack(struct.build()));
Value nullValue = Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build();
testAny.putAnyMap("list_value", Any.pack(ListValue.newBuilder().addValues(numberValue).addValues(nullValue).build()));
testAny.putAnyMap("number_value", Any.pack(numberValue));
testAny.putAnyMap("any_value_number", Any.pack(Any.pack(numberValue)));
testAny.putAnyMap("any_value_default", Any.pack(Any.getDefaultInstance()));
testAny.putAnyMap("default", Any.getDefaultInstance());
assertMatchesUpstream(testAny.build(), TestAllTypes.getDefaultInstance());
}
use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct in project flink by apache.
the class BeamPythonFunctionRunner method open.
// ------------------------------------------------------------------------
@Override
public void open(PythonConfig config) throws Exception {
this.bundleStarted = false;
this.resultBuffer = new LinkedBlockingQueue<>();
this.reusableResultTuple = new Tuple2<>();
// The creation of stageBundleFactory depends on the initialized environment manager.
environmentManager.open();
PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class);
if (jobOptions.containsKey(PythonOptions.STATE_CACHE_SIZE.key())) {
portableOptions.as(ExperimentalOptions.class).setExperiments(Collections.singletonList(ExperimentalOptions.STATE_CACHE_SIZE + "=" + jobOptions.get(PythonOptions.STATE_CACHE_SIZE.key())));
}
Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions);
if (memoryManager != null && config.isUsingManagedMemory()) {
Preconditions.checkArgument(managedMemoryFraction > 0 && managedMemoryFraction <= 1.0, "The configured managed memory fraction for Python worker process must be within (0, 1], was: %s. " + "It may be because the consumer type \"Python\" was missing or set to 0 for the config option \"taskmanager.memory.managed.consumer-weights\"." + managedMemoryFraction);
final LongFunctionWithException<PythonSharedResources, Exception> initializer = (size) -> new PythonSharedResources(createJobBundleFactory(pipelineOptions), createPythonExecutionEnvironment(size));
sharedResources = memoryManager.getSharedMemoryResourceForManagedMemory(MANAGED_MEMORY_RESOURCE_ID, initializer, managedMemoryFraction);
LOG.info("Obtained shared Python process of size {} bytes", sharedResources.getSize());
sharedResources.getResourceHandle().addPythonEnvironmentManager(environmentManager);
JobBundleFactory jobBundleFactory = sharedResources.getResourceHandle().getJobBundleFactory();
RunnerApi.Environment environment = sharedResources.getResourceHandle().getEnvironment();
stageBundleFactory = createStageBundleFactory(jobBundleFactory, environment);
} else {
// there is no way to access the MemoryManager for the batch job of old planner,
// fallback to the way that spawning a Python process for each Python operator
jobBundleFactory = createJobBundleFactory(pipelineOptions);
stageBundleFactory = createStageBundleFactory(jobBundleFactory, createPythonExecutionEnvironment(-1));
}
progressHandler = getProgressHandler(flinkMetricContainer);
}
use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct in project beam by apache.
the class FlinkPipelineRunner method main.
/**
* Main method to be called only as the entry point to an executable jar with structure as defined
* in {@link PortablePipelineJarUtils}.
*/
public static void main(String[] args) throws Exception {
// Register standard file systems.
FileSystems.setDefaultPipelineOptions(PipelineOptionsFactory.create());
FlinkPipelineRunnerConfiguration configuration = parseArgs(args);
String baseJobName = configuration.baseJobName == null ? PortablePipelineJarUtils.getDefaultJobName() : configuration.baseJobName;
Preconditions.checkArgument(baseJobName != null, "No default job name found. Job name must be set using --base-job-name.");
Pipeline pipeline = PortablePipelineJarUtils.getPipelineFromClasspath(baseJobName);
Struct originalOptions = PortablePipelineJarUtils.getPipelineOptionsFromClasspath(baseJobName);
// The retrieval token is only required by the legacy artifact service, which the Flink runner
// no longer uses.
String retrievalToken = ArtifactApi.CommitManifestResponse.Constants.NO_ARTIFACTS_STAGED_TOKEN.getValueDescriptor().getOptions().getExtension(RunnerApi.beamConstant);
FlinkPipelineOptions flinkOptions = PipelineOptionsTranslation.fromProto(originalOptions).as(FlinkPipelineOptions.class);
String invocationId = String.format("%s_%s", flinkOptions.getJobName(), UUID.randomUUID().toString());
FlinkPipelineRunner runner = new FlinkPipelineRunner(flinkOptions, configuration.flinkConfDir, detectClassPathResourcesToStage(FlinkPipelineRunner.class.getClassLoader(), flinkOptions));
JobInfo jobInfo = JobInfo.create(invocationId, flinkOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(flinkOptions));
try {
runner.run(pipeline, jobInfo);
} catch (Exception e) {
throw new RuntimeException(String.format("Job %s failed.", invocationId), e);
}
LOG.info("Job {} finished successfully.", invocationId);
}
use of org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.Struct in project beam by apache.
the class DefaultJobBundleFactoryTest method expiresEnvironment.
@Test
public void expiresEnvironment() throws Exception {
ServerFactory serverFactory = ServerFactory.createDefault();
Environment environmentA = Environment.newBuilder().setUrn("env:urn:a").build();
EnvironmentFactory envFactoryA = mock(EnvironmentFactory.class);
when(envFactoryA.createEnvironment(eq(environmentA), any())).thenReturn(remoteEnvironment);
EnvironmentFactory.Provider environmentProviderFactoryA = mock(EnvironmentFactory.Provider.class);
when(environmentProviderFactoryA.createEnvironmentFactory(any(), any(), any(), any(), any(), any())).thenReturn(envFactoryA);
when(environmentProviderFactoryA.getServerFactory()).thenReturn(serverFactory);
Map<String, Provider> environmentFactoryProviderMap = ImmutableMap.of(environmentA.getUrn(), environmentProviderFactoryA);
PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class);
portableOptions.setEnvironmentExpirationMillis(1);
Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions);
try (DefaultJobBundleFactory bundleFactory = new DefaultJobBundleFactory(JobInfo.create("testJob", "testJob", "token", pipelineOptions), environmentFactoryProviderMap, stageIdGenerator, serverInfo)) {
OutputReceiverFactory orf = mock(OutputReceiverFactory.class);
StateRequestHandler srh = mock(StateRequestHandler.class);
when(srh.getCacheTokens()).thenReturn(Collections.emptyList());
StageBundleFactory sbf = bundleFactory.forStage(getExecutableStage(environmentA));
// allow environment to expire
Thread.sleep(10);
sbf.getBundle(orf, srh, BundleProgressHandler.ignored()).close();
// allow environment to expire
Thread.sleep(10);
sbf.getBundle(orf, srh, BundleProgressHandler.ignored()).close();
}
verify(envFactoryA, Mockito.times(3)).createEnvironment(eq(environmentA), any());
verify(remoteEnvironment, Mockito.times(3)).close();
}
Aggregations