use of org.apache.flink.python.metric.FlinkMetricContainer in project flink by apache.
the class BeamPythonFunctionRunner method open.
// ------------------------------------------------------------------------
@Override
public void open(PythonConfig config) throws Exception {
this.bundleStarted = false;
this.resultBuffer = new LinkedBlockingQueue<>();
this.reusableResultTuple = new Tuple2<>();
// The creation of stageBundleFactory depends on the initialized environment manager.
environmentManager.open();
PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class);
if (jobOptions.containsKey(PythonOptions.STATE_CACHE_SIZE.key())) {
portableOptions.as(ExperimentalOptions.class).setExperiments(Collections.singletonList(ExperimentalOptions.STATE_CACHE_SIZE + "=" + jobOptions.get(PythonOptions.STATE_CACHE_SIZE.key())));
}
Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions);
if (memoryManager != null && config.isUsingManagedMemory()) {
Preconditions.checkArgument(managedMemoryFraction > 0 && managedMemoryFraction <= 1.0, "The configured managed memory fraction for Python worker process must be within (0, 1], was: %s. " + "It may be because the consumer type \"Python\" was missing or set to 0 for the config option \"taskmanager.memory.managed.consumer-weights\"." + managedMemoryFraction);
final LongFunctionWithException<PythonSharedResources, Exception> initializer = (size) -> new PythonSharedResources(createJobBundleFactory(pipelineOptions), createPythonExecutionEnvironment(size));
sharedResources = memoryManager.getSharedMemoryResourceForManagedMemory(MANAGED_MEMORY_RESOURCE_ID, initializer, managedMemoryFraction);
LOG.info("Obtained shared Python process of size {} bytes", sharedResources.getSize());
sharedResources.getResourceHandle().addPythonEnvironmentManager(environmentManager);
JobBundleFactory jobBundleFactory = sharedResources.getResourceHandle().getJobBundleFactory();
RunnerApi.Environment environment = sharedResources.getResourceHandle().getEnvironment();
stageBundleFactory = createStageBundleFactory(jobBundleFactory, environment);
} else {
// there is no way to access the MemoryManager for the batch job of old planner,
// fallback to the way that spawning a Python process for each Python operator
jobBundleFactory = createJobBundleFactory(pipelineOptions);
stageBundleFactory = createStageBundleFactory(jobBundleFactory, createPythonExecutionEnvironment(-1));
}
progressHandler = getProgressHandler(flinkMetricContainer);
}
Aggregations