Examples with PythonConfig org.apache.flink.python.PythonConfig used on opensource projects

Example 1 with PythonConfig

use of org.apache.flink.python.PythonConfig in project flink by apache.

the class PythonDependencyInfoTest method testParsePythonFiles.

@Test
public void testParsePythonFiles() {
    // Skip this test on Windows as we can not control the Window Driver letters.
    Assume.assumeFalse(OperatingSystem.isWindows());
    Configuration config = new Configuration();
    Map<String, String> pythonFiles = new HashMap<>();
    pythonFiles.put("python_file_{SHA256_0}", "test_file1.py");
    pythonFiles.put("python_file_{SHA256_1}", "test_file2.py");
    config.set(PythonDependencyUtils.PYTHON_FILES, pythonFiles);
    PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);
    Map<String, String> expected = new HashMap<>();
    expected.put("/distributed_cache/file0", "test_file1.py");
    expected.put("/distributed_cache/file1", "test_file2.py");
    assertEquals(expected, dependencyInfo.getPythonFiles());
}

Example 2 with PythonConfig

use of org.apache.flink.python.PythonConfig in project flink by apache.

the class PythonDependencyInfoTest method testParsePythonExec.

@Test
public void testParsePythonExec() {
    Configuration config = new Configuration();
    config.set(PythonOptions.PYTHON_EXECUTABLE, "/usr/bin/python3");
    PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);
    assertEquals("/usr/bin/python3", dependencyInfo.getPythonExec());
}

Example 3 with PythonConfig

use of org.apache.flink.python.PythonConfig in project flink by apache.

the class PythonDependencyInfoTest method testParsePythonArchives.

@Test
public void testParsePythonArchives() {
    // Skip this test on Windows as we can not control the Window Driver letters.
    Assume.assumeFalse(OperatingSystem.isWindows());
    Configuration config = new Configuration();
    Map<String, String> pythonArchives = new HashMap<>();
    pythonArchives.put("python_archive_{SHA256_0}", "py27.zip");
    pythonArchives.put("python_archive_{SHA256_1}", "py37");
    config.set(PythonDependencyUtils.PYTHON_ARCHIVES, pythonArchives);
    PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);
    Map<String, String> expected = new HashMap<>();
    expected.put("/distributed_cache/file4", "py27.zip");
    expected.put("/distributed_cache/file5", "py37");
    assertEquals(expected, dependencyInfo.getArchives());
}

Example 4 with PythonConfig

use of org.apache.flink.python.PythonConfig in project flink by apache.

the class PythonDependencyInfoTest method testParsePythonRequirements.

@Test
public void testParsePythonRequirements() throws IOException {
    // Skip this test on Windows as we can not control the Window Driver letters.
    Assume.assumeFalse(OperatingSystem.isWindows());
    Configuration config = new Configuration();
    config.set(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE, new HashMap<>());
    config.get(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE).put(PythonDependencyUtils.FILE, "python_requirements_file_{SHA256}");
    PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);
    assertEquals("/distributed_cache/file2", dependencyInfo.getRequirementsFilePath().get());
    assertFalse(dependencyInfo.getRequirementsCacheDir().isPresent());
    config.get(PythonDependencyUtils.PYTHON_REQUIREMENTS_FILE).put(PythonDependencyUtils.CACHE, "python_requirements_cache_{SHA256}");
    dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);
    assertEquals("/distributed_cache/file2", dependencyInfo.getRequirementsFilePath().get());
    assertEquals("/distributed_cache/file3", dependencyInfo.getRequirementsCacheDir().get());
}

Example 5 with PythonConfig

use of org.apache.flink.python.PythonConfig in project flink by apache.

the class BeamPythonFunctionRunner method open.

// ------------------------------------------------------------------------
@Override
public void open(PythonConfig config) throws Exception {
    this.bundleStarted = false;
    this.resultBuffer = new LinkedBlockingQueue<>();
    this.reusableResultTuple = new Tuple2<>();
    // The creation of stageBundleFactory depends on the initialized environment manager.
    environmentManager.open();
    PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class);
    if (jobOptions.containsKey(PythonOptions.STATE_CACHE_SIZE.key())) {
        portableOptions.as(ExperimentalOptions.class).setExperiments(Collections.singletonList(ExperimentalOptions.STATE_CACHE_SIZE + "=" + jobOptions.get(PythonOptions.STATE_CACHE_SIZE.key())));
    }
    Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions);
    if (memoryManager != null && config.isUsingManagedMemory()) {
        Preconditions.checkArgument(managedMemoryFraction > 0 && managedMemoryFraction <= 1.0, "The configured managed memory fraction for Python worker process must be within (0, 1], was: %s. " + "It may be because the consumer type \"Python\" was missing or set to 0 for the config option \"taskmanager.memory.managed.consumer-weights\"." + managedMemoryFraction);
        final LongFunctionWithException<PythonSharedResources, Exception> initializer = (size) -> new PythonSharedResources(createJobBundleFactory(pipelineOptions), createPythonExecutionEnvironment(size));
        sharedResources = memoryManager.getSharedMemoryResourceForManagedMemory(MANAGED_MEMORY_RESOURCE_ID, initializer, managedMemoryFraction);
        LOG.info("Obtained shared Python process of size {} bytes", sharedResources.getSize());
        sharedResources.getResourceHandle().addPythonEnvironmentManager(environmentManager);
        JobBundleFactory jobBundleFactory = sharedResources.getResourceHandle().getJobBundleFactory();
        RunnerApi.Environment environment = sharedResources.getResourceHandle().getEnvironment();
        stageBundleFactory = createStageBundleFactory(jobBundleFactory, environment);
    } else {
        // there is no way to access the MemoryManager for the batch job of old planner,
        // fallback to the way that spawning a Python process for each Python operator
        jobBundleFactory = createJobBundleFactory(pipelineOptions);
        stageBundleFactory = createStageBundleFactory(jobBundleFactory, createPythonExecutionEnvironment(-1));
    }
    progressHandler = getProgressHandler(flinkMetricContainer);
}