Examples with PythonDependencyInfo org.apache.flink.python.env.PythonDependencyInfo used on opensource projects

Example 1 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class ProcessPythonEnvironmentManagerTest method testRequirements.

@Test
public void testRequirements() throws Exception {
    PythonDependencyInfo dependencyInfo = new PythonDependencyInfo(new HashMap<>(), String.join(File.separator, tmpDir, "file0"), String.join(File.separator, tmpDir, "dir0"), new HashMap<>(), "python");
    try (ProcessPythonEnvironmentManager environmentManager = createBasicPythonEnvironmentManager(dependencyInfo)) {
        File baseDirectory = new File(tmpDir, "python-dist-" + UUID.randomUUID().toString());
        if (!baseDirectory.mkdirs()) {
            throw new IOException("Could not find a unique directory name in '" + tmpDir + "' for storing the generated files of python dependency.");
        }
        String tmpBase = baseDirectory.getAbsolutePath();
        Map<String, String> environmentVariable = environmentManager.constructEnvironmentVariables(tmpBase);
        Map<String, String> expected = new HashMap<>();
        expected.put("python", "python");
        expected.put("BOOT_LOG_DIR", tmpBase);
        expected.put(PYFLINK_GATEWAY_DISABLED, "true");
        expected.put(PYTHON_REQUIREMENTS_FILE, String.join(File.separator, tmpDir, "file0"));
        expected.put(PYTHON_REQUIREMENTS_CACHE, String.join(File.separator, tmpDir, "dir0"));
        expected.put(PYTHON_REQUIREMENTS_INSTALL_DIR, String.join(File.separator, tmpBase, PYTHON_REQUIREMENTS_DIR));
        assertEquals(expected, environmentVariable);
    }
}

Example 2 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class ProcessPythonEnvironmentManagerTest method testSetLogDirectory.

@Test
public void testSetLogDirectory() throws Exception {
    PythonDependencyInfo dependencyInfo = new PythonDependencyInfo(new HashMap<>(), null, null, new HashMap<>(), "python");
    try (ProcessPythonEnvironmentManager environmentManager = new ProcessPythonEnvironmentManager(dependencyInfo, new String[] { tmpDir }, new HashMap<>(), new JobID())) {
        environmentManager.open();
        Map<String, String> env = environmentManager.constructEnvironmentVariables(environmentManager.getBaseDirectory());
        Map<String, String> expected = getBasicExpectedEnv(environmentManager);
        expected.put("BOOT_LOG_DIR", environmentManager.getBaseDirectory());
        assertEquals(expected, env);
    }
}

Example 3 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class ProcessPythonEnvironmentManagerTest method testPythonExecutable.

@Test
public void testPythonExecutable() throws Exception {
    PythonDependencyInfo dependencyInfo = new PythonDependencyInfo(new HashMap<>(), null, null, new HashMap<>(), "/usr/local/bin/python");
    try (ProcessPythonEnvironmentManager environmentManager = createBasicPythonEnvironmentManager(dependencyInfo)) {
        environmentManager.open();
        Map<String, String> environmentVariable = environmentManager.getPythonEnv();
        Map<String, String> expected = getBasicExpectedEnv(environmentManager);
        expected.put("python", "/usr/local/bin/python");
        assertEquals(expected, environmentVariable);
    }
}

Example 4 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class PythonTestUtils method createTestProcessEnvironmentManager.

public static ProcessPythonEnvironmentManager createTestProcessEnvironmentManager() {
    Map<String, String> env = new HashMap<>();
    env.put(PythonEnvironmentManagerUtils.PYFLINK_UDF_RUNNER_DIR, "");
    return new ProcessPythonEnvironmentManager(new PythonDependencyInfo(new HashMap<>(), null, null, new HashMap<>(), "python"), new String[] { System.getProperty("java.io.tmpdir") }, env, new JobID());
}

Example 5 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class ProcessPythonEnvironmentManagerTest method testArchives.

@Test
public void testArchives() throws Exception {
    // use LinkedHashMap to preserve the file order in python working directory
    Map<String, String> archives = new LinkedHashMap<>();
    archives.put(String.join(File.separator, tmpDir, "zip0"), "py27.zip");
    archives.put(String.join(File.separator, tmpDir, "zip1"), "py37");
    PythonDependencyInfo dependencyInfo = new PythonDependencyInfo(new HashMap<>(), null, null, archives, "python");
    try (ProcessPythonEnvironmentManager environmentManager = createBasicPythonEnvironmentManager(dependencyInfo)) {
        environmentManager.open();
        String tmpBase = environmentManager.getBaseDirectory();
        Map<String, String> environmentVariable = environmentManager.getPythonEnv();
        Map<String, String> expected = getBasicExpectedEnv(environmentManager);
        expected.put(PYTHON_WORKING_DIR, String.join(File.separator, tmpBase, PYTHON_ARCHIVES_DIR));
        assertEquals(expected, environmentVariable);
        assertFileEquals(new File(String.join(File.separator, tmpDir, "zipExpected0")), new File(String.join(File.separator, tmpBase, PYTHON_ARCHIVES_DIR, "py27.zip")), true);
        assertFileEquals(new File(String.join(File.separator, tmpDir, "zipExpected1")), new File(String.join(File.separator, tmpBase, PYTHON_ARCHIVES_DIR, "py37")), true);
    }
}