Examples with PythonDependencyInfo org.apache.flink.python.env.PythonDependencyInfo used on opensource projects

Example 6 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class ProcessPythonEnvironmentManagerTest method testPythonFiles.

@Test
public void testPythonFiles() throws Exception {
    // use LinkedHashMap to preserve the path order in environment variable
    Map<String, String> pythonFiles = new LinkedHashMap<>();
    pythonFiles.put(String.join(File.separator, tmpDir, "zip0"), "test_zip.zip");
    pythonFiles.put(String.join(File.separator, tmpDir, "file1"), "test_file1.py");
    pythonFiles.put(String.join(File.separator, tmpDir, "file2"), "test_file2.egg");
    pythonFiles.put(String.join(File.separator, tmpDir, "dir0"), "test_dir");
    PythonDependencyInfo dependencyInfo = new PythonDependencyInfo(pythonFiles, null, null, new HashMap<>(), "python");
    try (ProcessPythonEnvironmentManager environmentManager = createBasicPythonEnvironmentManager(dependencyInfo)) {
        environmentManager.open();
        String baseDir = environmentManager.getBaseDirectory();
        Map<String, String> environmentVariable = environmentManager.getPythonEnv();
        String[] expectedUserPythonPaths = new String[] { String.join(File.separator, baseDir, PYTHON_FILES_DIR, "zip0", "test_zip"), String.join(File.separator, baseDir, PYTHON_FILES_DIR, "file1"), String.join(File.separator, baseDir, PYTHON_FILES_DIR, "file2", "test_file2.egg"), String.join(File.separator, baseDir, PYTHON_FILES_DIR, "dir0", "test_dir") };
        String expectedPythonPath = String.join(File.pathSeparator, expectedUserPythonPaths);
        assertEquals(expectedPythonPath, environmentVariable.get("PYTHONPATH"));
        assertFileEquals(new File(String.join(File.separator, tmpDir, "file1")), new File(String.join(File.separator, baseDir, PYTHON_FILES_DIR, "file1", "test_file1.py")));
        assertFileEquals(new File(String.join(File.separator, tmpDir, "zipExpected0")), new File(String.join(File.separator, baseDir, PYTHON_FILES_DIR, "zip0", "test_zip")));
        assertFileEquals(new File(String.join(File.separator, tmpDir, "file2")), new File(String.join(File.separator, baseDir, PYTHON_FILES_DIR, "file2", "test_file2.egg")));
        assertFileEquals(new File(String.join(File.separator, tmpDir, "dir0")), new File(String.join(File.separator, baseDir, PYTHON_FILES_DIR, "dir0", "test_dir")));
    }
}

Example 7 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class ProcessPythonEnvironmentManagerTest method testCreateRetrievalToken.

@Test
public void testCreateRetrievalToken() throws Exception {
    PythonDependencyInfo dependencyInfo = new PythonDependencyInfo(new HashMap<>(), null, null, new HashMap<>(), "python");
    Map<String, String> sysEnv = new HashMap<>();
    sysEnv.put("FLINK_HOME", "/flink");
    try (ProcessPythonEnvironmentManager environmentManager = new ProcessPythonEnvironmentManager(dependencyInfo, new String[] { tmpDir }, sysEnv, new JobID())) {
        environmentManager.open();
        String retrievalToken = environmentManager.createRetrievalToken();
        File retrievalTokenFile = new File(retrievalToken);
        byte[] content = new byte[(int) retrievalTokenFile.length()];
        try (DataInputStream input = new DataInputStream(new FileInputStream(retrievalToken))) {
            input.readFully(content);
        }
        assertEquals("{\"manifest\": {}}", new String(content));
    }
}

Example 8 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class ProcessPythonEnvironmentManagerTest method testOpenClose.

@Test
public void testOpenClose() throws Exception {
    PythonDependencyInfo dependencyInfo = new PythonDependencyInfo(new HashMap<>(), null, null, new HashMap<>(), "python");
    try (ProcessPythonEnvironmentManager environmentManager = createBasicPythonEnvironmentManager(dependencyInfo)) {
        environmentManager.open();
        environmentManager.createRetrievalToken();
        String tmpBase = environmentManager.getBaseDirectory();
        assertTrue(new File(tmpBase).isDirectory());
        environmentManager.close();
        assertFalse(new File(tmpBase).exists());
    }
}

Example 9 with PythonDependencyInfo

use of org.apache.flink.python.env.PythonDependencyInfo in project flink by apache.

the class AbstractExternalPythonFunctionOperator method createPythonEnvironmentManager.

@Override
protected ProcessPythonEnvironmentManager createPythonEnvironmentManager() {
    PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(pythonConfig, getRuntimeContext().getDistributedCache());
    PythonEnv pythonEnv = getPythonEnv();
    if (pythonEnv.getExecType() == PythonEnv.ExecType.PROCESS) {
        return new ProcessPythonEnvironmentManager(dependencyInfo, getContainingTask().getEnvironment().getTaskManagerInfo().getTmpDirectories(), new HashMap<>(System.getenv()), getRuntimeContext().getJobId());
    } else {
        throw new UnsupportedOperationException(String.format("Execution type '%s' is not supported.", pythonEnv.getExecType()));
    }
}