Examples with PythonInterpreterConfig pemja.core.PythonInterpreterConfig used on opensource projects

Example 1 with PythonInterpreterConfig

use of pemja.core.PythonInterpreterConfig in project flink by apache.

the class EmbeddedPythonEnvironmentManager method createEnvironment.

@Override
public PythonEnvironment createEnvironment() throws Exception {
    Map<String, String> env = new HashMap<>(getPythonEnv());
    PythonInterpreterConfig.ExecType execType;
    String executionMode = dependencyInfo.getExecutionMode();
    if (executionMode.equalsIgnoreCase("sub-interpreter")) {
        execType = PythonInterpreterConfig.ExecType.SUB_INTERPRETER;
    } else if (executionMode.equalsIgnoreCase("multi-thread")) {
        execType = PythonInterpreterConfig.ExecType.MULTI_THREAD;
    } else {
        throw new RuntimeException(String.format("Unsupported execution mode %s.", executionMode));
    }
    String pythonVersion = PythonEnvironmentManagerUtils.getPythonVersion(dependencyInfo.getPythonExec());
    if (execType == PythonInterpreterConfig.ExecType.SUB_INTERPRETER) {
        if (pythonVersion.compareTo("3.8") < 0) {
            throw new RuntimeException("`SUB-INTERPRETER` execution mode only supports Python 3.8+");
        }
    } else {
        if (pythonVersion.compareTo("3.7") < 0) {
            throw new RuntimeException("`MULTI-THREAD` execution mode only supports Python 3.7+");
        }
    }
    if (env.containsKey("FLINK_TESTING")) {
        String flinkHome = env.get("FLINK_HOME");
        String sourceRootDir = new File(flinkHome, "../../../../").getCanonicalPath();
        String flinkPython = sourceRootDir + "/flink-python";
        // add flink-python of source code to PYTHONPATH
        env.put("PYTHONPATH", flinkPython + File.pathSeparator + env.getOrDefault("PYTHONPATH", ""));
    }
    PythonInterpreterConfig interpreterConfig = PythonInterpreterConfig.newBuilder().setPythonExec(dependencyInfo.getPythonExec()).setExcType(execType).addPythonPaths(env.getOrDefault("PYTHONPATH", "")).build();
    return new EmbeddedPythonEnvironment(interpreterConfig, env);
}

Example 2 with PythonInterpreterConfig

use of pemja.core.PythonInterpreterConfig in project flink by apache.

the class AbstractEmbeddedPythonFunctionOperator method open.

@Override
public void open() throws Exception {
    super.open();
    pythonConfig = new PythonConfig(config);
    pythonEnvironmentManager = createPythonEnvironmentManager();
    pythonEnvironmentManager.open();
    EmbeddedPythonEnvironment environment = (EmbeddedPythonEnvironment) pythonEnvironmentManager.createEnvironment();
    PythonInterpreterConfig interpreterConfig = environment.getConfig();
    interpreter = new PythonInterpreter(interpreterConfig);
    Map<String, String> env = environment.getEnv();
    if (env.containsKey(PYTHON_WORKING_DIR)) {
        lock.lockInterruptibly();
        try {
            JobID jobId = getRuntimeContext().getJobId();
            Tuple2<String, Integer> dirAndNums;
            if (workingDirectories.containsKey(jobId)) {
                dirAndNums = workingDirectories.get(jobId);
            } else {
                dirAndNums = Tuple2.of(null, 0);
                workingDirectories.put(jobId, dirAndNums);
            }
            dirAndNums.f1 += 1;
            if (dirAndNums.f0 == null) {
                // get current directory.
                interpreter.exec("import os;cwd = os.getcwd();");
                dirAndNums.f0 = interpreter.get("cwd", String.class);
                String workingDirectory = env.get(PYTHON_WORKING_DIR);
                // set working directory
                interpreter.exec(String.format("import os;os.chdir('%s')", workingDirectory));
            }
        } finally {
            lock.unlock();
        }
    }
    openPythonInterpreter(pythonConfig.getPythonExec(), env, interpreterConfig.getExecType());
}