Examples with PythonConfig org.apache.flink.python.PythonConfig used on opensource projects

Example 6 with PythonConfig

use of org.apache.flink.python.PythonConfig in project flink by apache.

the class AbstractEmbeddedPythonFunctionOperator method open.

@Override
public void open() throws Exception {
    super.open();
    pythonConfig = new PythonConfig(config);
    pythonEnvironmentManager = createPythonEnvironmentManager();
    pythonEnvironmentManager.open();
    EmbeddedPythonEnvironment environment = (EmbeddedPythonEnvironment) pythonEnvironmentManager.createEnvironment();
    PythonInterpreterConfig interpreterConfig = environment.getConfig();
    interpreter = new PythonInterpreter(interpreterConfig);
    Map<String, String> env = environment.getEnv();
    if (env.containsKey(PYTHON_WORKING_DIR)) {
        lock.lockInterruptibly();
        try {
            JobID jobId = getRuntimeContext().getJobId();
            Tuple2<String, Integer> dirAndNums;
            if (workingDirectories.containsKey(jobId)) {
                dirAndNums = workingDirectories.get(jobId);
            } else {
                dirAndNums = Tuple2.of(null, 0);
                workingDirectories.put(jobId, dirAndNums);
            }
            dirAndNums.f1 += 1;
            if (dirAndNums.f0 == null) {
                // get current directory.
                interpreter.exec("import os;cwd = os.getcwd();");
                dirAndNums.f0 = interpreter.get("cwd", String.class);
                String workingDirectory = env.get(PYTHON_WORKING_DIR);
                // set working directory
                interpreter.exec(String.format("import os;os.chdir('%s')", workingDirectory));
            }
        } finally {
            lock.unlock();
        }
    }
    openPythonInterpreter(pythonConfig.getPythonExec(), env, interpreterConfig.getExecType());
}

Example 7 with PythonConfig

use of org.apache.flink.python.PythonConfig in project flink by apache.

the class AbstractPythonFunctionOperator method open.

@Override
public void open() throws Exception {
    try {
        this.pythonConfig = new PythonConfig(config);
        this.jobOptions = config.toMap();
        this.maxBundleSize = pythonConfig.getMaxBundleSize();
        if (this.maxBundleSize <= 0) {
            this.maxBundleSize = PythonOptions.MAX_BUNDLE_SIZE.defaultValue();
            LOG.error("Invalid value for the maximum bundle size. Using default value of " + this.maxBundleSize + '.');
        } else {
            LOG.info("The maximum bundle size is configured to {}.", this.maxBundleSize);
        }
        this.maxBundleTimeMills = pythonConfig.getMaxBundleTimeMills();
        if (this.maxBundleTimeMills <= 0L) {
            this.maxBundleTimeMills = PythonOptions.MAX_BUNDLE_TIME_MILLS.defaultValue();
            LOG.error("Invalid value for the maximum bundle time. Using default value of " + this.maxBundleTimeMills + '.');
        } else {
            LOG.info("The maximum bundle time is configured to {} milliseconds.", this.maxBundleTimeMills);
        }
        this.elementCount = 0;
        this.lastFinishBundleTime = getProcessingTimeService().getCurrentProcessingTime();
        // Schedule timer to check timeout of finish bundle.
        long bundleCheckPeriod = Math.max(this.maxBundleTimeMills, 1);
        this.checkFinishBundleTimer = getProcessingTimeService().scheduleAtFixedRate(// checkpointing lock
        timestamp -> checkInvokeFinishBundleByTime(), bundleCheckPeriod, bundleCheckPeriod);
    } finally {
        super.open();
    }
}