Examples with SubmarineJob org.apache.zeppelin.submarine.job.SubmarineJob used on opensource projects

Example 1 with SubmarineJob

use of org.apache.zeppelin.submarine.job.SubmarineJob in project zeppelin by apache.

the class SubmarineContext method addOrGetSubmarineJob.

public SubmarineJob addOrGetSubmarineJob(Properties properties, InterpreterContext context) {
    SubmarineJob submarineJob = null;
    String noteId = context.getNoteId();
    if (!mapSubmarineJob.containsKey(noteId)) {
        submarineJob = new SubmarineJob(context, properties);
        mapSubmarineJob.put(noteId, submarineJob);
    } else {
        submarineJob = mapSubmarineJob.get(noteId);
    }
    // need update InterpreterContext
    submarineJob.setIntpContext(context);
    return submarineJob;
}

Example 2 with SubmarineJob

use of org.apache.zeppelin.submarine.job.SubmarineJob in project zeppelin by apache.

the class PySubmarineInterpreter method interpret.

@Override
public InterpreterResult interpret(String st, InterpreterContext context) throws InterpreterException {
    setParagraphConfig(context);
    // algorithm & checkpoint path support replaces ${username} with real user name
    String algorithmPath = properties.getProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, "");
    if (algorithmPath.contains(SubmarineConstants.USERNAME_SYMBOL)) {
        algorithmPath = algorithmPath.replace(SubmarineConstants.USERNAME_SYMBOL, userName);
        properties.setProperty(SubmarineConstants.SUBMARINE_ALGORITHM_HDFS_PATH, algorithmPath);
    }
    String checkpointPath = properties.getProperty(SubmarineConstants.TF_CHECKPOINT_PATH, "");
    if (checkpointPath.contains(SubmarineConstants.USERNAME_SYMBOL)) {
        checkpointPath = checkpointPath.replace(SubmarineConstants.USERNAME_SYMBOL, userName);
        properties.setProperty(SubmarineConstants.TF_CHECKPOINT_PATH, checkpointPath);
    }
    if (null == submarineInterpreter) {
        submarineInterpreter = getInterpreterInTheSameSessionByClassName(SubmarineInterpreter.class);
        if (null != submarineInterpreter) {
            submarineInterpreter.setPythonWorkDir(context.getNoteId(), getPythonWorkDir());
        }
    }
    SubmarineJob submarineJob = submarineContext.addOrGetSubmarineJob(this.properties, context);
    if (null != submarineJob && null != submarineJob.getHdfsClient()) {
        try {
            String noteId = context.getNoteId();
            List<ParagraphInfo> paragraphInfos = context.getIntpEventClient().getParagraphList(userName, noteId);
            submarineJob.getHdfsClient().saveParagraphToFiles(noteId, paragraphInfos, getPythonWorkDir().getAbsolutePath(), properties);
        } catch (Exception e) {
            LOGGER.error(e.getMessage(), e);
        }
    }
    return super.interpret(st, context);
}

Example 3 with SubmarineJob

use of org.apache.zeppelin.submarine.job.SubmarineJob in project zeppelin by apache.

the class SubmarineInterpreter method interpret.

@Override
public InterpreterResult interpret(String script, InterpreterContext context) {
    try {
        setParagraphConfig(context);
        // algorithm & checkpoint path support replaces ${username} with real user name
        String algorithmPath = properties.getProperty(SUBMARINE_ALGORITHM_HDFS_PATH, "");
        if (algorithmPath.contains(USERNAME_SYMBOL)) {
            algorithmPath = algorithmPath.replace(USERNAME_SYMBOL, userName);
            properties.setProperty(SUBMARINE_ALGORITHM_HDFS_PATH, algorithmPath);
        }
        String checkpointPath = properties.getProperty(TF_CHECKPOINT_PATH, "");
        if (checkpointPath.contains(USERNAME_SYMBOL)) {
            checkpointPath = checkpointPath.replace(USERNAME_SYMBOL, userName);
            properties.setProperty(TF_CHECKPOINT_PATH, checkpointPath);
        }
        SubmarineJob submarineJob = submarineContext.addOrGetSubmarineJob(properties, context);
        LOGGER.debug("Run shell command '" + script + "'");
        String command = "", operation = "", cleanCheckpoint = "";
        String inputPath = "", chkPntPath = "", psLaunchCmd = "", workerLaunchCmd = "";
        String noteId = context.getNoteId();
        String noteName = context.getNoteName();
        if (script.equalsIgnoreCase(COMMAND_CLEAN)) {
            // Clean Registry Angular Object
            command = CLEAN_RUNTIME_CACHE.getCommand();
        } else {
            operation = SubmarineUtils.getAgulObjValue(context, OPERATION_TYPE);
            if (!StringUtils.isEmpty(operation)) {
                SubmarineUtils.removeAgulObjValue(context, OPERATION_TYPE);
                command = operation;
            } else {
                command = SubmarineUtils.getAgulObjValue(context, COMMAND_TYPE);
            }
        }
        String distributed = this.properties.getProperty(MACHINELEARNING_DISTRIBUTED_ENABLE, "false");
        SubmarineUtils.setAgulObjValue(context, unifyKey(MACHINELEARNING_DISTRIBUTED_ENABLE), distributed);
        inputPath = SubmarineUtils.getAgulObjValue(context, INPUT_PATH);
        cleanCheckpoint = SubmarineUtils.getAgulObjValue(context, CLEAN_CHECKPOINT);
        chkPntPath = submarineJob.getJobDefaultCheckpointPath();
        SubmarineUtils.setAgulObjValue(context, CHECKPOINT_PATH, chkPntPath);
        psLaunchCmd = SubmarineUtils.getAgulObjValue(context, PS_LAUNCH_CMD);
        workerLaunchCmd = SubmarineUtils.getAgulObjValue(context, WORKER_LAUNCH_CMD);
        properties.put(INPUT_PATH, inputPath != null ? inputPath : "");
        properties.put(CHECKPOINT_PATH, chkPntPath != null ? chkPntPath : "");
        properties.put(PS_LAUNCH_CMD, psLaunchCmd != null ? psLaunchCmd : "");
        properties.put(WORKER_LAUNCH_CMD, workerLaunchCmd != null ? workerLaunchCmd : "");
        SubmarineCommand submarineCmd = SubmarineCommand.fromCommand(command);
        switch(submarineCmd) {
            case USAGE:
                submarineJob.showUsage();
                break;
            case JOB_RUN:
                if (StringUtils.equals(cleanCheckpoint, "true")) {
                    submarineJob.cleanJobDefaultCheckpointPath();
                }
                submarineJob.runJob();
                break;
            case JOB_STOP:
                String jobName = SubmarineUtils.getJobName(userName, noteId);
                submarineJob.deleteJob(jobName);
                break;
            case TENSORBOARD_RUN:
                submarineJob.runTensorBoard();
                break;
            case TENSORBOARD_STOP:
                String user = context.getAuthenticationInfo().getUser();
                String tensorboardName = SubmarineUtils.getTensorboardName(user);
                submarineJob.deleteJob(tensorboardName);
                break;
            case OLD_UI:
                createOldGUI(context);
                break;
            case CLEAN_RUNTIME_CACHE:
                submarineJob.cleanRuntimeCache();
                break;
            default:
                submarineJob.onDashboard();
                break;
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        return new InterpreterResult(InterpreterResult.Code.ERROR, e.getMessage());
    }
    return new InterpreterResult(InterpreterResult.Code.SUCCESS);
}

Example 4 with SubmarineJob

use of org.apache.zeppelin.submarine.job.SubmarineJob in project zeppelin by apache.

the class SubmarineInterpreter method cancel.

@Override
public void cancel(InterpreterContext context) {
    SubmarineJob submarineJob = submarineContext.addOrGetSubmarineJob(properties, context);
    String userName = context.getAuthenticationInfo().getUser();
    String noteId = context.getNoteId();
    String jobName = SubmarineUtils.getJobName(userName, noteId);
    submarineJob.deleteJob(jobName);
}

Example 5 with SubmarineJob

use of org.apache.zeppelin.submarine.job.SubmarineJob in project zeppelin by apache.

the class SubmarineInterpreterTest method testJobRun.

@Test
public void testJobRun() throws InterpreterException {
    String script = "JOB_RUN";
    InterpreterContext intpContext = getIntpContext();
    intpContext.getAngularObjectRegistry().add(OPERATION_TYPE, "JOB_RUN", "noteId", "paragraphId");
    InterpreterResult interpreterResult = submarineIntp.interpret(script, intpContext);
    String message = interpreterResult.toJson();
    LOGGER.info(message);
    assertEquals(interpreterResult.code(), InterpreterResult.Code.SUCCESS);
    assertTrue(intpContext.out().size() >= 2);
    String template = intpContext.out().getOutputAt(0).toString();
    assertTrue("Did not generate template!", (template.length() > 500));
    SubmarineJob job = submarineIntp.getSubmarineContext().getSubmarineJob("noteId");
    int loop = 10;
    while (loop-- > 0 && !job.getRunning()) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOGGER.error(e.getMessage(), e);
        }
    }
    LOGGER.info("job.getRunning() = " + job.getRunning());
}