Examples with ExecutableCommand cbit.vcell.solvers.ExecutableCommand used on opensource projects

Example 1 with ExecutableCommand

use of cbit.vcell.solvers.ExecutableCommand in project vcell by virtualcell.

the class SlurmProxy method generateScript.

/**
 * write bash script for submission
 * @param jobName
 * @param sub_file
 * @param commandSet
 * @param ncpus
 * @param memSize
 * @param postProcessingCommands
 * @return String containing script
 */
String generateScript(String jobName, ExecutableCommand.Container commandSet, int ncpus, double memSize, Collection<PortableCommand> postProcessingCommands) {
    final boolean isParallel = ncpus > 1;
    LineStringBuilder lsb = new LineStringBuilder();
    lsb.write("#!/usr/bin/env bash");
    String partition = "vcell";
    lsb.write("#SBATCH --partition=" + partition);
    lsb.write("#SBATCH -J " + jobName);
    lsb.write("#SBATCH -o " + htcLogDirExternalString + jobName + ".slurm.log");
    lsb.write("#SBATCH -e " + htcLogDirExternalString + jobName + ".slurm.log");
    String nodelist = PropertyLoader.getProperty(PropertyLoader.htcNodeList, null);
    if (nodelist != null && nodelist.trim().length() > 0) {
        lsb.write("#SBATCH --nodelist=" + nodelist);
    }
    lsb.write("echo \"1 date=`date`\"");
    lsb.write("#export MODULEPATH=/isg/shared/modulefiles:/tgcapps/modulefiles");
    lsb.write("#source /usr/share/Modules/init/bash");
    lsb.write("#module load singularity");
    lsb.write("echo \"2 date=`date`\"");
    // sw.append("#$ -l mem=" + (int)(memSize + SLURM_MEM_OVERHEAD_MB) + "mb");
    // int JOB_MEM_OVERHEAD_MB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jobMemoryOverheadMB));
    // long jobMemoryMB = (JOB_MEM_OVERHEAD_MB+((long)memSize));
    // lsb.write("#$ -j y");
    // sw.append("#$ -l h_vmem="+jobMemoryMB+"m\n");
    // lsb.write("#$ -cwd");
    String primaryDataDirExternal = PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirExternalProperty);
    lsb.write("# determine temp directory to use for storing singularity images");
    lsb.write("#if [ ! -e \"$TMPDIR\" ]; then");
    lsb.write("#  if [ -e /local ]; then");
    lsb.write("#      TMPDIR=/local");
    lsb.write("#      mkdir -p /local/singularityImages");
    lsb.write("#  else");
    lsb.write("#     if [ -e /state/partition1 ]; then");
    lsb.write("         TMPDIR=/state/partition1");
    lsb.write("         mkdir -p /state/partition1/singularityImages");
    lsb.write("         if [[ $? -ne 0 ]]; then");
    lsb.write("             echo \"couldn't create /state/partition1/singularityImages temp directory for singularity\"");
    lsb.write("             exit 1");
    lsb.write("         fi");
    lsb.write("#     fi");
    lsb.write("#  fi");
    lsb.write("#else");
    lsb.write("#   mkdir -p $TMPDIR/singularityImages");
    lsb.write("#fi");
    lsb.write("echo \"using TMPDIR=$TMPDIR\"");
    // 
    // Initialize Singularity
    // 
    lsb.write("echo \"job running on host `hostname -f`\"");
    lsb.newline();
    lsb.write("echo \"id is `id`\"");
    lsb.newline();
    lsb.write("echo \"bash version is `bash --version`\"");
    lsb.write("date");
    lsb.newline();
    lsb.write("echo ENVIRONMENT");
    lsb.write("env");
    lsb.newline();
    String jmshost_external = PropertyLoader.getRequiredProperty(PropertyLoader.jmsHostExternal);
    String jmsport_external = PropertyLoader.getRequiredProperty(PropertyLoader.jmsPortExternal);
    String jmsrestport_external = PropertyLoader.getRequiredProperty(PropertyLoader.jmsRestPortExternal);
    String htclogdir_external = PropertyLoader.getRequiredProperty(PropertyLoader.htcLogDirExternal);
    String jmsuser = PropertyLoader.getRequiredProperty(PropertyLoader.jmsUser);
    String jmspswd = PropertyLoader.getSecretValue(PropertyLoader.jmsPasswordValue, PropertyLoader.jmsPasswordFile);
    String jmsblob_minsize = PropertyLoader.getProperty(PropertyLoader.jmsBlobMessageMinSize, "10000");
    String mongodbhost_external = PropertyLoader.getRequiredProperty(PropertyLoader.mongodbHostExternal);
    String mongodbport_external = PropertyLoader.getRequiredProperty(PropertyLoader.mongodbPortExternal);
    String mongodb_database = PropertyLoader.getRequiredProperty(PropertyLoader.mongodbDatabase);
    String serverid = PropertyLoader.getRequiredProperty(PropertyLoader.vcellServerIDProperty);
    String softwareVersion = PropertyLoader.getRequiredProperty(PropertyLoader.vcellSoftwareVersion);
    String remote_singularity_image = PropertyLoader.getRequiredProperty(PropertyLoader.vcellbatch_singularity_image);
    String docker_image = PropertyLoader.getRequiredProperty(PropertyLoader.vcellbatch_docker_name);
    String Singularity_file = remote_singularity_image + ".Singularity";
    String singularityImageName = new File(remote_singularity_image).getName();
    String[] environmentVars = new String[] { "jmshost_internal=" + jmshost_external, "jmsport_internal=" + jmsport_external, "jmsrestport_internal=" + jmsrestport_external, "jmsuser=" + jmsuser, "jmspswd=" + jmspswd, "jmsblob_minsize=" + jmsblob_minsize, "mongodbhost_internal=" + mongodbhost_external, "mongodbport_internal=" + mongodbport_external, "mongodb_database=" + mongodb_database, "datadir_external=" + primaryDataDirExternal, "htclogdir_external=" + htclogdir_external, "softwareVersion=" + softwareVersion, "serverid=" + serverid };
    lsb.write("container_prefix=");
    lsb.write("if command -v singularity >/dev/null 2>&1; then");
    lsb.write("   # singularity command exists");
    lsb.write("   if [ ! -e " + remote_singularity_image + " ] ; then");
    lsb.write("      echo \"remote singularity image " + remote_singularity_image + " not found, building from docker image\"");
    lsb.write("      echo \"assuming Singularity version 2.4 or later is installed on host system.\"");
    lsb.write("");
    lsb.write("cat <<EOF >" + Singularity_file);
    lsb.write("Bootstrap: docker");
    lsb.write("From: " + docker_image);
    lsb.write("");
    lsb.write("%runscript");
    lsb.write("");
    lsb.write("    exec /vcellscripts/entrypoint.sh \"$@\"");
    lsb.write("");
    lsb.write("%labels");
    lsb.write("");
    lsb.write("AUTHOR jcschaff");
    lsb.write("EOF");
    lsb.write("");
    lsb.write("      singularity build " + remote_singularity_image + " " + Singularity_file);
    lsb.write("      stat=$?");
    lsb.write("      if [ $stat -ne 0 ]; then");
    lsb.write("         echo \"failed to build remote singularity image, returning $stat to Slurm\"");
    lsb.write("         exit $stat");
    lsb.write("      fi");
    lsb.write("   else");
    lsb.write("      echo \"remote singularity image " + remote_singularity_image + " found\"");
    lsb.write("   fi");
    lsb.write("");
    lsb.write("   #");
    lsb.write("   # assure that a local singularity image is present in the temp dir $TMPDIR");
    lsb.write("   #");
    lsb.write("   localSingularityImage=\"${TMPDIR}/singularityImages/" + singularityImageName + "\"");
    lsb.write("   if [ ! -e \"$localSingularityImage\" ]; then");
    lsb.write("       cp -p " + remote_singularity_image + " $localSingularityImage");
    lsb.write("   fi");
    StringBuffer singularityEnvironmentVars = new StringBuffer();
    for (String envVar : environmentVars) {
        singularityEnvironmentVars.append(" --env " + envVar);
    }
    lsb.write("   container_prefix=\"singularity run --bind " + primaryDataDirExternal + ":/simdata --bind " + htclogdir_external + ":/htclogs $localSingularityImage " + singularityEnvironmentVars + " \"");
    lsb.write("else");
    // lsb.write("   if command -v docker >/dev/null 2>&1; then");
    StringBuffer dockerEnvironmentVars = new StringBuffer();
    for (String envVar : environmentVars) {
        dockerEnvironmentVars.append(" -e " + envVar);
    }
    lsb.write("       container_prefix=\"docker run --rm -v " + primaryDataDirExternal + ":/simdata -v " + htclogdir_external + ":/htclogs " + dockerEnvironmentVars + " " + docker_image + " \"");
    // lsb.write("   fi");
    lsb.write("fi");
    lsb.write("echo \"container_prefix is '${container_prefix}'\"");
    lsb.write("echo \"3 date=`date`\"");
    lsb.newline();
    if (isParallel) {
        // #SBATCH
        // lsb.append("#$ -pe mpich ");
        // lsb.append(ncpus);
        // lsb.newline();
        lsb.append("#SBATCH -n " + ncpus);
        lsb.newline();
        lsb.append("#$ -v LD_LIBRARY_PATH=");
        lsb.append(MPI_HOME_EXTERNAL + "/lib");
        lsb.write(":" + primaryDataDirExternal);
    }
    lsb.newline();
    final boolean hasExitProcessor = commandSet.hasExitCodeCommand();
    // lsb.write("run_in_container=\"singularity /path/to/data:/simdata /path/to/image/vcell-batch.img);
    if (hasExitProcessor) {
        ExecutableCommand exitCmd = commandSet.getExitCodeCommand();
        exitCmd.stripPathFromCommand();
        lsb.write("callExitProcessor( ) {");
        lsb.write("echo \"4 date=`date`\"");
        lsb.append("\techo exitCommand = ");
        lsb.write("$container_prefix" + exitCmd.getJoinedCommands("$1"));
        lsb.append('\t');
        lsb.write("$container_prefix" + exitCmd.getJoinedCommands());
        lsb.write("}");
        lsb.write("echo");
    }
    for (ExecutableCommand ec : commandSet.getExecCommands()) {
        lsb.write("echo");
        ec.stripPathFromCommand();
        // 
        // The first token in the command list is always the name of the executable.
        // if an executable with that name exists in the nativesolvers directory, then use that instead.
        // 
        String cmd = ec.getJoinedCommands();
        String exeName = ec.getCommands().get(0);
        File nativeSolverDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.nativeSolverDir_External));
        File nativeExe = new File(nativeSolverDir, exeName);
        lsb.write("echo \"testing existance of native exe '" + nativeExe.getAbsolutePath() + "' which overrides container invocations\"");
        lsb.write("nativeExe=" + nativeExe.getAbsolutePath());
        lsb.write("if [ -e \"${nativeExe}\" ]; then");
        lsb.write("   cmd_prefix=\"" + nativeSolverDir.getAbsolutePath() + "/" + "\"");
        lsb.write("else");
        lsb.write("   cmd_prefix=\"$container_prefix\"");
        lsb.write("fi");
        lsb.write("echo \"cmd_prefix is '${cmd_prefix}'\"");
        lsb.write("echo \"5 date=`date`\"");
        if (ec.isParallel()) {
            if (isParallel) {
                cmd = buildExeCommand(ncpus, cmd);
            } else {
                throw new UnsupportedOperationException("parallel command " + ec.getJoinedCommands() + " called in non-parallel submit");
            }
        }
        lsb.append("echo command = ");
        lsb.write("${cmd_prefix}" + cmd);
        lsb.write("(");
        if (ec.getLdLibraryPath() != null) {
            lsb.write("    export LD_LIBRARY_PATH=" + ec.getLdLibraryPath().path + ":$LD_LIBRARY_PATH");
        }
        lsb.write("    " + "${cmd_prefix}" + cmd);
        lsb.write(")");
        lsb.write("stat=$?");
        lsb.append("echo ");
        lsb.append("${cmd_prefix}" + cmd);
        lsb.write("returned $stat");
        lsb.write("echo \"6 date=`date`\"");
        lsb.write("if [ $stat -ne 0 ]; then");
        if (hasExitProcessor) {
            lsb.write("\tcallExitProcessor $stat");
        }
        lsb.write("\techo returning $stat to Slurm");
        lsb.write("\texit $stat");
        lsb.write("fi");
    }
    Objects.requireNonNull(postProcessingCommands);
    PortableCommandWrapper.insertCommands(lsb.sb, postProcessingCommands);
    lsb.newline();
    if (hasExitProcessor) {
        lsb.write("callExitProcessor 0");
    }
    lsb.write("echo \"7 date=`date`\"");
    lsb.newline();
    return lsb.sb.toString();
}

Example 2 with ExecutableCommand

use of cbit.vcell.solvers.ExecutableCommand in project vcell by virtualcell.

the class HtcSimulationWorker method submit2PBS.

private HtcJobID submit2PBS(SimulationTask simTask, HtcProxy clonedHtcProxy, PostProcessingChores chores) throws XmlParseException, IOException, SolverException, ExecutableException {
    HtcJobID jobid = null;
    String htcLogDirExternalString = new File(PropertyLoader.getRequiredProperty(PropertyLoader.htcLogDirExternal)).getAbsolutePath();
    if (!htcLogDirExternalString.endsWith("/")) {
        htcLogDirExternalString = htcLogDirExternalString + "/";
    }
    String htcLogDirInternalString = new File(PropertyLoader.getRequiredProperty(PropertyLoader.htcLogDirInternal)).getAbsolutePath();
    if (!htcLogDirInternalString.endsWith("/")) {
        htcLogDirInternalString = htcLogDirInternalString + "/";
    }
    // "S_" + simTask.getSimKey() + "_" + simTask.getSimulationJob().getJobIndex()+ "_" + simTask.getTaskID();
    String jobname = HtcProxy.createHtcSimJobName(new HtcProxy.SimTaskInfo(simTask.getSimKey(), simTask.getSimulationJob().getJobIndex(), simTask.getTaskID()));
    String subFileExternal = htcLogDirExternalString + jobname + clonedHtcProxy.getSubmissionFileExtension();
    String subFileInternal = htcLogDirInternalString + jobname + clonedHtcProxy.getSubmissionFileExtension();
    File parallelDirInternal = new File(chores.runDirectoryInternal);
    File parallelDirExternal = new File(chores.runDirectoryExternal);
    File primaryUserDirInternal = new File(chores.finalDataDirectoryInternal);
    File primaryUserDirExternal = new File(chores.finalDataDirectoryExternal);
    Solver realSolver = (AbstractSolver) SolverFactory.createSolver(primaryUserDirInternal, parallelDirInternal, simTask, true);
    realSolver.setUnixMode();
    String simTaskXmlText = XmlHelper.simTaskToXML(simTask);
    String simTaskFilePathInternal = ResourceUtil.forceUnixPath(new File(primaryUserDirInternal, simTask.getSimulationJobID() + "_" + simTask.getTaskID() + ".simtask.xml").toString());
    String simTaskFilePathExternal = ResourceUtil.forceUnixPath(new File(primaryUserDirExternal, simTask.getSimulationJobID() + "_" + simTask.getTaskID() + ".simtask.xml").toString());
    if (!primaryUserDirInternal.exists()) {
        FileUtils.forceMkdir(primaryUserDirInternal);
        // 
        // directory create from container (possibly) as root, make this user directory accessible from user "vcell"
        // 
        primaryUserDirInternal.setWritable(true, false);
        primaryUserDirInternal.setExecutable(true, false);
        primaryUserDirInternal.setReadable(true, false);
    }
    XmlUtil.writeXMLStringToFile(simTaskXmlText, simTaskFilePathInternal, true);
    final String SOLVER_EXIT_CODE_REPLACE_STRING = "SOLVER_EXIT_CODE_REPLACE_STRING";
    KeyValue simKey = simTask.getSimKey();
    User simOwner = simTask.getSimulation().getVersion().getOwner();
    final int jobId = simTask.getSimulationJob().getJobIndex();
    ExecutableCommand.Container commandContainer = new ExecutableCommand.Container();
    // the post processor command itself is neither messaging nor parallel; it's independent of the previous solver call
    ExecutableCommand postprocessorCmd = new ExecutableCommand(null, false, false, PropertyLoader.getRequiredProperty(PropertyLoader.simulationPostprocessor), simKey.toString(), simOwner.getName(), simOwner.getID().toString(), Integer.toString(jobId), Integer.toString(simTask.getTaskID()), SOLVER_EXIT_CODE_REPLACE_STRING, subFileExternal);
    postprocessorCmd.setExitCodeToken(SOLVER_EXIT_CODE_REPLACE_STRING);
    commandContainer.add(postprocessorCmd);
    // CBN?
    int ncpus = simTask.getSimulation().getSolverTaskDescription().getNumProcessors();
    Collection<PortableCommand> postProcessingCommands = new ArrayList<PortableCommand>();
    if (realSolver instanceof AbstractCompiledSolver) {
        AbstractCompiledSolver compiledSolver = (AbstractCompiledSolver) realSolver;
        List<String> args = new ArrayList<>(4);
        args.add(PropertyLoader.getRequiredProperty(PropertyLoader.simulationPreprocessor));
        args.add(simTaskFilePathExternal);
        args.add(primaryUserDirExternal.getAbsolutePath());
        if (chores.isParallel()) {
            args.add(chores.runDirectoryExternal);
        }
        // compiled solver ...used to be only single executable, now we pass 2 commands to PBSUtils.submitJob that invokes SolverPreprocessor.main() and then the native executable
        // the pre-processor command itself is neither messaging nor parallel; it's independent of the subsequent solver call
        ExecutableCommand preprocessorCmd = new ExecutableCommand(null, false, false, args);
        commandContainer.add(preprocessorCmd);
        for (ExecutableCommand ec : compiledSolver.getCommands()) {
            if (ec.isMessaging()) {
                ec.addArgument("-tid");
                ec.addArgument(simTask.getTaskID());
            }
            commandContainer.add(ec);
        }
        if (chores.isCopyNeeded()) {
            String logName = chores.finalDataDirectoryInternal + '/' + SimulationData.createCanonicalSimLogFileName(simKey, jobId, false);
            CopySimFiles csf = new CopySimFiles(simTask.getSimulationJobID(), chores.runDirectoryInternal, chores.finalDataDirectoryInternal, logName);
            postProcessingCommands.add(csf);
        }
        if (chores.isVtkUser()) {
            VtkMeshGenerator vmg = new VtkMeshGenerator(simOwner, simKey, jobId);
            postProcessingCommands.add(vmg);
        }
    } else {
        ExecutableCommand ec = new ExecutableCommand(null, false, false, PropertyLoader.getRequiredProperty(PropertyLoader.javaSimulationExecutable), simTaskFilePathExternal, ResourceUtil.forceUnixPath(parallelDirExternal.getAbsolutePath()));
        commandContainer.add(ec);
    }
    jobid = clonedHtcProxy.submitJob(jobname, subFileInternal, subFileExternal, commandContainer, ncpus, simTask.getEstimatedMemorySizeMB(), postProcessingCommands);
    if (jobid == null) {
        throw new RuntimeException("Failed. (error message: submitting to job scheduler failed).");
    }
    return jobid;
}