Examples with LineStringBuilder edu.uchc.connjur.wb.LineStringBuilder used on opensource projects

Example 1 with LineStringBuilder

use of edu.uchc.connjur.wb.LineStringBuilder in project vcell by virtualcell.

the class SlurmProxy method generateScript.

/**
 * write bash script for submission
 * @param jobName
 * @param sub_file
 * @param commandSet
 * @param ncpus
 * @param memSize
 * @param postProcessingCommands
 * @return String containing script
 */
String generateScript(String jobName, ExecutableCommand.Container commandSet, int ncpus, double memSize, Collection<PortableCommand> postProcessingCommands) {
    final boolean isParallel = ncpus > 1;
    LineStringBuilder lsb = new LineStringBuilder();
    lsb.write("#!/usr/bin/env bash");
    String partition = "vcell";
    lsb.write("#SBATCH --partition=" + partition);
    lsb.write("#SBATCH -J " + jobName);
    lsb.write("#SBATCH -o " + htcLogDirExternalString + jobName + ".slurm.log");
    lsb.write("#SBATCH -e " + htcLogDirExternalString + jobName + ".slurm.log");
    String nodelist = PropertyLoader.getProperty(PropertyLoader.htcNodeList, null);
    if (nodelist != null && nodelist.trim().length() > 0) {
        lsb.write("#SBATCH --nodelist=" + nodelist);
    }
    lsb.write("echo \"1 date=`date`\"");
    lsb.write("#export MODULEPATH=/isg/shared/modulefiles:/tgcapps/modulefiles");
    lsb.write("#source /usr/share/Modules/init/bash");
    lsb.write("#module load singularity");
    lsb.write("echo \"2 date=`date`\"");
    // sw.append("#$ -l mem=" + (int)(memSize + SLURM_MEM_OVERHEAD_MB) + "mb");
    // int JOB_MEM_OVERHEAD_MB = Integer.parseInt(PropertyLoader.getRequiredProperty(PropertyLoader.jobMemoryOverheadMB));
    // long jobMemoryMB = (JOB_MEM_OVERHEAD_MB+((long)memSize));
    // lsb.write("#$ -j y");
    // sw.append("#$ -l h_vmem="+jobMemoryMB+"m\n");
    // lsb.write("#$ -cwd");
    String primaryDataDirExternal = PropertyLoader.getRequiredProperty(PropertyLoader.primarySimDataDirExternalProperty);
    lsb.write("# determine temp directory to use for storing singularity images");
    lsb.write("#if [ ! -e \"$TMPDIR\" ]; then");
    lsb.write("#  if [ -e /local ]; then");
    lsb.write("#      TMPDIR=/local");
    lsb.write("#      mkdir -p /local/singularityImages");
    lsb.write("#  else");
    lsb.write("#     if [ -e /state/partition1 ]; then");
    lsb.write("         TMPDIR=/state/partition1");
    lsb.write("         mkdir -p /state/partition1/singularityImages");
    lsb.write("         if [[ $? -ne 0 ]]; then");
    lsb.write("             echo \"couldn't create /state/partition1/singularityImages temp directory for singularity\"");
    lsb.write("             exit 1");
    lsb.write("         fi");
    lsb.write("#     fi");
    lsb.write("#  fi");
    lsb.write("#else");
    lsb.write("#   mkdir -p $TMPDIR/singularityImages");
    lsb.write("#fi");
    lsb.write("echo \"using TMPDIR=$TMPDIR\"");
    // 
    // Initialize Singularity
    // 
    lsb.write("echo \"job running on host `hostname -f`\"");
    lsb.newline();
    lsb.write("echo \"id is `id`\"");
    lsb.newline();
    lsb.write("echo \"bash version is `bash --version`\"");
    lsb.write("date");
    lsb.newline();
    lsb.write("echo ENVIRONMENT");
    lsb.write("env");
    lsb.newline();
    String jmshost_external = PropertyLoader.getRequiredProperty(PropertyLoader.jmsHostExternal);
    String jmsport_external = PropertyLoader.getRequiredProperty(PropertyLoader.jmsPortExternal);
    String jmsrestport_external = PropertyLoader.getRequiredProperty(PropertyLoader.jmsRestPortExternal);
    String htclogdir_external = PropertyLoader.getRequiredProperty(PropertyLoader.htcLogDirExternal);
    String jmsuser = PropertyLoader.getRequiredProperty(PropertyLoader.jmsUser);
    String jmspswd = PropertyLoader.getSecretValue(PropertyLoader.jmsPasswordValue, PropertyLoader.jmsPasswordFile);
    String jmsblob_minsize = PropertyLoader.getProperty(PropertyLoader.jmsBlobMessageMinSize, "10000");
    String mongodbhost_external = PropertyLoader.getRequiredProperty(PropertyLoader.mongodbHostExternal);
    String mongodbport_external = PropertyLoader.getRequiredProperty(PropertyLoader.mongodbPortExternal);
    String mongodb_database = PropertyLoader.getRequiredProperty(PropertyLoader.mongodbDatabase);
    String serverid = PropertyLoader.getRequiredProperty(PropertyLoader.vcellServerIDProperty);
    String softwareVersion = PropertyLoader.getRequiredProperty(PropertyLoader.vcellSoftwareVersion);
    String remote_singularity_image = PropertyLoader.getRequiredProperty(PropertyLoader.vcellbatch_singularity_image);
    String docker_image = PropertyLoader.getRequiredProperty(PropertyLoader.vcellbatch_docker_name);
    String Singularity_file = remote_singularity_image + ".Singularity";
    String singularityImageName = new File(remote_singularity_image).getName();
    String[] environmentVars = new String[] { "jmshost_internal=" + jmshost_external, "jmsport_internal=" + jmsport_external, "jmsrestport_internal=" + jmsrestport_external, "jmsuser=" + jmsuser, "jmspswd=" + jmspswd, "jmsblob_minsize=" + jmsblob_minsize, "mongodbhost_internal=" + mongodbhost_external, "mongodbport_internal=" + mongodbport_external, "mongodb_database=" + mongodb_database, "datadir_external=" + primaryDataDirExternal, "htclogdir_external=" + htclogdir_external, "softwareVersion=" + softwareVersion, "serverid=" + serverid };
    lsb.write("container_prefix=");
    lsb.write("if command -v singularity >/dev/null 2>&1; then");
    lsb.write("   # singularity command exists");
    lsb.write("   if [ ! -e " + remote_singularity_image + " ] ; then");
    lsb.write("      echo \"remote singularity image " + remote_singularity_image + " not found, building from docker image\"");
    lsb.write("      echo \"assuming Singularity version 2.4 or later is installed on host system.\"");
    lsb.write("");
    lsb.write("cat <<EOF >" + Singularity_file);
    lsb.write("Bootstrap: docker");
    lsb.write("From: " + docker_image);
    lsb.write("");
    lsb.write("%runscript");
    lsb.write("");
    lsb.write("    exec /vcellscripts/entrypoint.sh \"$@\"");
    lsb.write("");
    lsb.write("%labels");
    lsb.write("");
    lsb.write("AUTHOR jcschaff");
    lsb.write("EOF");
    lsb.write("");
    lsb.write("      singularity build " + remote_singularity_image + " " + Singularity_file);
    lsb.write("      stat=$?");
    lsb.write("      if [ $stat -ne 0 ]; then");
    lsb.write("         echo \"failed to build remote singularity image, returning $stat to Slurm\"");
    lsb.write("         exit $stat");
    lsb.write("      fi");
    lsb.write("   else");
    lsb.write("      echo \"remote singularity image " + remote_singularity_image + " found\"");
    lsb.write("   fi");
    lsb.write("");
    lsb.write("   #");
    lsb.write("   # assure that a local singularity image is present in the temp dir $TMPDIR");
    lsb.write("   #");
    lsb.write("   localSingularityImage=\"${TMPDIR}/singularityImages/" + singularityImageName + "\"");
    lsb.write("   if [ ! -e \"$localSingularityImage\" ]; then");
    lsb.write("       cp -p " + remote_singularity_image + " $localSingularityImage");
    lsb.write("   fi");
    StringBuffer singularityEnvironmentVars = new StringBuffer();
    for (String envVar : environmentVars) {
        singularityEnvironmentVars.append(" --env " + envVar);
    }
    lsb.write("   container_prefix=\"singularity run --bind " + primaryDataDirExternal + ":/simdata --bind " + htclogdir_external + ":/htclogs $localSingularityImage " + singularityEnvironmentVars + " \"");
    lsb.write("else");
    // lsb.write("   if command -v docker >/dev/null 2>&1; then");
    StringBuffer dockerEnvironmentVars = new StringBuffer();
    for (String envVar : environmentVars) {
        dockerEnvironmentVars.append(" -e " + envVar);
    }
    lsb.write("       container_prefix=\"docker run --rm -v " + primaryDataDirExternal + ":/simdata -v " + htclogdir_external + ":/htclogs " + dockerEnvironmentVars + " " + docker_image + " \"");
    // lsb.write("   fi");
    lsb.write("fi");
    lsb.write("echo \"container_prefix is '${container_prefix}'\"");
    lsb.write("echo \"3 date=`date`\"");
    lsb.newline();
    if (isParallel) {
        // #SBATCH
        // lsb.append("#$ -pe mpich ");
        // lsb.append(ncpus);
        // lsb.newline();
        lsb.append("#SBATCH -n " + ncpus);
        lsb.newline();
        lsb.append("#$ -v LD_LIBRARY_PATH=");
        lsb.append(MPI_HOME_EXTERNAL + "/lib");
        lsb.write(":" + primaryDataDirExternal);
    }
    lsb.newline();
    final boolean hasExitProcessor = commandSet.hasExitCodeCommand();
    // lsb.write("run_in_container=\"singularity /path/to/data:/simdata /path/to/image/vcell-batch.img);
    if (hasExitProcessor) {
        ExecutableCommand exitCmd = commandSet.getExitCodeCommand();
        exitCmd.stripPathFromCommand();
        lsb.write("callExitProcessor( ) {");
        lsb.write("echo \"4 date=`date`\"");
        lsb.append("\techo exitCommand = ");
        lsb.write("$container_prefix" + exitCmd.getJoinedCommands("$1"));
        lsb.append('\t');
        lsb.write("$container_prefix" + exitCmd.getJoinedCommands());
        lsb.write("}");
        lsb.write("echo");
    }
    for (ExecutableCommand ec : commandSet.getExecCommands()) {
        lsb.write("echo");
        ec.stripPathFromCommand();
        // 
        // The first token in the command list is always the name of the executable.
        // if an executable with that name exists in the nativesolvers directory, then use that instead.
        // 
        String cmd = ec.getJoinedCommands();
        String exeName = ec.getCommands().get(0);
        File nativeSolverDir = new File(PropertyLoader.getRequiredProperty(PropertyLoader.nativeSolverDir_External));
        File nativeExe = new File(nativeSolverDir, exeName);
        lsb.write("echo \"testing existance of native exe '" + nativeExe.getAbsolutePath() + "' which overrides container invocations\"");
        lsb.write("nativeExe=" + nativeExe.getAbsolutePath());
        lsb.write("if [ -e \"${nativeExe}\" ]; then");
        lsb.write("   cmd_prefix=\"" + nativeSolverDir.getAbsolutePath() + "/" + "\"");
        lsb.write("else");
        lsb.write("   cmd_prefix=\"$container_prefix\"");
        lsb.write("fi");
        lsb.write("echo \"cmd_prefix is '${cmd_prefix}'\"");
        lsb.write("echo \"5 date=`date`\"");
        if (ec.isParallel()) {
            if (isParallel) {
                cmd = buildExeCommand(ncpus, cmd);
            } else {
                throw new UnsupportedOperationException("parallel command " + ec.getJoinedCommands() + " called in non-parallel submit");
            }
        }
        lsb.append("echo command = ");
        lsb.write("${cmd_prefix}" + cmd);
        lsb.write("(");
        if (ec.getLdLibraryPath() != null) {
            lsb.write("    export LD_LIBRARY_PATH=" + ec.getLdLibraryPath().path + ":$LD_LIBRARY_PATH");
        }
        lsb.write("    " + "${cmd_prefix}" + cmd);
        lsb.write(")");
        lsb.write("stat=$?");
        lsb.append("echo ");
        lsb.append("${cmd_prefix}" + cmd);
        lsb.write("returned $stat");
        lsb.write("echo \"6 date=`date`\"");
        lsb.write("if [ $stat -ne 0 ]; then");
        if (hasExitProcessor) {
            lsb.write("\tcallExitProcessor $stat");
        }
        lsb.write("\techo returning $stat to Slurm");
        lsb.write("\texit $stat");
        lsb.write("fi");
    }
    Objects.requireNonNull(postProcessingCommands);
    PortableCommandWrapper.insertCommands(lsb.sb, postProcessingCommands);
    lsb.newline();
    if (hasExitProcessor) {
        lsb.write("callExitProcessor 0");
    }
    lsb.write("echo \"7 date=`date`\"");
    lsb.newline();
    return lsb.sb.toString();
}