Examples with Template com.hashicorp.nomad.apimodel.Template used on opensource projects

Example 1 with Template

use of com.hashicorp.nomad.apimodel.Template in project incubator-heron by apache.

the class NomadScheduler method getTaskSpecRawDriver.

/**
 * Get the task spec for using raw_exec driver in Nomad
 * In raw exec mode, Heron will be run directly on the machine
 */
Task getTaskSpecRawDriver(Task task, String taskName, int containerIndex) {
    String executorBinary = Context.executorBinary(this.clusterConfig);
    // get arguments for heron executor command
    String[] executorArgs = SchedulerUtils.executorCommandArgs(this.clusterConfig, this.runtimeConfig, NomadConstants.EXECUTOR_PORTS, String.valueOf(containerIndex));
    // get complete heron executor command
    String executorCmd = executorBinary + " " + String.join(" ", executorArgs);
    // get heron_downloader command for downloading topology package
    String topologyDownloadCmd = getFetchCommand(this.clusterConfig, this.runtimeConfig);
    // read nomad heron executor start up script from file
    String heronNomadScript = getHeronNomadScript(this.localConfig);
    task.setName(taskName);
    // use raw_exec driver
    task.setDriver(NomadConstants.NomadDriver.RAW_EXEC.getName());
    // call nomad heron start up script
    task.addConfig(NomadConstants.NOMAD_TASK_COMMAND, NomadConstants.SHELL_CMD);
    String[] args = { NomadConstants.NOMAD_HERON_SCRIPT_NAME };
    task.addConfig(NomadConstants.NOMAD_TASK_COMMAND_ARGS, args);
    Template template = new Template();
    template.setEmbeddedTmpl(heronNomadScript);
    template.setDestPath(NomadConstants.NOMAD_HERON_SCRIPT_NAME);
    task.addTemplates(template);
    Resources resourceReqs = new Resources();
    // configure nomad to allocate dynamic ports
    Port[] ports = new Port[NomadConstants.EXECUTOR_PORTS.size()];
    int i = 0;
    for (SchedulerUtils.ExecutorPort port : NomadConstants.EXECUTOR_PORTS.keySet()) {
        ports[i] = new Port().setLabel(port.getName().replace("-", "_"));
        i++;
    }
    // set enviroment variables used int the heron nomad start up script
    Map<String, String> envVars = new HashMap<>();
    envVars.put(NomadConstants.HERON_NOMAD_WORKING_DIR, NomadContext.workingDirectory(this.localConfig) + "/container-" + String.valueOf(containerIndex));
    if (NomadContext.useCorePackageUri(this.localConfig)) {
        envVars.put(NomadConstants.HERON_USE_CORE_PACKAGE_URI, "true");
        envVars.put(NomadConstants.HERON_CORE_PACKAGE_URI, NomadContext.corePackageUri(this.localConfig));
    } else {
        envVars.put(NomadConstants.HERON_USE_CORE_PACKAGE_URI, "false");
        envVars.put(NomadConstants.HERON_CORE_PACKAGE_DIR, NomadContext.corePackageDirectory(this.localConfig));
    }
    envVars.put(NomadConstants.HERON_TOPOLOGY_DOWNLOAD_CMD, topologyDownloadCmd);
    envVars.put(NomadConstants.HERON_EXECUTOR_CMD, executorCmd);
    task.setEnv(envVars);
    return task;
}

Example 2 with Template

use of com.hashicorp.nomad.apimodel.Template in project heron by twitter.

the class NomadScheduler method getTaskSpecRawDriver.

/**
 * Get the task spec for using raw_exec driver in Nomad
 * In raw exec mode, Heron will be run directly on the machine
 */
Task getTaskSpecRawDriver(Task task, String taskName, int containerIndex) {
    String executorBinary = Context.executorBinary(this.clusterConfig);
    // get arguments for heron executor command
    String[] executorArgs = SchedulerUtils.executorCommandArgs(this.clusterConfig, this.runtimeConfig, NomadConstants.EXECUTOR_PORTS, String.valueOf(containerIndex));
    // get complete heron executor command
    String executorCmd = executorBinary + " " + String.join(" ", executorArgs);
    // get heron_downloader command for downloading topology package
    String topologyDownloadCmd = getFetchCommand(this.localConfig, this.clusterConfig, this.runtimeConfig);
    // read nomad heron executor start up script from file
    String heronNomadScript = getHeronNomadScript(this.localConfig);
    task.setName(taskName);
    // use raw_exec driver
    task.setDriver(NomadConstants.NomadDriver.RAW_EXEC.getName());
    // call nomad heron start up script
    task.addConfig(NomadConstants.NOMAD_TASK_COMMAND, NomadConstants.SHELL_CMD);
    String[] args = { NomadConstants.NOMAD_HERON_SCRIPT_NAME };
    task.addConfig(NomadConstants.NOMAD_TASK_COMMAND_ARGS, args);
    Template template = new Template();
    template.setEmbeddedTmpl(heronNomadScript);
    template.setDestPath(NomadConstants.NOMAD_HERON_SCRIPT_NAME);
    task.addTemplates(template);
    // configure nomad to allocate dynamic ports
    Port[] ports = new Port[NomadConstants.EXECUTOR_PORTS.size()];
    int i = 0;
    for (SchedulerUtils.ExecutorPort port : NomadConstants.EXECUTOR_PORTS.keySet()) {
        ports[i] = new Port().setLabel(port.getName().replace("-", "_"));
        i++;
    }
    // set enviroment variables used int the heron nomad start up script
    Map<String, String> envVars = new HashMap<>();
    envVars.put(NomadConstants.HERON_NOMAD_WORKING_DIR, NomadContext.workingDirectory(this.localConfig) + "/container-" + String.valueOf(containerIndex));
    if (NomadContext.useCorePackageUri(this.localConfig)) {
        envVars.put(NomadConstants.HERON_USE_CORE_PACKAGE_URI, "true");
        envVars.put(NomadConstants.HERON_CORE_PACKAGE_URI, NomadContext.corePackageUri(this.localConfig));
    } else {
        envVars.put(NomadConstants.HERON_USE_CORE_PACKAGE_URI, "false");
        envVars.put(NomadConstants.HERON_CORE_PACKAGE_DIR, NomadContext.corePackageDirectory(this.localConfig));
    }
    envVars.put(NomadConstants.HERON_TOPOLOGY_DOWNLOAD_CMD, topologyDownloadCmd);
    envVars.put(NomadConstants.HERON_EXECUTOR_CMD, executorCmd);
    task.setEnv(envVars);
    return task;
}

Example 3 with Template

use of com.hashicorp.nomad.apimodel.Template in project twister2 by DSC-SPIDAL.

the class NomadController method getShellDriver.

private Task getShellDriver(JobAPI.Job job) {
    String taskName = job.getJobId();
    Task task = new Task();
    // get the job working directory
    String workingDirectory = NomadContext.workingDirectory(config);
    String jobWorkingDirectory = Paths.get(workingDirectory, job.getJobId()).toString();
    String configDirectoryName = Paths.get(workingDirectory, job.getJobId(), SchedulerContext.clusterType(config)).toString();
    String corePackageFile = SchedulerContext.temporaryPackagesPath(config) + "/" + SchedulerContext.corePackageFileName(config);
    String jobPackageFile = SchedulerContext.temporaryPackagesPath(config) + "/" + SchedulerContext.jobPackageFileName(config);
    String nomadScriptContent = getNomadScriptContent(config, configDirectoryName);
    task.setName(taskName);
    task.setDriver("raw_exec");
    task.addConfig(NomadContext.NOMAD_TASK_COMMAND, NomadContext.SHELL_CMD);
    String[] args = workerProcessCommand(workingDirectory, job);
    task.addConfig(NomadContext.NOMAD_TASK_COMMAND_ARGS, args);
    Template template = new Template();
    template.setEmbeddedTmpl(nomadScriptContent);
    template.setDestPath(NomadContext.NOMAD_SCRIPT_NAME);
    task.addTemplates(template);
    Resources resourceReqs = new Resources();
    String portNamesConfig = NomadContext.networkPortNames(config);
    String[] portNames = portNamesConfig.split(",");
    // configure nomad to allocate dynamic ports
    Port[] ports = new Port[portNames.length];
    int i = 0;
    for (String p : portNames) {
        ports[i] = new Port().setLabel(p);
        i++;
    }
    NetworkResource networkResource = new NetworkResource();
    networkResource.addDynamicPorts(ports);
    resourceReqs.addNetworks(networkResource);
    JobAPI.ComputeResource computeResource = JobUtils.getComputeResource(job, 0);
    if (computeResource == null) {
        LOG.log(Level.SEVERE, "Error: there is no compute resource");
        return null;
    }
    int cpu = (int) computeResource.getCpu();
    int disk = (int) computeResource.getDiskGigaBytes();
    int memory = computeResource.getRamMegaBytes();
    resourceReqs.setCpu(cpu * 200);
    resourceReqs.setMemoryMb(memory);
    resourceReqs.setDiskMb(disk * 1024);
    LOG.log(Level.INFO, "Compute resources are " + cpu + " " + memory + " " + disk);
    Map<String, String> envVars = new HashMap<>();
    envVars.put(NomadContext.WORKING_DIRECTORY_ENV, NomadContext.workingDirectory(config));
    if (!NomadContext.sharedFileSystem(config)) {
        envVars.put(NomadContext.DOWNLOAD_PACKAGE_ENV, "false");
    } else {
        envVars.put(NomadContext.DOWNLOAD_PACKAGE_ENV, "true");
    }
    // we are putting the core packages as env variable
    envVars.put(NomadContext.CORE_PACKAGE_ENV, corePackageFile);
    envVars.put(NomadContext.JOB_PACKAGE_ENV, jobPackageFile);
    task.setEnv(envVars);
    task.setResources(resourceReqs);
    return task;
}