Examples with Task com.hashicorp.nomad.apimodel.Task used on opensource projects

Example 6 with Task

use of com.hashicorp.nomad.apimodel.Task in project incubator-heron by apache.

the class NomadSchedulerTest method testGetTaskRawExec.

@SuppressWarnings("unchecked")
@Test
public void testGetTaskRawExec() {
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    PowerMockito.mockStatic(SchedulerUtils.class);
    Resource resource = new Resource(CPU_RESOURCE, MEMORY_RESOURCE, DISK_RESOURCE);
    PowerMockito.when(SchedulerUtils.executorCommandArgs(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.anyString())).thenReturn(EXECUTOR_CMD_ARGS);
    PowerMockito.mockStatic(NomadScheduler.class);
    PowerMockito.when(NomadScheduler.getFetchCommand(Mockito.any(), Mockito.any())).thenReturn(TOPOLOGY_DOWNLOAD_CMD);
    PowerMockito.when(NomadScheduler.getHeronNomadScript(this.mockConfig)).thenReturn(HERON_NOMAD_SCRIPT);
    PowerMockito.when(NomadScheduler.longToInt(MEMORY_RESOURCE.asMegabytes())).thenReturn((int) MEMORY_RESOURCE.asMegabytes());
    PowerMockito.when(NomadScheduler.longToInt(DISK_RESOURCE.asMegabytes())).thenReturn((int) DISK_RESOURCE.asMegabytes());
    scheduler.initialize(this.mockConfig, this.mockRuntime);
    Task task = scheduler.getTask(TASK_NAME, CONTAINER_INDEX, resource);
    LOG.info("task: " + task);
    Assert.assertEquals(TASK_NAME, task.getName());
    Assert.assertEquals(NomadConstants.NomadDriver.RAW_EXEC.getName(), task.getDriver());
    Assert.assertTrue(task.getConfig().containsKey(NomadConstants.NOMAD_TASK_COMMAND));
    Assert.assertEquals(NomadConstants.SHELL_CMD, task.getConfig().get(NomadConstants.NOMAD_TASK_COMMAND));
    Assert.assertTrue(task.getConfig().containsKey(NomadConstants.NOMAD_TASK_COMMAND_ARGS));
    Assert.assertArrayEquals(Arrays.asList(NomadConstants.NOMAD_HERON_SCRIPT_NAME).toArray(), (String[]) task.getConfig().get(NomadConstants.NOMAD_TASK_COMMAND_ARGS));
    Assert.assertEquals(1, task.getTemplates().size());
    Assert.assertEquals(HERON_NOMAD_SCRIPT, task.getTemplates().get(0).getEmbeddedTmpl());
    Assert.assertEquals(NomadConstants.NOMAD_HERON_SCRIPT_NAME, task.getTemplates().get(0).getDestPath());
    Assert.assertEquals((int) CPU_RESOURCE * HERON_NOMAD_CORE_FREQ_MAPPING, task.getResources().getCpu().intValue());
    Assert.assertEquals((int) MEMORY_RESOURCE.asMegabytes(), task.getResources().getMemoryMb().intValue());
    Assert.assertEquals((int) DISK_RESOURCE.asMegabytes(), task.getResources().getDiskMb().intValue());
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_NOMAD_WORKING_DIR));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_USE_CORE_PACKAGE_URI));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_CORE_PACKAGE_URI));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_TOPOLOGY_DOWNLOAD_CMD));
    Assert.assertTrue(task.getEnv().containsKey(NomadConstants.HERON_EXECUTOR_CMD));
    Assert.assertEquals(NomadKey.WORKING_DIRECTORY.getDefaultString() + "/container-" + String.valueOf(CONTAINER_INDEX), task.getEnv().get(NomadConstants.HERON_NOMAD_WORKING_DIR));
    Assert.assertEquals(USE_CORE_PACKAGE_URI.toString(), task.getEnv().get(NomadConstants.HERON_USE_CORE_PACKAGE_URI));
    Assert.assertEquals(CORE_PACKAGE_URI, task.getEnv().get(NomadConstants.HERON_CORE_PACKAGE_URI));
    Assert.assertEquals(TOPOLOGY_DOWNLOAD_CMD, task.getEnv().get(NomadConstants.HERON_TOPOLOGY_DOWNLOAD_CMD));
    Assert.assertEquals("./heron-core/bin/heron-executor args1 args2", task.getEnv().get(NomadConstants.HERON_EXECUTOR_CMD));
}

Example 7 with Task

use of com.hashicorp.nomad.apimodel.Task in project incubator-heron by apache.

the class NomadScheduler method getTask.

Task getTask(String taskName, int containerIndex, Resource containerResource) {
    String nomadDriver = NomadContext.getHeronNomadDriver(this.localConfig);
    Task task = new Task();
    if (nomadDriver.equals(NomadConstants.NomadDriver.RAW_EXEC.getName())) {
        getTaskSpecRawDriver(task, taskName, containerIndex);
    } else if (nomadDriver.equals(NomadConstants.NomadDriver.DOCKER.getName())) {
        getTaskSpecDockerDriver(task, taskName, containerIndex);
    } else {
        throw new IllegalArgumentException("Invalid Nomad driver specified: " + nomadDriver);
    }
    // set resources requests
    Resources resourceReqs = new Resources();
    // configure nomad to allocate dynamic ports
    Port[] ports = new Port[NomadConstants.EXECUTOR_PORTS.size()];
    int i = 0;
    for (SchedulerUtils.ExecutorPort port : NomadConstants.EXECUTOR_PORTS.keySet()) {
        ports[i] = new Port().setLabel(port.getName().replace("-", "_"));
        i++;
    }
    resourceReqs.addNetworks(new NetworkResource().addDynamicPorts(ports));
    // set memory requirements
    long memoryReqMb = containerResource.getRam().asMegabytes();
    resourceReqs.setMemoryMb(longToInt(memoryReqMb));
    // set cpu requirements
    double coresReq = containerResource.getCpu();
    double coresReqFreq = NomadContext.getCoreFreqMapping(this.localConfig) * coresReq;
    resourceReqs.setCpu(Integer.valueOf((int) Math.round(coresReqFreq)));
    // set disk requirements
    long diskReqMb = containerResource.getDisk().asMegabytes();
    resourceReqs.setDiskMb(longToInt(diskReqMb));
    task.setResources(resourceReqs);
    return task;
}

Example 8 with Task

use of com.hashicorp.nomad.apimodel.Task in project heron by twitter.

the class NomadSchedulerTest method testGetTaskGroup.

@Test
public void testGetTaskGroup() {
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    Resource resource = new Resource(CPU_RESOURCE, MEMORY_RESOURCE, DISK_RESOURCE);
    PackingPlan.ContainerPlan containerPlan = new PackingPlan.ContainerPlan(CONTAINER_INDEX, new HashSet<>(), Mockito.mock(Resource.class));
    scheduler.initialize(this.mockConfig, this.mockRuntime);
    Mockito.doReturn(new Task()).when(scheduler).getTask(Mockito.anyString(), Mockito.anyInt(), Mockito.any());
    TaskGroup taskGroup = scheduler.getTaskGroup(GROUP_NAME, CONTAINER_INDEX, resource);
    LOG.info("taskGroup: " + taskGroup);
    Assert.assertEquals(GROUP_NAME, taskGroup.getName());
    Assert.assertNotNull(taskGroup.getCount());
    Assert.assertNotNull(taskGroup.getTasks());
}

Example 9 with Task

use of com.hashicorp.nomad.apimodel.Task in project heron by twitter.

the class NomadSchedulerTest method testServiceCheck.

@SuppressWarnings("unchecked")
@Test
public void testServiceCheck() {
    this.mockConfig = this.mockConfig.newBuilder().putAll(this.mockConfig).put(NomadContext.HERON_NOMAD_METRICS_SERVICE_ADDITIONAL_TAGS, "foo,bar").put(NomadContext.HERON_NOMAD_METRICS_SERVICE_CHECK_TIMEOUT_SEC, "2").put(NomadContext.HERON_NOMAD_METRICS_SERVICE_CHECK_INTERVAL_SEC, "10").put(NomadContext.HERON_NOMAD_METRICS_SERVICE_REGISTER, true).build();
    this.mockRuntime = this.mockRuntime.newBuilder().putAll(this.mockConfig).put(NomadContext.HERON_NOMAD_METRICS_SERVICE_ADDITIONAL_TAGS, "foo,bar").put(NomadContext.HERON_NOMAD_METRICS_SERVICE_CHECK_TIMEOUT_SEC, "2").put(NomadContext.HERON_NOMAD_METRICS_SERVICE_CHECK_INTERVAL_SEC, "10").put(NomadContext.HERON_NOMAD_METRICS_SERVICE_REGISTER, true).build();
    Set<PackingPlan.ContainerPlan> containers = new HashSet<>();
    containers.add(Mockito.mock(PackingPlan.ContainerPlan.class));
    PowerMockito.mockStatic(SchedulerUtils.class);
    Resource resource = new Resource(CPU_RESOURCE, MEMORY_RESOURCE, DISK_RESOURCE);
    PowerMockito.when(SchedulerUtils.executorCommandArgs(Mockito.any(), Mockito.any(), Mockito.anyMap(), Mockito.anyString())).thenReturn(EXECUTOR_CMD_ARGS);
    PowerMockito.mockStatic(NomadScheduler.class);
    PowerMockito.when(NomadScheduler.getFetchCommand(Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(TOPOLOGY_DOWNLOAD_CMD);
    PowerMockito.when(NomadScheduler.getHeronNomadScript(this.mockConfig)).thenReturn(HERON_NOMAD_SCRIPT);
    PowerMockito.when(NomadScheduler.longToInt(MEMORY_RESOURCE.asMegabytes())).thenReturn((int) MEMORY_RESOURCE.asMegabytes());
    PowerMockito.when(NomadScheduler.longToInt(DISK_RESOURCE.asMegabytes())).thenReturn((int) DISK_RESOURCE.asMegabytes());
    PowerMockito.when(NomadScheduler.getPrometheusMetricsFile(Mockito.any())).thenReturn(PORT_FILE);
    PowerMockito.when(NomadScheduler.getMetricsServiceName(Mockito.any(), Mockito.anyInt())).thenReturn(String.format("metrics-heron-%s-%s", TOPOLOGY_NAME, CONTAINER_INDEX));
    scheduler.initialize(this.mockConfig, this.mockRuntime);
    Task task = scheduler.getTask(TASK_NAME, CONTAINER_INDEX, resource);
    LOG.info("task: " + task);
    Assert.assertEquals(task.getServices().size(), 1);
    Assert.assertEquals(task.getServices().get(0).getName(), String.format("metrics-heron-%s-%s", TOPOLOGY_NAME, CONTAINER_INDEX));
    String[] tags = { String.format("%s-%s", TOPOLOGY_NAME, CONTAINER_INDEX), "foo", "bar" };
    Assert.assertEquals(task.getServices().get(0).getTags(), Arrays.asList(tags));
    Assert.assertEquals(task.getServices().get(0).getPortLabel(), NomadConstants.METRICS_PORT);
    Assert.assertEquals(task.getServices().get(0).getChecks().size(), 1);
    Assert.assertEquals(task.getServices().get(0).getChecks().get(0).getPortLabel(), NomadConstants.METRICS_PORT);
    Assert.assertEquals(task.getServices().get(0).getChecks().get(0).getType(), NomadConstants.NOMAD_SERVICE_CHECK_TYPE);
    TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS);
    Assert.assertEquals(task.getServices().get(0).getChecks().get(0).getInterval(), TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS));
    Assert.assertEquals(task.getServices().get(0).getChecks().get(0).getTimeout(), TimeUnit.NANOSECONDS.convert(2, TimeUnit.SECONDS));
    // if service registration is turned off
    this.mockConfig = this.mockConfig.newBuilder().putAll(this.mockConfig).put(NomadContext.HERON_NOMAD_METRICS_SERVICE_REGISTER, false).build();
    this.mockRuntime = this.mockRuntime.newBuilder().putAll(this.mockConfig).put(NomadContext.HERON_NOMAD_METRICS_SERVICE_REGISTER, false).build();
    scheduler.initialize(this.mockConfig, this.mockRuntime);
    task = scheduler.getTask(TASK_NAME, CONTAINER_INDEX, resource);
    LOG.info("task: " + task);
    Assert.assertTrue(task.getServices() == null);
}

Example 10 with Task

use of com.hashicorp.nomad.apimodel.Task in project twister2 by DSC-SPIDAL.

the class NomadController method getShellDriver.

private Task getShellDriver(JobAPI.Job job) {
    String taskName = job.getJobId();
    Task task = new Task();
    // get the job working directory
    String workingDirectory = NomadContext.workingDirectory(config);
    String jobWorkingDirectory = Paths.get(workingDirectory, job.getJobId()).toString();
    String configDirectoryName = Paths.get(workingDirectory, job.getJobId(), SchedulerContext.clusterType(config)).toString();
    String corePackageFile = SchedulerContext.temporaryPackagesPath(config) + "/" + SchedulerContext.corePackageFileName(config);
    String jobPackageFile = SchedulerContext.temporaryPackagesPath(config) + "/" + SchedulerContext.jobPackageFileName(config);
    String nomadScriptContent = getNomadScriptContent(config, configDirectoryName);
    task.setName(taskName);
    task.setDriver("raw_exec");
    task.addConfig(NomadContext.NOMAD_TASK_COMMAND, NomadContext.SHELL_CMD);
    String[] args = workerProcessCommand(workingDirectory, job);
    task.addConfig(NomadContext.NOMAD_TASK_COMMAND_ARGS, args);
    Template template = new Template();
    template.setEmbeddedTmpl(nomadScriptContent);
    template.setDestPath(NomadContext.NOMAD_SCRIPT_NAME);
    task.addTemplates(template);
    Resources resourceReqs = new Resources();
    String portNamesConfig = NomadContext.networkPortNames(config);
    String[] portNames = portNamesConfig.split(",");
    // configure nomad to allocate dynamic ports
    Port[] ports = new Port[portNames.length];
    int i = 0;
    for (String p : portNames) {
        ports[i] = new Port().setLabel(p);
        i++;
    }
    NetworkResource networkResource = new NetworkResource();
    networkResource.addDynamicPorts(ports);
    resourceReqs.addNetworks(networkResource);
    JobAPI.ComputeResource computeResource = JobUtils.getComputeResource(job, 0);
    if (computeResource == null) {
        LOG.log(Level.SEVERE, "Error: there is no compute resource");
        return null;
    }
    int cpu = (int) computeResource.getCpu();
    int disk = (int) computeResource.getDiskGigaBytes();
    int memory = computeResource.getRamMegaBytes();
    resourceReqs.setCpu(cpu * 200);
    resourceReqs.setMemoryMb(memory);
    resourceReqs.setDiskMb(disk * 1024);
    LOG.log(Level.INFO, "Compute resources are " + cpu + " " + memory + " " + disk);
    Map<String, String> envVars = new HashMap<>();
    envVars.put(NomadContext.WORKING_DIRECTORY_ENV, NomadContext.workingDirectory(config));
    if (!NomadContext.sharedFileSystem(config)) {
        envVars.put(NomadContext.DOWNLOAD_PACKAGE_ENV, "false");
    } else {
        envVars.put(NomadContext.DOWNLOAD_PACKAGE_ENV, "true");
    }
    // we are putting the core packages as env variable
    envVars.put(NomadContext.CORE_PACKAGE_ENV, corePackageFile);
    envVars.put(NomadContext.JOB_PACKAGE_ENV, jobPackageFile);
    task.setEnv(envVars);
    task.setResources(resourceReqs);
    return task;
}