Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 16 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class DataLocalityTaskSchedulerTest method testUniqueSchedules3.

@Test
public void testUniqueSchedules3() {
    int parallel = 10;
    int workers = 3;
    ComputeGraph graph = createGraphWithComputeTaskAndConstraints(parallel);
    DataLocalityStreamingTaskScheduler scheduler = new DataLocalityStreamingTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config, 1);
    generateData(config);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Assert.assertNotNull(plan1);
    Map<Integer, WorkerSchedulePlan> containersMap = plan1.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        Assert.assertEquals(containerPlanTaskInstances.size(), Integer.parseInt(graph.getGraphConstraints().get(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER)));
    }
}

Example 17 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getParallelTaskMap.

public Map<String, Integer> getParallelTaskMap(Vertex taskVertex) {
    Map<String, Integer> parallelTaskMap = new LinkedHashMap<>();
    Config config = taskVertex.getConfig();
    String taskName = taskVertex.getName();
    int parallelTaskCount;
    if (taskVertex.getParallelism() >= 1) {
        parallelTaskCount = taskVertex.getParallelism();
    } else {
        parallelTaskCount = TaskSchedulerContext.taskParallelism(config);
    }
    parallelTaskMap.put(taskName, parallelTaskCount);
    return parallelTaskMap;
}

Example 18 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getParallelTaskMap.

/**
 * This method is mainly to generate the parallel task map (maintain order) for the task vertex.
 * If the user specifies the parallelism value greater than or equal "1" will be considered as a
 * parallelism value. Otherwise, the system assign the default parallelism value to the task
 * vertex from the task scheduling configuration file.
 */
public Map<String, Integer> getParallelTaskMap(Set<Vertex> iTaskSet, Map<String, Map<String, String>> nodeConstraintsMap) {
    Map<String, Integer> parallelTaskMap = new LinkedHashMap<>();
    for (Vertex taskVertex : iTaskSet) {
        Config config = taskVertex.getConfig();
        String taskName = taskVertex.getName();
        int parallelTaskCount;
        if (!nodeConstraintsMap.get(taskName).isEmpty()) {
            Map<String, String> vertexMap = nodeConstraintsMap.get(taskName);
            if (vertexMap.containsKey(Context.TWISTER2_TASK_INSTANCE_ODD_PARALLELISM)) {
                parallelTaskCount = Integer.valueOf(String.valueOf(vertexMap.get(Context.TWISTER2_TASK_INSTANCE_ODD_PARALLELISM)));
            } else {
                parallelTaskCount = taskVertex.getParallelism();
            }
        } else {
            if (taskVertex.getParallelism() >= 1) {
                parallelTaskCount = taskVertex.getParallelism();
            } else {
                parallelTaskCount = TaskSchedulerContext.taskParallelism(config);
            }
        }
        parallelTaskMap.put(taskName, parallelTaskCount);
    }
    return parallelTaskMap;
}

Example 19 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TaskSchedulerTest method getConfig.

private Config getConfig() {
    String twister2Home = "/home/" + System.getProperty("user.dir") + "/twister2/bazel-bin/scripts/package/twister2-0.8.0-SNAPSHOT";
    String configDir = "/home/" + System.getProperty("user.dir") + "/twister2/twister2/taskscheduler/tests/conf/";
    String clusterType = "standalone";
    Config config = ConfigLoader.loadConfig(twister2Home, configDir, clusterType);
    return Config.newBuilder().putAll(config).build();
}

Example 20 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class StormBenchmark method main.

public static void main(String[] args) {
    Config config = ResourceAllocator.loadConfig(Collections.emptyMap());
    JobConfig jobConfig = new JobConfig();
    jobConfig.put(PARAM_SIZE, Integer.valueOf(args[0]));
    int parallelSources = Integer.valueOf(args[1]);
    jobConfig.put(PARAM_PARALLEL_SOURCES, parallelSources);
    jobConfig.put(PARAM_MESSAGES_COUNT, Integer.valueOf(args[2]));
    String operation = "reduce";
    if (args.length == 4) {
        operation = args[3];
    }
    jobConfig.put(PARAM_OPERATION, operation);
    Twister2Job.Twister2JobBuilder jobBuilder = Twister2Job.newBuilder();
    jobBuilder.setJobName("storm-bench-mark");
    jobBuilder.setWorkerClass(StormBenchmark.class.getName());
    jobBuilder.setConfig(jobConfig);
    jobBuilder.addComputeResource(1, 1024, parallelSources + 1);
    // now submit the job
    Twister2Submitter.submitJob(jobBuilder.build(), config);
}