use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.
the class KMeansUtils method createWordCountPlan.
/**
* Let assume we have 2 tasks per container and one additional for first container,
* which will be the destination
* @param plan the resource plan from scheduler
* @return task plan
*/
public static TaskPlan createWordCountPlan(Config cfg, ResourcePlan plan, int noOfTasks) {
int noOfProcs = plan.noOfContainers();
LOG.log(Level.INFO, "No of containers: " + noOfProcs);
Map<Integer, Set<Integer>> executorToGraphNodes = new HashMap<>();
Map<Integer, Set<Integer>> groupsToExeuctors = new HashMap<>();
int thisExecutor = plan.getThisId();
List<ResourceContainer> containers = plan.getContainers();
Map<String, List<ResourceContainer>> containersPerNode = new HashMap<>();
for (ResourceContainer c : containers) {
String name = (String) c.getProperty(SchedulerContext.WORKER_NAME);
List<ResourceContainer> containerList;
if (!containersPerNode.containsKey(name)) {
containerList = new ArrayList<>();
containersPerNode.put(name, containerList);
} else {
containerList = containersPerNode.get(name);
}
containerList.add(c);
}
int taskPerExecutor = noOfTasks / noOfProcs;
for (int i = 0; i < noOfProcs; i++) {
Set<Integer> nodesOfExecutor = new HashSet<>();
for (int j = 0; j < taskPerExecutor; j++) {
nodesOfExecutor.add(i * taskPerExecutor + j);
}
executorToGraphNodes.put(i, nodesOfExecutor);
}
int i = 0;
// we take each container as an executor
for (Map.Entry<String, List<ResourceContainer>> e : containersPerNode.entrySet()) {
Set<Integer> executorsOfGroup = new HashSet<>();
for (ResourceContainer c : e.getValue()) {
executorsOfGroup.add(c.getId());
}
groupsToExeuctors.put(i, executorsOfGroup);
i++;
}
String print = printMap(executorToGraphNodes);
LOG.fine("Executor To Graph: " + print);
print = printMap(groupsToExeuctors);
LOG.fine("Groups to executors: " + print);
// and reduce task in 0th process
return new TaskPlan(executorToGraphNodes, groupsToExeuctors, thisExecutor);
}
use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.
the class WordAggregator method init.
@Override
public void init(Config cfg, DataFlowOperation op, Map<Integer, List<Integer>> expectedIds) {
TaskPlan plan = op.getTaskPlan();
this.executor = op.getTaskPlan().getThisExecutor();
LOG.fine(String.format("%d final expected task ids %s", plan.getThisExecutor(), expectedIds));
}
use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.
the class BaseBroadcastCommunication method init.
@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
try {
this.config = cfg;
this.resourcePlan = plan;
this.id = containerId;
this.status = Status.INIT;
this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
// lets create the task plan
TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
LOG.log(Level.INFO, "Task plan: " + taskPlan);
// first get the communication config file
TWSNetwork network = new TWSNetwork(cfg, taskPlan);
TWSCommunication channel = network.getDataFlowTWSCommunication();
Set<Integer> sources = new HashSet<>();
for (int i = 0; i < NO_OF_TASKS; i++) {
sources.add(i);
}
int dest = NO_OF_TASKS;
Map<String, Object> newCfg = new HashMap<>();
LOG.info(String.format("Setting up reduce dataflow operation %d %s", dest, sources));
// this method calls the init method
// I think this is wrong
broadcast = channel.broadCast(newCfg, MessageType.OBJECT, 0, dest, sources, new BCastReceive());
// the map thread where data is produced
if (id == 0) {
Thread mapThread = new Thread(new MapWorker());
mapThread.start();
}
// we need to progress the communication
while (true) {
try {
// progress the channel
channel.progress();
// we should progress the communication directive
broadcast.progress();
Thread.yield();
} catch (Throwable t) {
t.printStackTrace();
}
}
} catch (Throwable t) {
t.printStackTrace();
}
}
use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.
the class BaseKeyedReduceCommunication method init.
@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
this.config = cfg;
this.resourcePlan = plan;
this.id = containerId;
this.status = Status.INIT;
this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
this.reduceTask = NO_OF_TASKS / 2;
// lets create the task plan
TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
// first get the communication config file
TWSNetwork network = new TWSNetwork(cfg, taskPlan);
TWSCommunication channel = network.getDataFlowTWSCommunication();
Set<Integer> sources = new HashSet<>();
for (int i = 0; i < NO_OF_TASKS / 2; i++) {
sources.add(i);
}
Set<Integer> destinations = new HashSet<>();
for (int i = 0; i < NO_OF_TASKS / 2; i++) {
destinations.add(NO_OF_TASKS / 2 + i);
}
Map<String, Object> newCfg = new HashMap<>();
LOG.info("Setting up reduce dataflow operation");
// this method calls the init method
// I think this is wrong
reduce = channel.keyedReduce(newCfg, MessageType.OBJECT, destinations, sources, destinations, new FinalReduceReceive(), new PartialReduceWorker());
if (id == 0 || id == 1) {
for (int i = 0; i < noOfTasksPerExecutor; i++) {
// the map thread where data is produced
LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
mapThread.start();
}
}
// we need to progress the communication
while (true) {
try {
// progress the channel
channel.progress();
// we should progress the communication directive
reduce.progress();
Thread.yield();
} catch (Throwable t) {
LOG.severe("Error occurred: " + id);
t.printStackTrace();
}
}
}
use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.
the class BasePartitionMultiByteCommunication method init.
@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
this.config = cfg;
this.resourcePlan = plan;
this.id = containerId;
this.status = Status.INIT;
this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
// lets create the task plan
TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
// first get the communication config file
TWSNetwork network = new TWSNetwork(cfg, taskPlan);
TWSCommunication channel = network.getDataFlowTWSCommunication();
Set<Integer> sources = new HashSet<>();
Set<Integer> dests = new HashSet<>();
for (int i = 0; i < NO_OF_TASKS; i++) {
sources.add(i);
dests.add(i);
}
Map<String, Object> newCfg = new HashMap<>();
LOG.info("Setting up partition dataflow operation");
try {
// this method calls the init method
// I think this is wrong
Map<Integer, List<Integer>> expectedIds = new HashMap<>();
for (int i = 0; i < NO_OF_TASKS; i++) {
expectedIds.put(i, new ArrayList<>());
for (int j = 0; j < NO_OF_TASKS; j++) {
if (!(i == j)) {
expectedIds.get(i).add(j);
}
}
}
FinalPartitionReciver finalPartitionRec = new FinalPartitionReciver();
partition = channel.partition(newCfg, MessageType.MULTI_FIXED_BYTE, MessageType.MULTI_FIXED_BYTE, 2, sources, dests, finalPartitionRec);
finalPartitionRec.setMap(expectedIds);
partition.setMemoryMapped(true);
for (int i = 0; i < noOfTasksPerExecutor; i++) {
// the map thread where data is produced
LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
mapThread.start();
}
// we need to progress the communication
while (true) {
try {
// progress the channel
channel.progress();
// we should progress the communication directive
partition.progress();
Thread.yield();
} catch (Throwable t) {
t.printStackTrace();
}
}
} catch (Throwable t) {
t.printStackTrace();
}
}
Aggregations