Search in sources :

Example 6 with TWSCommunication

use of edu.iu.dsc.tws.comms.core.TWSCommunication in project twister2 by DSC-SPIDAL.

the class BasicGatherTestCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    this.randomString = new RandomString(128000, new Random(), RandomString.ALPHANUM);
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
    }
    int dest = NO_OF_TASKS;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        aggregate = channel.gather(newCfg, MessageType.INTEGER, 0, sources, dest, new FinalGatherReceive());
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                aggregate.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}
Also used : HashMap(java.util.HashMap) TWSCommunication(edu.iu.dsc.tws.comms.core.TWSCommunication) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) TaskPlan(edu.iu.dsc.tws.comms.core.TaskPlan) TWSNetwork(edu.iu.dsc.tws.comms.core.TWSNetwork) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) Random(java.util.Random) HashSet(java.util.HashSet)

Example 7 with TWSCommunication

use of edu.iu.dsc.tws.comms.core.TWSCommunication in project twister2 by DSC-SPIDAL.

the class BasicMemoryManagerByteKeyedGatherCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    this.randomString = new RandomString(128000, new Random(), RandomString.ALPHANUM);
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
    }
    int dest = NO_OF_TASKS;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up keyed gather MM dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        aggregate = channel.gather(newCfg, MessageType.BYTE, MessageType.BYTE, 0, sources, dest, new GatherBatchFinalReceiver(new FinalGatherReceive()), new GatherBatchPartialReceiver(dest));
        aggregate.setMemoryMapped(true);
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                aggregate.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}
Also used : HashMap(java.util.HashMap) GatherBatchFinalReceiver(edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchFinalReceiver) TWSCommunication(edu.iu.dsc.tws.comms.core.TWSCommunication) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) TaskPlan(edu.iu.dsc.tws.comms.core.TaskPlan) TWSNetwork(edu.iu.dsc.tws.comms.core.TWSNetwork) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) GatherBatchPartialReceiver(edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchPartialReceiver) Random(java.util.Random) HashSet(java.util.HashSet)

Example 8 with TWSCommunication

use of edu.iu.dsc.tws.comms.core.TWSCommunication in project twister2 by DSC-SPIDAL.

the class BasicMemoryManagerContainer method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    this.randomString = new RandomString(128000, new Random(), RandomString.ALPHANUM);
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
    }
    int dest = NO_OF_TASKS;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("###################### Running LMDB unit tests ######################");
    testPrimitivesLMDB();
    LOG.info("################# Running BufferedMemoryManager unit tests ##################");
    testPrimitivesBuffered();
}
Also used : HashMap(java.util.HashMap) TWSCommunication(edu.iu.dsc.tws.comms.core.TWSCommunication) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) TaskPlan(edu.iu.dsc.tws.comms.core.TaskPlan) TWSNetwork(edu.iu.dsc.tws.comms.core.TWSNetwork) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) Random(java.util.Random) HashSet(java.util.HashSet)

Example 9 with TWSCommunication

use of edu.iu.dsc.tws.comms.core.TWSCommunication in project twister2 by DSC-SPIDAL.

the class BasicMemoryManagerGatherCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    this.randomString = new RandomString(128000, new Random(), RandomString.ALPHANUM);
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
    }
    int dest = NO_OF_TASKS;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up gather MM dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        aggregate = channel.gather(newCfg, MessageType.INTEGER, 0, sources, dest, new GatherBatchFinalReceiver(new FinalGatherReceive()), new GatherBatchPartialReceiver(dest));
        aggregate.setMemoryMapped(true);
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                aggregate.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}
Also used : HashMap(java.util.HashMap) GatherBatchFinalReceiver(edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchFinalReceiver) TWSCommunication(edu.iu.dsc.tws.comms.core.TWSCommunication) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) TaskPlan(edu.iu.dsc.tws.comms.core.TaskPlan) TWSNetwork(edu.iu.dsc.tws.comms.core.TWSNetwork) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) GatherBatchPartialReceiver(edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchPartialReceiver) Random(java.util.Random) HashSet(java.util.HashSet)

Example 10 with TWSCommunication

use of edu.iu.dsc.tws.comms.core.TWSCommunication in project twister2 by DSC-SPIDAL.

the class BasicMemoryManagerKeyedGatherCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    this.randomString = new RandomString(128000, new Random(), RandomString.ALPHANUM);
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
    }
    int dest = NO_OF_TASKS;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up keyed gather MM dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        aggregate = channel.gather(newCfg, MessageType.INTEGER, MessageType.INTEGER, 0, sources, dest, new GatherBatchFinalReceiver(new FinalGatherReceive()), new GatherBatchPartialReceiver(dest));
        aggregate.setMemoryMapped(true);
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                aggregate.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}
Also used : HashMap(java.util.HashMap) GatherBatchFinalReceiver(edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchFinalReceiver) TWSCommunication(edu.iu.dsc.tws.comms.core.TWSCommunication) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) TaskPlan(edu.iu.dsc.tws.comms.core.TaskPlan) TWSNetwork(edu.iu.dsc.tws.comms.core.TWSNetwork) RandomString(edu.iu.dsc.tws.examples.utils.RandomString) GatherBatchPartialReceiver(edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchPartialReceiver) Random(java.util.Random) HashSet(java.util.HashSet)

Aggregations

TWSCommunication (edu.iu.dsc.tws.comms.core.TWSCommunication)28 TWSNetwork (edu.iu.dsc.tws.comms.core.TWSNetwork)28 TaskPlan (edu.iu.dsc.tws.comms.core.TaskPlan)28 HashMap (java.util.HashMap)28 HashSet (java.util.HashSet)28 LinkedQueue (edu.iu.dsc.tws.task.api.LinkedQueue)10 Message (edu.iu.dsc.tws.task.api.Message)10 TaskExecutorFixedThread (edu.iu.dsc.tws.task.core.TaskExecutorFixedThread)10 ArrayList (java.util.ArrayList)8 RandomString (edu.iu.dsc.tws.examples.utils.RandomString)7 DataflowTaskGraphGenerator (edu.iu.dsc.tws.task.taskgraphbuilder.DataflowTaskGraphGenerator)7 Random (java.util.Random)7 GatherBatchFinalReceiver (edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchFinalReceiver)4 GatherBatchPartialReceiver (edu.iu.dsc.tws.comms.mpi.io.gather.GatherBatchPartialReceiver)4 SinkTask (edu.iu.dsc.tws.task.api.SinkTask)4 SourceTask (edu.iu.dsc.tws.task.api.SourceTask)4 Task (edu.iu.dsc.tws.task.api.Task)4 DataflowTaskGraphParser (edu.iu.dsc.tws.task.taskgraphbuilder.DataflowTaskGraphParser)4 DataflowOperation (edu.iu.dsc.tws.task.taskgraphbuilder.DataflowOperation)3 List (java.util.List)3