Examples with LogicalPlanBuilder edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder used on opensource projects

Example 21 with LogicalPlanBuilder

use of edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder in project twister2 by DSC-SPIDAL.

the class BJoinExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    Set<Integer> sources = new HashSet<>();
    Set<Integer> targets = new HashSet<>();
    Integer noOfSourceTasks = jobParameters.getTaskStages().get(0);
    for (int i = 0; i < noOfSourceTasks; i++) {
        sources.add(i);
    }
    Integer noOfTargetTasks = jobParameters.getTaskStages().get(1);
    for (int i = 0; i < noOfTargetTasks; i++) {
        targets.add(noOfSourceTasks + i);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    join = new BJoin(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, MessageTypes.INTEGER_ARRAY, new JoinReceiver(), new SimpleKeyBasedSelector(), false, Comparator.comparingInt(o -> (Integer) o), CommunicationContext.JoinType.INNER, CommunicationContext.JoinAlgorithm.SORT);
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, sources, 1, tasksOfExecutor));
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new JoinedKeyedBenchWorker.MapWorker(t));
        mapThread.start();
    }
}

Example 22 with LogicalPlanBuilder

use of edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder in project twister2 by DSC-SPIDAL.

the class BKeyedReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    keyedReduce = new BKeyedReduce(workerEnv.getCommunicator(), logicalPlanBuilder, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalBulkReceiver(), MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new SimpleKeyBasedSelector());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.parseInt(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        int[] reduced = GeneratorUtils.multiplyIntArray(ints, logicalPlanBuilder.getSources().size());
        List<Tuple<Integer, int[]>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            expectedData.add(new Tuple<>(key, reduced));
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new KeyedBenchWorker.MapWorker(t));
        mapThread.start();
    }
}

Example 23 with LogicalPlanBuilder

use of edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder in project twister2 by DSC-SPIDAL.

the class SAllGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    gather = new SAllGather(workerEnv.getCommunicator(), logicalPlanBuilder, new FinalReduceReceiver(), MessageTypes.INTEGER_ARRAY);
    Set<Integer> sourceTasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : sourceTasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (sourceTasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    Set<Integer> targetTasksOfExecutor = logicalPlanBuilder.getTargetsOnThisWorker();
    for (int taskId : targetTasksOfExecutor) {
        if (logicalPlanBuilder.getTargets().contains(taskId)) {
            gatherDone = false;
            if (workerId == 0) {
                receiverInWorker0 = taskId;
            }
        }
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (dataArray, args) -> {
        List<Tuple<Integer, int[]>> listOfArrays = new ArrayList<>();
        for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
            listOfArrays.add(new Tuple<>(i, dataArray));
        }
        return listOfArrays.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    // now initialize the workers
    for (int t : sourceTasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}

Example 24 with LogicalPlanBuilder

use of edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder in project twister2 by DSC-SPIDAL.

the class SKeyedGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    keyedGather = new SKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.OBJECT, MessageTypes.OBJECT, new GatherBulkReceiver(), new LoadBalanceSelector());
    Set<Integer> sourceTasks = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : sourceTasks) {
        finishedSources.put(t, false);
    }
    if (sourceTasks.size() == 0) {
        sourcesDone = true;
    }
    Set<Integer> sinkTasks = logicalPlanBuilder.getTargetsOnThisWorker();
    LOG.log(Level.INFO, String.format("Worker[%d], Source Tasks %s , Sink Tasks %s", workerId, sourceTasks, sinkTasks));
    // now initialize the workers
    for (int t : sourceTasks) {
        // the map thread where data is produced
        MapWorker mapWorker = new MapWorker(t);
        mapWorker.setTimingForLowestTargetOnly(true);
        Thread mapThread = new Thread(mapWorker);
        mapThread.start();
    }
}

Example 25 with LogicalPlanBuilder

use of edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder in project twister2 by DSC-SPIDAL.

the class SReduceExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (jobParameters.getTargets() != 1) {
        LOG.warning("Setting targets to 1. Found, " + jobParameters.getTargets());
        jobParameters.getTaskStages().set(1, 1);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv).withFairDistribution();
    // create the communication
    reduce = new SReduce(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new ReduceOperationFunction(Op.SUM, MessageTypes.INTEGER_ARRAY), new FinalSingularReceiver());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    sourcesDone = tasksOfExecutor.size() == 0;
    reduceDone = !logicalPlan.getLogicalIdsOfWorker(workerId).contains(logicalPlanBuilder.getTargets().iterator().next());
    // generating the expectedIterations results at the end
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (array, args) -> {
        int sourcesCount = jobParameters.getTaskStages().get(0);
        int[] outArray = new int[array.length];
        for (int i = 0; i < array.length; i++) {
            outArray[i] = array[i] * sourcesCount;
        }
        return outArray;
    }, IntArrayComparator.getInstance());
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}