Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 6 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class BKeyedPartitionExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
    // create the communication
    partition = new BKeyedPartition(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new PartitionReceiver(), new SimpleKeyBasedSelector());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    // now initialize the workers
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.parseInt(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        List<Tuple<Integer, int[]>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
                expectedData.add(new Tuple<>(key, ints));
            }
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(// any int
    (d1, d2) -> true, IntArrayComparator.getInstance())));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new KeyedBenchWorker.MapWorker(t));
        mapThread.start();
    }
}

Example 7 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class BKeyedGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
    // create the communication
    keyedGather = new BKeyedGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver(), new SimpleKeyBasedSelector());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (ints, args) -> {
        int lowestTarget = logicalPlanBuilder.getTargets().stream().min(Comparator.comparingInt(o -> (Integer) o)).get();
        int target = Integer.valueOf(args.get("target").toString());
        Set<Integer> keysRoutedToThis = new HashSet<>();
        for (int i = 0; i < jobParameters.getTotalIterations(); i++) {
            if (i % logicalPlanBuilder.getTargets().size() == target - lowestTarget) {
                keysRoutedToThis.add(i);
            }
        }
        List<int[]> dataForEachKey = new ArrayList<>();
        for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
            dataForEachKey.add(ints);
        }
        List<Tuple<Integer, Iterator<int[]>>> expectedData = new ArrayList<>();
        for (Integer key : keysRoutedToThis) {
            expectedData.add(new Tuple<>(key, dataForEachKey.iterator()));
        }
        return expectedData.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(// any int
    (d1, d2) -> true, new IteratorComparator<>(IntArrayComparator.getInstance()))));
    LOG.log(Level.INFO, String.format("%d Sources %s target %d this %s", workerId, logicalPlanBuilder.getSources(), 1, tasksOfExecutor));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new MapWorker(t));
        mapThread.start();
    }
}

Example 8 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class RecordSource method run.

public void run() {
    int count = 0;
    for (int i = 0; i < toSend; i++) {
        byte[] randomKey = new byte[this.keySize];
        this.random.nextBytes(randomKey);
        Tuple t = new Tuple(randomKey, this.value);
        while (!operation.gather(taskId, t, 0)) {
            count++;
            for (int j = 0; j < 4; j++) {
                operation.progressChannel();
            }
        }
    }
    LOG.info("Failed attempts: " + count);
    operation.finish(taskId);
}

Example 9 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceBatchFinalReceiver method merge.

@Override
protected void merge(int dest, List<Object> dests) {
    Map<Object, Object> targetValues = reduced.get(dest);
    for (int i = 0; i < dests.size(); i++) {
        Object val = dests.get(i);
        Tuple t;
        if (val instanceof Tuple) {
            t = (Tuple) val;
        } else {
            throw new RuntimeException("Un-expected type: " + val.getClass());
        }
        Object currentVal = targetValues.get(t.getKey());
        if (currentVal != null) {
            Object newVal = reduceFunction.reduce(currentVal, t.getValue());
            targetValues.put(t.getKey(), newVal);
        } else {
            targetValues.put(t.getKey(), t.getValue());
        }
    }
    dests.clear();
}

Example 10 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KReduceBatchReceiver method moveMessagesToSendQueue.

/**
 * moves all the buffered messages into the sendQueue for the given target, this method assumes
 * that for each target that there is only one object in the queue. This is required when working
 * with reduce operations
 *
 * @param target target for which the move needs to be done
 * @return true if the messagesPerTarget is not empty at the end of the moving process or false
 * otherwise
 */
@Override
protected boolean moveMessagesToSendQueue(int target, Map<Object, Queue<Object>> messagesPerTarget) {
    Queue<Object> targetSendQueue = sendQueue.get(target);
    messagesPerTarget.entrySet().removeIf(entry -> {
        Tuple send = new Tuple(entry.getKey(), entry.getValue().peek());
        return targetSendQueue.offer(send);
    });
    return messagesPerTarget.isEmpty();
}