Examples with Tuple edu.iu.dsc.tws.api.comms.structs.Tuple used on opensource projects

Example 51 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class AllGatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    LOG.info("test foreach");
    src.allGather().forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    src.allGather().map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    src.allGather().flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test computec");
    src.allGather().compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        output.collect("sum=" + sum);
    }).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test compute");
    src.allGather().compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        return "sum=" + sum;
    }).direct().forEach(s -> LOG.info("compute: " + s));
}

Example 52 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class DirectExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src");
    DirectTLink<Integer> direct = src.direct().setName("direct");
    LOG.info("test foreach");
    direct.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    direct.map(i -> i.toString() + "$$").setName("map").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    direct.flatmap((i, c) -> c.collect(i.toString() + "##")).setName("flatmap").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    direct.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).setName("compute").withSchema(PrimitiveSchemas.STRING).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    direct.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).setName("ccompute").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test map2tup");
    direct.mapToTuple(i -> new Tuple<>(i, i.toString())).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = direct.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}

Example 53 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class BJoin method join.

/**
 * Send a data to be partitioned
 *
 * @param source source
 * @param key key for the data
 * @param data data
 * @param flags data flag
 * @return true if the data is accepted
 */
public boolean join(int source, Object key, Object data, int flags, int tag) {
    int dest = destinationSelector.next(source, key, data);
    boolean send;
    if (tag == 0) {
        Tuple message = new Tuple<>(key, data);
        send = partitionLeft.send(source, message, flags, dest);
    } else if (tag == 1) {
        Tuple message = new Tuple<>(key, data);
        send = partitionRight.send(source, message, flags, dest);
    } else {
        throw new RuntimeException("Tag value must be either 0(left) or 1(right) for join operation");
    }
    if (send) {
        destinationSelector.commit(source, dest);
    }
    return send;
}

Example 54 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KGatherBatchFinalReceiver method merge.

@Override
protected void merge(int dest, List<Object> dests) {
    Map<Object, List<Object>> targetValues = gathered.get(dest);
    for (int i = 0; i < dests.size(); i++) {
        Object val = dests.get(i);
        Tuple t;
        if (val instanceof Tuple) {
            t = (Tuple) val;
        } else {
            throw new RuntimeException("Un-expected type: " + val.getClass());
        }
        List<Object> currentVal = targetValues.get(t.getKey());
        if (currentVal == null) {
            currentVal = new AggregatedObjects<>();
            targetValues.put(t.getKey(), currentVal);
        }
        currentVal.add(t.getValue());
    }
    dests.clear();
}

Example 55 with Tuple

use of edu.iu.dsc.tws.api.comms.structs.Tuple in project twister2 by DSC-SPIDAL.

the class KGatherStreamingReceiver method offerMessage.

/**
 * saves the given message (or messages if the object is a list) into the messages data structure
 * if possible and rejects the message if the whole message cannot be added to the messages
 * data structure.
 *
 * @param target target for which the messages are to be added
 * @param object the message/messages to be added
 * @return true if the message was added or false otherwise
 */
@SuppressWarnings("rawtypes")
protected boolean offerMessage(int target, Object object) {
    Map<Object, Queue<Object>> messagesPerTarget = messages.get(target);
    if (messagesPerTarget.size() > keyLimit) {
        LOG.fine(String.format("Executor %d Partial cannot add any further keys needs flush ", executor));
        moveMessagesToSendQueue(target, messagesPerTarget);
        return false;
    } else if (localWindowCount > windowSize) {
        if (moveMessagesToSendQueue(target, messagesPerTarget)) {
            // TODO: what if the move returns false, do we still set the localWindowCount to zero?
            localWindowCount = 0;
        }
    }
    if (object instanceof List) {
        List dataList = (List) object;
        Map<Object, List<Object>> tempList = new HashMap<>();
        for (Object dataEntry : dataList) {
            Tuple tuple = (Tuple) dataEntry;
            // If any of the keys are full the method returns false because partial objects cannot be
            // added to the messages data structure
            Object key = tuple.getKey();
            if (!isFinalBatchReceiver && messagesPerTarget.containsKey(key) && messagesPerTarget.get(key).size() >= limitPerKey) {
                moveMessageToSendQueue(target, messagesPerTarget, tuple.getKey());
                LOG.fine(String.format("Executor %d Partial cannot add any further values for key " + "needs flush ", executor));
                return false;
            }
            if (tempList.containsKey(key)) {
                tempList.get(key).add(tuple.getValue());
            } else {
                tempList.put(key, new AggregatedObjects<>());
                tempList.get(key).add(tuple.getValue());
            }
        }
        boolean offerDone = true;
        for (Object key : tempList.keySet()) {
            if (messagesPerTarget.containsKey(key)) {
                List<Object> values = tempList.get(key);
                for (Object value : values) {
                    offerDone &= messagesPerTarget.get(key).offer(value);
                    localWindowCount++;
                }
            } else {
                ArrayDeque<Object> messagesPerKey = new ArrayDeque<>();
                List<Object> values = tempList.get(key);
                for (Object value : values) {
                    offerDone &= messagesPerKey.offer(value);
                    localWindowCount++;
                }
                messagesPerTarget.put(key, messagesPerKey);
            }
        }
        // cannot be recovered
        if (!offerDone) {
            throw new RuntimeException("Message lost during processing");
        }
    } else {
        Tuple tuple = (Tuple) object;
        if (messagesPerTarget.containsKey(tuple.getKey())) {
            if (messagesPerTarget.get(tuple.getKey()).size() < limitPerKey || isFinalBatchReceiver) {
                localWindowCount++;
                return messagesPerTarget.get(tuple.getKey()).offer(tuple.getValue());
            } else {
                LOG.fine(String.format("Executor %d Partial cannot add any further values for key " + "needs flush ", executor));
                moveMessageToSendQueue(target, messagesPerTarget, tuple.getKey());
                return false;
            }
        } else {
            ArrayDeque<Object> messagesPerKey = new ArrayDeque<>();
            messagesPerKey.add(tuple.getValue());
            messagesPerTarget.put(tuple.getKey(), messagesPerKey);
            localWindowCount++;
        }
    }
    if (localWindowCount > windowSize) {
        if (moveMessagesToSendQueue(target, messagesPerTarget)) {
            // TODO: what if the move returns false, do we still set the localWindowCount to zero?
            localWindowCount = 0;
        }
    }
    return true;
}