Examples with MessageSerializer edu.iu.dsc.tws.api.comms.packing.MessageSerializer used on opensource projects

Example 1 with MessageSerializer

use of edu.iu.dsc.tws.api.comms.packing.MessageSerializer in project twister2 by DSC-SPIDAL.

the class MToNSimple method init.

/**
 * Initialize
 */
public void init(Config cfg, MessageType t, LogicalPlan logicalPlan, int ed) {
    this.edge = ed;
    Set<Integer> thisSources = TaskPlanUtils.getTasksOfThisWorker(logicalPlan, sources);
    int executor = logicalPlan.getThisWorker();
    LOG.log(Level.FINE, String.format("%d setup loadbalance routing %s %s", logicalPlan.getThisWorker(), sources, destinations));
    this.router = new PartitionRouter(logicalPlan, sources, destinations);
    Map<Integer, Set<Integer>> internal = router.getInternalSendTasks();
    Map<Integer, Set<Integer>> external = router.getExternalSendTasks();
    this.instancePlan = logicalPlan;
    this.dataType = t;
    if (this.receiveType == null) {
        this.receiveType = dataType;
    }
    LOG.log(Level.FINE, String.format("%d adding internal/external routing", logicalPlan.getThisWorker()));
    for (int s : thisSources) {
        Set<Integer> integerSetMap = internal.get(s);
        if (integerSetMap != null) {
            this.internalDestinations.addAll(integerSetMap);
        }
        Set<Integer> integerSetMap1 = external.get(s);
        if (integerSetMap1 != null) {
            this.externalDestinations.addAll(integerSetMap1);
        }
        LOG.fine(String.format("%d adding internal/external routing %d", logicalPlan.getThisWorker(), s));
        break;
    }
    LOG.log(Level.FINE, String.format("%d done adding internal/external routing", logicalPlan.getThisWorker()));
    this.finalReceiver.init(cfg, this, receiveExpectedTaskIds());
    this.partialReceiver.init(cfg, this, router.partialExpectedTaskIds());
    Map<Integer, ArrayBlockingQueue<OutMessage>> pendingSendMessagesPerSource = new HashMap<>();
    Map<Integer, Queue<InMessage>> pendingReceiveMessagesPerSource = new HashMap<>();
    Map<Integer, Queue<InMessage>> pendingReceiveDeSerializations = new HashMap<>();
    Map<Integer, MessageSerializer> serializerMap = new HashMap<>();
    Map<Integer, MessageDeSerializer> deSerializerMap = new HashMap<>();
    Set<Integer> srcs = TaskPlanUtils.getTasksOfThisWorker(logicalPlan, sources);
    Set<Integer> tempsrcs = TaskPlanUtils.getTasksOfThisWorker(logicalPlan, sources);
    // need to set minus tasks as well
    for (Integer src : tempsrcs) {
        srcs.add((src * -1) - 1);
    }
    for (int s : srcs) {
        // later look at how not to allocate pairs for this each time
        pendingSendMessagesPerSource.put(s, new ArrayBlockingQueue<>(CommunicationContext.sendPendingMax(cfg)));
        serializerMap.put(s, Serializers.get(isKeyed, this.messageSchema));
    }
    int maxReceiveBuffers = CommunicationContext.receiveBufferCount(cfg);
    int receiveExecutorsSize = receivingExecutors().size();
    if (receiveExecutorsSize == 0) {
        receiveExecutorsSize = 1;
    }
    for (int ex : sources) {
        int capacity = maxReceiveBuffers * 2 * receiveExecutorsSize;
        pendingReceiveMessagesPerSource.put(ex, new ArrayBlockingQueue<>(capacity));
        pendingReceiveDeSerializations.put(ex, new ArrayBlockingQueue<>(capacity));
        deSerializerMap.put(ex, Deserializers.get(isKeyed, this.messageSchema));
    }
    for (int src : srcs) {
        routingParamCache.put(src, new Int2ObjectOpenHashMap<>());
        for (int dest : destinations) {
            sendRoutingParameters(src, dest);
        }
    }
    delegete.init(cfg, dataType, receiveType, keyType, receiveKeyType, logicalPlan, edge, router.receivingExecutors(), this, pendingSendMessagesPerSource, pendingReceiveMessagesPerSource, pendingReceiveDeSerializations, serializerMap, deSerializerMap, isKeyed);
}

Example 2 with MessageSerializer

use of edu.iu.dsc.tws.api.comms.packing.MessageSerializer in project twister2 by DSC-SPIDAL.

the class MToOneTree method init.

/**
 * Initialize
 */
public void init(Config cfg, MessageType dType, MessageType rcvDType, MessageType kType, MessageType rcvKType, LogicalPlan logicalPlan, int edge) {
    this.instancePlan = logicalPlan;
    this.dataType = dType;
    this.keyType = kType;
    int workerId = instancePlan.getThisWorker();
    this.edgeValue = edge;
    // we only have one path
    this.router = new InvertedBinaryTreeRouter(cfg, logicalPlan, destination, sources, index);
    // initialize the receive
    if (this.partialReceiver != null && !router.isLastReceiver()) {
        partialReceiver.init(cfg, this, receiveExpectedTaskIds());
    }
    if (this.finalReceiver != null && router.isLastReceiver()) {
        this.finalReceiver.init(cfg, this, receiveExpectedTaskIds());
    }
    LOG.log(Level.FINE, String.format("%d reduce sources %s dest %d send tasks: %s", workerId, sources, destination, router.sendQueueIds()));
    Map<Integer, ArrayBlockingQueue<OutMessage>> pendingSendMessagesPerSource = new HashMap<>();
    Map<Integer, Queue<InMessage>> pendingReceiveMessagesPerSource = new HashMap<>();
    Map<Integer, Queue<InMessage>> pendingReceiveDeSerializations = new HashMap<>();
    Map<Integer, MessageSerializer> serializerMap = new HashMap<>();
    Map<Integer, MessageDeSerializer> deSerializerMap = new HashMap<>();
    Set<Integer> srcs = router.sendQueueIds();
    for (int s : srcs) {
        // later look at how not to allocate pairs for this each time
        ArrayBlockingQueue<OutMessage> pendingSendMessages = new ArrayBlockingQueue<>(CommunicationContext.sendPendingMax(cfg));
        pendingSendMessagesPerSource.put(s, pendingSendMessages);
        serializerMap.put(s, Serializers.get(isKeyed, this.messageSchema));
    }
    int maxReceiveBuffers = CommunicationContext.receiveBufferCount(cfg);
    int receiveExecutorsSize = receivingExecutors().size();
    if (receiveExecutorsSize == 0) {
        receiveExecutorsSize = 1;
    }
    Set<Integer> execs = router.getReceiveSources();
    for (int e : execs) {
        int capacity = maxReceiveBuffers * 2 * receiveExecutorsSize;
        Queue<InMessage> pendingReceiveMessages = new ArrayBlockingQueue<>(capacity);
        pendingReceiveMessagesPerSource.put(e, pendingReceiveMessages);
        pendingReceiveDeSerializations.put(e, new ArrayBlockingQueue<>(capacity));
        deSerializerMap.put(e, Deserializers.get(isKeyed, this.messageSchema));
    }
    Set<Integer> sourcesOfThisExec = TaskPlanUtils.getTasksOfThisWorker(logicalPlan, sources);
    for (int s : sourcesOfThisExec) {
        sendRoutingParameters(s, pathToUse);
        partialSendRoutingParameters(s, pathToUse);
    }
    delegete.init(cfg, dType, rcvDType, kType, rcvKType, logicalPlan, edge, router.receivingExecutors(), this, pendingSendMessagesPerSource, pendingReceiveMessagesPerSource, pendingReceiveDeSerializations, serializerMap, deSerializerMap, isKeyed);
}

Example 3 with MessageSerializer

use of edu.iu.dsc.tws.api.comms.packing.MessageSerializer in project twister2 by DSC-SPIDAL.

the class TreeBroadcast method init.

/**
 * Initialize
 */
public void init(Config cfg, MessageType dType, MessageType recvDType, MessageType kType, MessageType recvKType, LogicalPlan tPlan, int ed) {
    this.config = cfg;
    this.instancePlan = tPlan;
    this.dataType = dType;
    this.recvDataType = recvDType;
    this.keyType = kType;
    this.recvKeyType = recvKType;
    this.edge = ed;
    this.executor = tPlan.getThisWorker();
    this.currentReceiveMessage = new ArrayBlockingQueue<>(CommunicationContext.sendPendingMax(cfg));
    // we will only have one distinct route
    router = new BinaryTreeRouter(cfg, tPlan, source, destinations);
    if (this.finalReceiver != null) {
        this.finalReceiver.init(cfg, this, receiveExpectedTaskIds());
    } else {
        throw new RuntimeException("Final receiver is required");
    }
    LOG.log(Level.FINE, String.format("%d bast sources %d dest %s send tasks: %s", executor, source, destinations, router.sendQueueIds()));
    thisSources = TaskPlanUtils.getTasksOfThisWorker(tPlan, sourceSet);
    Map<Integer, Queue<InMessage>> pendingReceiveMessagesPerSource = new HashMap<>();
    Map<Integer, Queue<InMessage>> pendingReceiveDeSerializations = new HashMap<>();
    Map<Integer, MessageSerializer> serializerMap = new HashMap<>();
    Map<Integer, MessageDeSerializer> deSerializerMap = new HashMap<>();
    Set<Integer> srcs = router.sendQueueIds();
    for (int s : srcs) {
        // later look at how not to allocate pairs for this each time
        ArrayBlockingQueue<OutMessage> pendingSendMessages = new ArrayBlockingQueue<>(CommunicationContext.sendPendingMax(cfg));
        pendingSendMessagesPerSource.put(s, pendingSendMessages);
        serializerMap.put(s, Serializers.get(kType != null, this.messageSchema));
    }
    int maxReceiveBuffers = CommunicationContext.receiveBufferCount(cfg);
    int receiveExecutorsSize = receivingExecutors().size();
    if (receiveExecutorsSize == 0) {
        receiveExecutorsSize = 1;
    }
    Set<Integer> execs = router.getReceiveSources();
    for (int e : execs) {
        int capacity = maxReceiveBuffers * 2 * receiveExecutorsSize;
        Queue<InMessage> pendingReceiveMessages = new ArrayBlockingQueue<>(capacity);
        pendingReceiveMessagesPerSource.put(source, pendingReceiveMessages);
        pendingReceiveDeSerializations.put(source, new ArrayBlockingQueue<>(capacity));
        deSerializerMap.put(source, Deserializers.get(kType != null, this.messageSchema));
    }
    calculateRoutingParameters();
    for (Integer s : srcs) {
        routingParametersCache.put(s, sendRoutingParameters(s, 0));
    }
    if (this.keyType != null) {
        delegate.init(cfg, dType, recvDataType, kType, recvKType, tPlan, ed, router.receivingExecutors(), this, pendingSendMessagesPerSource, pendingReceiveMessagesPerSource, pendingReceiveDeSerializations, serializerMap, deSerializerMap, true);
    } else {
        delegate.init(cfg, dType, recvDataType, tPlan, ed, router.receivingExecutors(), this, pendingSendMessagesPerSource, pendingReceiveMessagesPerSource, pendingReceiveDeSerializations, serializerMap, deSerializerMap, false);
    }
}