Examples with MkGrouper com.alibaba.jstorm.task.group.MkGrouper used on opensource projects

Example 1 with MkGrouper

use of com.alibaba.jstorm.task.group.MkGrouper in project jstorm by alibaba.

the class Common method outbound_components.

/**
     * get current task's output <Stream_id, <componentId, MkGrouper>>
     */
public static Map<String, Map<String, MkGrouper>> outbound_components(TopologyContext topology_context, WorkerData workerData) {
    Map<String, Map<String, MkGrouper>> rr = new HashMap<String, Map<String, MkGrouper>>();
    // <Stream_id,<component,Grouping>>
    Map<String, Map<String, Grouping>> output_groupings = topology_context.getThisTargets();
    for (Entry<String, Map<String, Grouping>> entry : output_groupings.entrySet()) {
        String stream_id = entry.getKey();
        Map<String, Grouping> component_grouping = entry.getValue();
        Fields out_fields = topology_context.getThisOutputFields(stream_id);
        Map<String, MkGrouper> componentGrouper = new HashMap<String, MkGrouper>();
        for (Entry<String, Grouping> cg : component_grouping.entrySet()) {
            String component = cg.getKey();
            Grouping tgrouping = cg.getValue();
            List<Integer> outTasks = topology_context.getComponentTasks(component);
            // so we don't need send tuple to it
            if (outTasks.size() > 0) {
                MkGrouper grouper = new MkGrouper(topology_context, out_fields, tgrouping, component, stream_id, workerData);
                componentGrouper.put(component, grouper);
            }
            LOG.info("outbound_components, {}-{} for task-{} on {}", component, outTasks, topology_context.getThisTaskId(), stream_id);
        }
        if (componentGrouper.size() > 0) {
            rr.put(stream_id, componentGrouper);
        }
    }
    return rr;
}

Example 2 with MkGrouper

use of com.alibaba.jstorm.task.group.MkGrouper in project jstorm by alibaba.

the class TaskSendTargets method get.

// send tuple according to grouping
public List<Integer> get(String stream, List<Object> tuple, Collection<Tuple> anchors, Object root_id) {
    List<Integer> out_tasks = new ArrayList<Integer>();
    // get grouper, then get which task should tuple be sent to.
    Map<String, MkGrouper> componentCrouping = streamComponentgrouper.get(stream);
    if (componentCrouping == null) {
        // if the target component's parallelism is 0, don't need send to
        // them
        LOG.debug("Failed to get Grouper of " + stream + " when " + debugIdStr);
        return out_tasks;
    }
    for (Entry<String, MkGrouper> ee : componentCrouping.entrySet()) {
        String targetComponent = ee.getKey();
        MkGrouper g = ee.getValue();
        if (GrouperType.direct.equals(g.gettype())) {
            throw new IllegalArgumentException("Cannot do regular emit to direct stream");
        }
        out_tasks.addAll(g.grouper(tuple));
    }
    if (isDebug(anchors, root_id)) {
        LOG.info(debugIdStr + stream + " to " + out_tasks + ":" + tuple.toString());
    }
    int num_out_tasks = out_tasks.size();
    taskStats.send_tuple(stream, num_out_tasks);
    return out_tasks;
}

Example 3 with MkGrouper

use of com.alibaba.jstorm.task.group.MkGrouper in project jstorm by alibaba.

the class TaskSendTargets method getBatch.

public Map<Object, List<MsgInfo>> getBatch(String stream, List<MsgInfo> batch) {
    Map<Object, List<MsgInfo>> outTasks = new HashMap<Object, List<MsgInfo>>();
    // get grouper, then get which task should tuple be sent to.
    Map<String, MkGrouper> componentCrouping = streamComponentgrouper.get(stream);
    if (componentCrouping == null) {
        // if the target component's parallelism is 0, don't need send to
        // them
        LOG.debug("Failed to get Grouper of " + stream + " in " + debugIdStr);
        return outTasks;
    }
    for (Entry<String, MkGrouper> ee : componentCrouping.entrySet()) {
        MkGrouper g = ee.getValue();
        if (GrouperType.direct.equals(g.gettype())) {
            throw new IllegalArgumentException("Cannot do regular emit to direct stream");
        }
        outTasks.putAll(g.grouperBatch(batch));
    }
    /*
        if (isDebug(anchors, root_id)) {

            LOG.info(debugIdStr + stream + " to " + out_tasks);
        }*/
    int num_out_tasks = 0;
    for (Entry<Object, List<MsgInfo>> entry : outTasks.entrySet()) {
        if (entry.getKey() instanceof Integer) {
            num_out_tasks += entry.getValue().size();
        } else {
            num_out_tasks += ((List<Integer>) entry.getKey()).size() * entry.getValue().size();
        }
    }
    taskStats.send_tuple(stream, num_out_tasks);
    return outTasks;
}