Examples with LoadAwareCustomStreamGrouping org.apache.storm.grouping.LoadAwareCustomStreamGrouping used on opensource projects

Example 1 with LoadAwareCustomStreamGrouping

use of org.apache.storm.grouping.LoadAwareCustomStreamGrouping in project storm by apache.

the class Executor method outboundComponents.

/**
     * Returns map of stream id to component id to grouper
     */
private Map<String, Map<String, LoadAwareCustomStreamGrouping>> outboundComponents(WorkerTopologyContext workerTopologyContext, String componentId, Map stormConf) {
    Map<String, Map<String, LoadAwareCustomStreamGrouping>> ret = new HashMap<>();
    Map<String, Map<String, Grouping>> outputGroupings = workerTopologyContext.getTargets(componentId);
    for (Map.Entry<String, Map<String, Grouping>> entry : outputGroupings.entrySet()) {
        String streamId = entry.getKey();
        Map<String, Grouping> componentGrouping = entry.getValue();
        Fields outFields = workerTopologyContext.getComponentOutputFields(componentId, streamId);
        Map<String, LoadAwareCustomStreamGrouping> componentGrouper = new HashMap<String, LoadAwareCustomStreamGrouping>();
        for (Map.Entry<String, Grouping> cg : componentGrouping.entrySet()) {
            String component = cg.getKey();
            Grouping grouping = cg.getValue();
            List<Integer> outTasks = workerTopologyContext.getComponentTasks(component);
            LoadAwareCustomStreamGrouping grouper = GrouperFactory.mkGrouper(workerTopologyContext, componentId, streamId, outFields, grouping, outTasks, stormConf);
            componentGrouper.put(component, grouper);
        }
        if (componentGrouper.size() > 0) {
            ret.put(streamId, componentGrouper);
        }
    }
    for (String stream : workerTopologyContext.getComponentCommon(componentId).get_streams().keySet()) {
        if (!ret.containsKey(stream)) {
            ret.put(stream, null);
        }
    }
    return ret;
}

Example 2 with LoadAwareCustomStreamGrouping

use of org.apache.storm.grouping.LoadAwareCustomStreamGrouping in project storm by apache.

the class Task method getOutgoingTasks.

public List<Integer> getOutgoingTasks(String stream, List<Object> values) {
    if (debug) {
        LOG.info("Emitting Tuple: taskId={} componentId={} stream={} values={}", taskId, componentId, stream, values);
    }
    ArrayList<Integer> outTasks = new ArrayList<>();
    ArrayList<LoadAwareCustomStreamGrouping> groupers = streamToGroupers.get(stream);
    if (null != groupers) {
        for (int i = 0; i < groupers.size(); ++i) {
            LoadAwareCustomStreamGrouping grouper = groupers.get(i);
            if (grouper == GrouperFactory.DIRECT) {
                throw new IllegalArgumentException("Cannot do regular emit to direct stream");
            }
            List<Integer> compTasks = grouper.chooseTasks(taskId, values);
            outTasks.addAll(compTasks);
        }
    } else {
        throw new IllegalArgumentException("Unknown stream ID: " + stream);
    }
    if (!userTopologyContext.getHooks().isEmpty()) {
        new EmitInfo(values, stream, taskId, outTasks).applyOn(userTopologyContext);
    }
    try {
        if (emitSampler.getAsBoolean()) {
            executorStats.emittedTuple(stream);
            this.taskMetrics.emittedTuple(stream);
            executorStats.transferredTuples(stream, outTasks.size());
            this.taskMetrics.transferredTuples(stream, outTasks.size());
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    return outTasks;
}

Example 3 with LoadAwareCustomStreamGrouping

use of org.apache.storm.grouping.LoadAwareCustomStreamGrouping in project storm by apache.

the class Task method getOutgoingTasks.

public List<Integer> getOutgoingTasks(Integer outTaskId, String stream, List<Object> values) {
    if (debug) {
        LOG.info("Emitting direct: {}; {} {} {} ", outTaskId, componentId, stream, values);
    }
    String targetComponent = workerTopologyContext.getComponentId(outTaskId);
    Map<String, LoadAwareCustomStreamGrouping> componentGrouping = streamComponentToGrouper.get(stream);
    LoadAwareCustomStreamGrouping grouping = componentGrouping.get(targetComponent);
    if (null == grouping) {
        outTaskId = null;
    }
    if (grouping != null && grouping != GrouperFactory.DIRECT) {
        throw new IllegalArgumentException("Cannot emitDirect to a task expecting a regular grouping");
    }
    if (!userTopologyContext.getHooks().isEmpty()) {
        new EmitInfo(values, stream, taskId, Collections.singletonList(outTaskId)).applyOn(userTopologyContext);
    }
    try {
        if (emitSampler.getAsBoolean()) {
            executorStats.emittedTuple(stream);
            this.taskMetrics.emittedTuple(stream);
            if (null != outTaskId) {
                executorStats.transferredTuples(stream, 1);
                this.taskMetrics.transferredTuples(stream, 1);
            }
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    if (null != outTaskId) {
        return Collections.singletonList(outTaskId);
    }
    return new ArrayList<>(0);
}

Example 4 with LoadAwareCustomStreamGrouping

use of org.apache.storm.grouping.LoadAwareCustomStreamGrouping in project storm by apache.

the class Executor method outboundComponents.

/**
 * Returns map of stream id to component id to grouper.
 */
private Map<String, Map<String, LoadAwareCustomStreamGrouping>> outboundComponents(WorkerTopologyContext workerTopologyContext, String componentId, Map<String, Object> topoConf) {
    Map<String, Map<String, LoadAwareCustomStreamGrouping>> ret = new HashMap<>();
    Map<String, Map<String, Grouping>> outputGroupings = workerTopologyContext.getTargets(componentId);
    for (Map.Entry<String, Map<String, Grouping>> entry : outputGroupings.entrySet()) {
        String streamId = entry.getKey();
        Map<String, Grouping> componentGrouping = entry.getValue();
        Fields outFields = workerTopologyContext.getComponentOutputFields(componentId, streamId);
        Map<String, LoadAwareCustomStreamGrouping> componentGrouper = new HashMap<String, LoadAwareCustomStreamGrouping>();
        for (Map.Entry<String, Grouping> cg : componentGrouping.entrySet()) {
            String component = cg.getKey();
            Grouping grouping = cg.getValue();
            List<Integer> outTasks = workerTopologyContext.getComponentTasks(component);
            LoadAwareCustomStreamGrouping grouper = GrouperFactory.mkGrouper(workerTopologyContext, componentId, streamId, outFields, grouping, outTasks, topoConf);
            componentGrouper.put(component, grouper);
        }
        if (componentGrouper.size() > 0) {
            ret.put(streamId, componentGrouper);
        }
    }
    for (String stream : workerTopologyContext.getComponentCommon(componentId).get_streams().keySet()) {
        if (!ret.containsKey(stream)) {
            ret.put(stream, null);
        }
    }
    return ret;
}

Example 5 with LoadAwareCustomStreamGrouping

use of org.apache.storm.grouping.LoadAwareCustomStreamGrouping in project storm by apache.

the class GrouperFactory method mkGrouper.

public static LoadAwareCustomStreamGrouping mkGrouper(WorkerTopologyContext context, String componentId, String streamId, Fields outFields, Grouping thriftGrouping, List<Integer> unsortedTargetTasks, Map topoConf) {
    List<Integer> targetTasks = Ordering.natural().sortedCopy(unsortedTargetTasks);
    final boolean isNotLoadAware = (null != topoConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING) && (boolean) topoConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING));
    CustomStreamGrouping result = null;
    switch(Thrift.groupingType(thriftGrouping)) {
        case FIELDS:
            if (Thrift.isGlobalGrouping(thriftGrouping)) {
                result = new GlobalGrouper();
            } else {
                result = new FieldsGrouper(outFields, thriftGrouping);
            }
            break;
        case SHUFFLE:
            if (isNotLoadAware) {
                result = new ShuffleGrouping();
            } else {
                result = new LoadAwareShuffleGrouping();
            }
            break;
        case ALL:
            result = new AllGrouper();
            break;
        case LOCAL_OR_SHUFFLE:
            // Prefer local tasks as target tasks if possible
            Set<Integer> sameTasks = Sets.intersection(Sets.newHashSet(targetTasks), Sets.newHashSet(context.getThisWorkerTasks()));
            targetTasks = (sameTasks.isEmpty()) ? targetTasks : new ArrayList<>(sameTasks);
            if (isNotLoadAware) {
                result = new ShuffleGrouping();
            } else {
                result = new LoadAwareShuffleGrouping();
            }
            break;
        case NONE:
            result = new NoneGrouper();
            break;
        case CUSTOM_OBJECT:
            result = (CustomStreamGrouping) Thrift.instantiateJavaObject(thriftGrouping.get_custom_object());
            break;
        case CUSTOM_SERIALIZED:
            result = Utils.javaDeserialize(thriftGrouping.get_custom_serialized(), CustomStreamGrouping.class);
            break;
        case DIRECT:
            result = DIRECT;
            break;
        default:
            result = null;
            break;
    }
    if (null != result) {
        result.prepare(context, new GlobalStreamId(componentId, streamId), targetTasks);
    }
    if (result instanceof LoadAwareCustomStreamGrouping) {
        return (LoadAwareCustomStreamGrouping) result;
    } else {
        return new BasicLoadAwareCustomStreamGrouping(result);
    }
}