Examples with LoadAwareCustomStreamGrouping org.apache.storm.grouping.LoadAwareCustomStreamGrouping used on opensource projects

Example 6 with LoadAwareCustomStreamGrouping

use of org.apache.storm.grouping.LoadAwareCustomStreamGrouping in project storm by apache.

the class GrouperFactory method mkGrouper.

public static LoadAwareCustomStreamGrouping mkGrouper(WorkerTopologyContext context, String componentId, String streamId, Fields outFields, Grouping thriftGrouping, List<Integer> unsortedTargetTasks, Map<String, Object> topoConf) {
    List<Integer> targetTasks = Ordering.natural().sortedCopy(unsortedTargetTasks);
    final boolean isNotLoadAware = (null != topoConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING) && (boolean) topoConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING));
    CustomStreamGrouping result = null;
    switch(Thrift.groupingType(thriftGrouping)) {
        case FIELDS:
            if (Thrift.isGlobalGrouping(thriftGrouping)) {
                result = new GlobalGrouper();
            } else {
                result = new FieldsGrouper(outFields, thriftGrouping);
            }
            break;
        case SHUFFLE:
            if (isNotLoadAware) {
                result = new ShuffleGrouping();
            } else {
                result = new LoadAwareShuffleGrouping();
            }
            break;
        case ALL:
            result = new AllGrouper();
            break;
        case LOCAL_OR_SHUFFLE:
            // Prefer local tasks as target tasks if possible
            Set<Integer> sameTasks = Sets.intersection(Sets.newHashSet(targetTasks), Sets.newHashSet(context.getThisWorkerTasks()));
            targetTasks = (sameTasks.isEmpty()) ? targetTasks : new ArrayList<>(sameTasks);
            if (isNotLoadAware) {
                result = new ShuffleGrouping();
            } else {
                result = new LoadAwareShuffleGrouping();
            }
            break;
        case NONE:
            result = new NoneGrouper();
            break;
        case CUSTOM_OBJECT:
            result = (CustomStreamGrouping) Thrift.instantiateJavaObject(thriftGrouping.get_custom_object());
            break;
        case CUSTOM_SERIALIZED:
            result = Utils.javaDeserialize(thriftGrouping.get_custom_serialized(), CustomStreamGrouping.class);
            break;
        case DIRECT:
            result = DIRECT;
            break;
        default:
            result = null;
            break;
    }
    if (null != result) {
        result.prepare(context, new GlobalStreamId(componentId, streamId), targetTasks);
    }
    if (result instanceof LoadAwareCustomStreamGrouping) {
        return (LoadAwareCustomStreamGrouping) result;
    } else {
        return new BasicLoadAwareCustomStreamGrouping(result);
    }
}

Example 7 with LoadAwareCustomStreamGrouping

use of org.apache.storm.grouping.LoadAwareCustomStreamGrouping in project storm by apache.

the class Task method getGroupersPerStream.

private static HashMap<String, ArrayList<LoadAwareCustomStreamGrouping>> getGroupersPerStream(Map<String, Map<String, LoadAwareCustomStreamGrouping>> streamComponentToGrouper) {
    HashMap<String, ArrayList<LoadAwareCustomStreamGrouping>> result = new HashMap<>(streamComponentToGrouper.size());
    for (Entry<String, Map<String, LoadAwareCustomStreamGrouping>> entry : streamComponentToGrouper.entrySet()) {
        String stream = entry.getKey();
        Map<String, LoadAwareCustomStreamGrouping> groupers = entry.getValue();
        ArrayList<LoadAwareCustomStreamGrouping> perStreamGroupers = new ArrayList<>();
        if (groupers != null) {
            // null for __system bolt
            for (LoadAwareCustomStreamGrouping grouper : groupers.values()) {
                perStreamGroupers.add(grouper);
            }
        }
        result.put(stream, perStreamGroupers);
    }
    return result;
}