Examples with GlobalStreamId org.apache.storm.generated.GlobalStreamId used on opensource projects

Example 16 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project incubator-atlas by apache.

the class StormTopologyUtil method getTerminalUserBoltNames.

public static Set<String> getTerminalUserBoltNames(StormTopology topology) {
    Set<String> terminalBolts = new HashSet<>();
    Set<String> inputs = new HashSet<>();
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String name = entry.getKey();
        Set<GlobalStreamId> inputsForBolt = entry.getValue().get_common().get_inputs().keySet();
        if (!isSystemComponent(name)) {
            for (GlobalStreamId streamId : inputsForBolt) {
                inputs.add(streamId.get_componentId());
            }
        }
    }
    for (String boltName : topology.get_bolts().keySet()) {
        if (!isSystemComponent(boltName) && !inputs.contains(boltName)) {
            terminalBolts.add(boltName);
        }
    }
    return terminalBolts;
}

Example 17 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project flink by apache.

the class StormTupleTest method testGetSourceGlobalStreamid.

@Test
public void testGetSourceGlobalStreamid() {
    GlobalStreamId globalStreamid = new StormTuple<>(null, null, -1, "streamId", "componentID", null).getSourceGlobalStreamid();
    Assert.assertEquals("streamId", globalStreamid.get_streamId());
    Assert.assertEquals("componentID", globalStreamid.get_componentId());
}

Example 18 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class StatsUtil method aggregateBoltStats.

/**
     * aggregate bolt stats
     *
     * @param statsSeq   a seq of ExecutorStats
     * @param includeSys whether to include system streams
     * @return aggregated bolt stats: {metric -> win -> global stream id -> value}
     */
public static <T> Map<String, Map> aggregateBoltStats(List<ExecutorSummary> statsSeq, boolean includeSys) {
    Map<String, Map> ret = new HashMap<>();
    Map<String, Map<String, Map<T, Long>>> commonStats = aggregateCommonStats(statsSeq);
    // filter sys streams if necessary
    commonStats = preProcessStreamSummary(commonStats, includeSys);
    List<Map<String, Map<GlobalStreamId, Long>>> acked = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Long>>> failed = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Long>>> executed = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Double>>> processLatencies = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Double>>> executeLatencies = new ArrayList<>();
    for (ExecutorSummary summary : statsSeq) {
        ExecutorStats stat = summary.get_stats();
        acked.add(stat.get_specific().get_bolt().get_acked());
        failed.add(stat.get_specific().get_bolt().get_failed());
        executed.add(stat.get_specific().get_bolt().get_executed());
        processLatencies.add(stat.get_specific().get_bolt().get_process_ms_avg());
        executeLatencies.add(stat.get_specific().get_bolt().get_execute_ms_avg());
    }
    mergeMaps(ret, commonStats);
    putKV(ret, ACKED, aggregateCounts(acked));
    putKV(ret, FAILED, aggregateCounts(failed));
    putKV(ret, EXECUTED, aggregateCounts(executed));
    putKV(ret, PROC_LATENCIES, aggregateAverages(processLatencies, acked));
    putKV(ret, EXEC_LATENCIES, aggregateAverages(executeLatencies, executed));
    return ret;
}

Example 19 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class TopologyContext method toJSONString.

@Override
public String toJSONString() {
    Map<String, Object> obj = new HashMap<>();
    obj.put("task->component", this.getTaskToComponent());
    obj.put("taskid", this.getThisTaskId());
    obj.put("componentid", this.getThisComponentId());
    List<String> streamList = new ArrayList<>();
    streamList.addAll(this.getThisStreams());
    obj.put("streams", streamList);
    obj.put("stream->outputfields", this.getThisOutputFieldsForStreams());
    // Convert targets to a JSON serializable format
    Map<String, Map<String, Object>> stringTargets = new HashMap<>();
    for (Map.Entry<String, Map<String, Grouping>> entry : this.getThisTargets().entrySet()) {
        Map<String, Object> stringTargetMap = new HashMap<>();
        for (Map.Entry<String, Grouping> innerEntry : entry.getValue().entrySet()) {
            stringTargetMap.put(innerEntry.getKey(), groupingToJSONableMap(innerEntry.getValue()));
        }
        stringTargets.put(entry.getKey(), stringTargetMap);
    }
    obj.put("stream->target->grouping", stringTargets);
    // Convert sources to a JSON serializable format
    Map<String, Map<String, Object>> stringSources = new HashMap<>();
    for (Map.Entry<GlobalStreamId, Grouping> entry : this.getThisSources().entrySet()) {
        GlobalStreamId gid = entry.getKey();
        Map<String, Object> stringSourceMap = stringSources.get(gid.get_componentId());
        if (stringSourceMap == null) {
            stringSourceMap = new HashMap<>();
            stringSources.put(gid.get_componentId(), stringSourceMap);
        }
        stringSourceMap.put(gid.get_streamId(), groupingToJSONableMap(entry.getValue()));
    }
    obj.put("source->stream->grouping", stringSources);
    obj.put("source->stream->fields", this.getThisInputFields());
    return JSONValue.toJSONString(obj);
}

Example 20 with GlobalStreamId

use of org.apache.storm.generated.GlobalStreamId in project storm by apache.

the class GrouperFactory method mkGrouper.

public static LoadAwareCustomStreamGrouping mkGrouper(WorkerTopologyContext context, String componentId, String streamId, Fields outFields, Grouping thriftGrouping, List<Integer> unsortedTargetTasks, Map topoConf) {
    List<Integer> targetTasks = Ordering.natural().sortedCopy(unsortedTargetTasks);
    final boolean isNotLoadAware = (null != topoConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING) && (boolean) topoConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING));
    CustomStreamGrouping result = null;
    switch(Thrift.groupingType(thriftGrouping)) {
        case FIELDS:
            if (Thrift.isGlobalGrouping(thriftGrouping)) {
                result = new GlobalGrouper();
            } else {
                result = new FieldsGrouper(outFields, thriftGrouping);
            }
            break;
        case SHUFFLE:
            if (isNotLoadAware) {
                result = new ShuffleGrouping();
            } else {
                result = new LoadAwareShuffleGrouping();
            }
            break;
        case ALL:
            result = new AllGrouper();
            break;
        case LOCAL_OR_SHUFFLE:
            // Prefer local tasks as target tasks if possible
            Set<Integer> sameTasks = Sets.intersection(Sets.newHashSet(targetTasks), Sets.newHashSet(context.getThisWorkerTasks()));
            targetTasks = (sameTasks.isEmpty()) ? targetTasks : new ArrayList<>(sameTasks);
            if (isNotLoadAware) {
                result = new ShuffleGrouping();
            } else {
                result = new LoadAwareShuffleGrouping();
            }
            break;
        case NONE:
            result = new NoneGrouper();
            break;
        case CUSTOM_OBJECT:
            result = (CustomStreamGrouping) Thrift.instantiateJavaObject(thriftGrouping.get_custom_object());
            break;
        case CUSTOM_SERIALIZED:
            result = Utils.javaDeserialize(thriftGrouping.get_custom_serialized(), CustomStreamGrouping.class);
            break;
        case DIRECT:
            result = DIRECT;
            break;
        default:
            result = null;
            break;
    }
    if (null != result) {
        result.prepare(context, new GlobalStreamId(componentId, streamId), targetTasks);
    }
    if (result instanceof LoadAwareCustomStreamGrouping) {
        return (LoadAwareCustomStreamGrouping) result;
    } else {
        return new BasicLoadAwareCustomStreamGrouping(result);
    }
}