Search in sources :

Example 31 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class Nimbus method getComponentPageInfo.

@Override
public ComponentPageInfo getComponentPageInfo(String topoId, String componentId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getComponentPageInfoCalls.mark();
        CommonTopoInfo info = getCommonTopoInfo(topoId, "getComponentPageInfo");
        if (info.base == null) {
            throw new NotAliveException(topoId);
        }
        StormTopology topology = info.topology;
        Map<String, Object> topoConf = info.topoConf;
        Assignment assignment = info.assignment;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        Map<String, String> nodeToHost;
        Map<List<Long>, List<Object>> exec2HostPort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                List<Object> hostPort = Arrays.asList(nodeToHost.get(ni.get_node()), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
                exec2HostPort.put(entry.getKey(), hostPort);
            }
        } else {
            nodeToHost = Collections.emptyMap();
        }
        ComponentPageInfo compPageInfo = StatsUtil.aggCompExecsStats(exec2HostPort, info.taskToComponent, info.beats, window, includeSys, topoId, topology, componentId);
        if (compPageInfo.get_component_type() == ComponentType.SPOUT) {
            compPageInfo.set_resources_map(setResourcesDefaultIfNotSet(ResourceUtils.getSpoutsResources(topology, topoConf), componentId, topoConf));
        } else {
            //bolt
            compPageInfo.set_resources_map(setResourcesDefaultIfNotSet(ResourceUtils.getBoltsResources(topology, topoConf), componentId, topoConf));
        }
        compPageInfo.set_topology_name(info.topoName);
        compPageInfo.set_errors(stormClusterState.errors(topoId, componentId));
        compPageInfo.set_topology_status(extractStatusStr(info.base));
        if (info.base.is_set_component_debug()) {
            DebugOptions debug = info.base.get_component_debug().get(componentId);
            if (debug != null) {
                compPageInfo.set_debug_options(debug);
            }
        }
        // Add the event logger details.
        Map<String, List<Integer>> compToTasks = Utils.reverseMap(info.taskToComponent);
        if (compToTasks.containsKey(StormCommon.EVENTLOGGER_COMPONENT_ID)) {
            List<Integer> tasks = compToTasks.get(StormCommon.EVENTLOGGER_COMPONENT_ID);
            tasks.sort(null);
            // Find the task the events from this component route to.
            int taskIndex = TupleUtils.chooseTaskIndex(Collections.singletonList(componentId), tasks.size());
            int taskId = tasks.get(taskIndex);
            String host = null;
            Integer port = null;
            for (Entry<List<Long>, List<Object>> entry : exec2HostPort.entrySet()) {
                int start = entry.getKey().get(0).intValue();
                int end = entry.getKey().get(1).intValue();
                if (taskId >= start && taskId <= end) {
                    host = (String) entry.getValue().get(0);
                    port = ((Number) entry.getValue().get(1)).intValue();
                    break;
                }
            }
            if (host != null && port != null) {
                compPageInfo.set_eventlog_host(host);
                compPageInfo.set_eventlog_port(port);
            }
        }
        return compPageInfo;
    } catch (Exception e) {
        LOG.warn("getComponentPageInfo exception. (topo id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}
Also used : TException(org.apache.thrift.TException) HashMap(java.util.HashMap) StormTopology(org.apache.storm.generated.StormTopology) DebugOptions(org.apache.storm.generated.DebugOptions) Assignment(org.apache.storm.generated.Assignment) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) NotAliveException(org.apache.storm.generated.NotAliveException) ArrayList(java.util.ArrayList) List(java.util.List) DataPoint(org.apache.storm.metric.api.DataPoint) AuthorizationException(org.apache.storm.generated.AuthorizationException) NotAliveException(org.apache.storm.generated.NotAliveException) InterruptedIOException(java.io.InterruptedIOException) TException(org.apache.thrift.TException) IOException(java.io.IOException) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) KeyAlreadyExistsException(org.apache.storm.generated.KeyAlreadyExistsException) KeyNotFoundException(org.apache.storm.generated.KeyNotFoundException) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) BindException(java.net.BindException) ComponentPageInfo(org.apache.storm.generated.ComponentPageInfo) NodeInfo(org.apache.storm.generated.NodeInfo)

Example 32 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class Nimbus method readTopologyDetails.

private TopologyDetails readTopologyDetails(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
    assert (base != null);
    assert (topoId != null);
    BlobStore store = blobStore;
    Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
    StormTopology topo = readStormTopologyAsNimbus(topoId, store);
    Map<List<Integer>, String> rawExecToComponent = computeExecutorToComponent(topoId, base);
    Map<ExecutorDetails, String> executorsToComponent = new HashMap<>();
    for (Entry<List<Integer>, String> entry : rawExecToComponent.entrySet()) {
        List<Integer> execs = entry.getKey();
        ExecutorDetails execDetails = new ExecutorDetails(execs.get(0), execs.get(1));
        executorsToComponent.put(execDetails, entry.getValue());
    }
    return new TopologyDetails(topoId, topoConf, topo, base.get_num_workers(), executorsToComponent, base.get_launch_time_secs());
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) StormTopology(org.apache.storm.generated.StormTopology) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) ArrayList(java.util.ArrayList) List(java.util.List) BlobStore(org.apache.storm.blobstore.BlobStore) LocalFsBlobStore(org.apache.storm.blobstore.LocalFsBlobStore)

Example 33 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class Nimbus method getTopologyPageInfo.

@Override
public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyPageInfoCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyPageInfo");
        String topoName = common.topoName;
        IStormClusterState state = stormClusterState;
        int launchTimeSecs = common.launchTimeSecs;
        Assignment assignment = common.assignment;
        Map<List<Integer>, Map<String, Object>> beats = common.beats;
        Map<Integer, String> taskToComp = common.taskToComponent;
        StormTopology topology = common.topology;
        Map<String, Object> topoConf = common.topoConf;
        StormBase base = common.base;
        if (base == null) {
            throw new NotAliveException(topoId);
        }
        Map<WorkerSlot, WorkerResources> workerToResources = getWorkerResourcesForTopology(topoId);
        List<WorkerSummary> workerSummaries = null;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            Map<String, String> nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
            }
            workerSummaries = StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerToResources, includeSys, //this is the topology page, so we know the user is authorized
            true);
        }
        TopologyPageInfo topoPageInfo = StatsUtil.aggTopoExecsStats(topoId, exec2NodePort, taskToComp, beats, topology, window, includeSys, state);
        Map<String, Map<String, Double>> spoutResources = ResourceUtils.getSpoutsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_spout_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(spoutResources, entry.getKey(), topoConf));
        }
        Map<String, Map<String, Double>> boltResources = ResourceUtils.getBoltsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_bolt_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(boltResources, entry.getKey(), topoConf));
        }
        if (workerSummaries != null) {
            topoPageInfo.set_workers(workerSummaries);
        }
        if (base.is_set_owner()) {
            topoPageInfo.set_owner(base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoPageInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            topoPageInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoPageInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoPageInfo.set_requested_cpu(resources.getRequestedCpu());
            topoPageInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoPageInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoPageInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        topoPageInfo.set_name(topoName);
        topoPageInfo.set_status(extractStatusStr(base));
        topoPageInfo.set_uptime_secs(Time.deltaSecs(launchTimeSecs));
        topoPageInfo.set_topology_conf(JSONValue.toJSONString(topoConf));
        topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        if (base.is_set_component_debug()) {
            DebugOptions debug = base.get_component_debug().get(topoId);
            if (debug != null) {
                topoPageInfo.set_debug_options(debug);
            }
        }
        return topoPageInfo;
    } catch (Exception e) {
        LOG.warn("Get topo page info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}
Also used : TException(org.apache.thrift.TException) HashMap(java.util.HashMap) StormTopology(org.apache.storm.generated.StormTopology) StormBase(org.apache.storm.generated.StormBase) DebugOptions(org.apache.storm.generated.DebugOptions) Assignment(org.apache.storm.generated.Assignment) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) NotAliveException(org.apache.storm.generated.NotAliveException) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) ArrayList(java.util.ArrayList) List(java.util.List) IStormClusterState(org.apache.storm.cluster.IStormClusterState) CommonAggregateStats(org.apache.storm.generated.CommonAggregateStats) WorkerResources(org.apache.storm.generated.WorkerResources) ComponentAggregateStats(org.apache.storm.generated.ComponentAggregateStats) TopologyPageInfo(org.apache.storm.generated.TopologyPageInfo) DataPoint(org.apache.storm.metric.api.DataPoint) AuthorizationException(org.apache.storm.generated.AuthorizationException) NotAliveException(org.apache.storm.generated.NotAliveException) InterruptedIOException(java.io.InterruptedIOException) TException(org.apache.thrift.TException) IOException(java.io.IOException) AlreadyAliveException(org.apache.storm.generated.AlreadyAliveException) KeyAlreadyExistsException(org.apache.storm.generated.KeyAlreadyExistsException) KeyNotFoundException(org.apache.storm.generated.KeyNotFoundException) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) BindException(java.net.BindException) WorkerSummary(org.apache.storm.generated.WorkerSummary) NodeInfo(org.apache.storm.generated.NodeInfo) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 34 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class Task method mkTaskObject.

private Object mkTaskObject() {
    StormTopology topology = systemTopologyContext.getRawTopology();
    Map<String, SpoutSpec> spouts = topology.get_spouts();
    Map<String, Bolt> bolts = topology.get_bolts();
    Map<String, StateSpoutSpec> stateSpouts = topology.get_state_spouts();
    Object result;
    ComponentObject componentObject;
    if (spouts.containsKey(componentId)) {
        componentObject = spouts.get(componentId).get_spout_object();
    } else if (bolts.containsKey(componentId)) {
        componentObject = bolts.get(componentId).get_bolt_object();
    } else if (stateSpouts.containsKey(componentId)) {
        componentObject = stateSpouts.get(componentId).get_state_spout_object();
    } else {
        throw new RuntimeException("Could not find " + componentId + " in " + topology);
    }
    result = Utils.getSetComponentObject(componentObject);
    if (result instanceof ShellComponent) {
        if (spouts.containsKey(componentId)) {
            result = new ShellSpout((ShellComponent) result);
        } else {
            result = new ShellBolt((ShellComponent) result);
        }
    }
    if (result instanceof JavaObject) {
        result = Thrift.instantiateJavaObject((JavaObject) result);
    }
    return result;
}
Also used : StormTopology(org.apache.storm.generated.StormTopology) ShellSpout(org.apache.storm.spout.ShellSpout) Bolt(org.apache.storm.generated.Bolt) ShellBolt(org.apache.storm.task.ShellBolt) StateSpoutSpec(org.apache.storm.generated.StateSpoutSpec) StateSpoutSpec(org.apache.storm.generated.StateSpoutSpec) SpoutSpec(org.apache.storm.generated.SpoutSpec) JavaObject(org.apache.storm.generated.JavaObject) ComponentObject(org.apache.storm.generated.ComponentObject) ShellBolt(org.apache.storm.task.ShellBolt) JavaObject(org.apache.storm.generated.JavaObject) ComponentObject(org.apache.storm.generated.ComponentObject) ShellComponent(org.apache.storm.generated.ShellComponent)

Example 35 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class StormCommon method makeWorkerContext.

@SuppressWarnings("unchecked")
public static WorkerTopologyContext makeWorkerContext(Map<String, Object> workerData) {
    try {
        StormTopology stormTopology = (StormTopology) workerData.get(Constants.SYSTEM_TOPOLOGY);
        Map stormConf = (Map) workerData.get(Constants.STORM_CONF);
        Map<Integer, String> taskToComponent = (Map<Integer, String>) workerData.get(Constants.TASK_TO_COMPONENT);
        Map<String, List<Integer>> componentToSortedTasks = (Map<String, List<Integer>>) workerData.get(Constants.COMPONENT_TO_SORTED_TASKS);
        Map<String, Map<String, Fields>> componentToStreamToFields = (Map<String, Map<String, Fields>>) workerData.get(Constants.COMPONENT_TO_STREAM_TO_FIELDS);
        String stormId = (String) workerData.get(Constants.STORM_ID);
        Map conf = (Map) workerData.get(Constants.CONF);
        Integer port = (Integer) workerData.get(Constants.PORT);
        String codeDir = ConfigUtils.supervisorStormResourcesPath(ConfigUtils.supervisorStormDistRoot(conf, stormId));
        String pidDir = ConfigUtils.workerPidsRoot(conf, stormId);
        List<Integer> workerTasks = (List<Integer>) workerData.get(Constants.TASK_IDS);
        Map<String, Object> defaultResources = (Map<String, Object>) workerData.get(Constants.DEFAULT_SHARED_RESOURCES);
        Map<String, Object> userResources = (Map<String, Object>) workerData.get(Constants.USER_SHARED_RESOURCES);
        return new WorkerTopologyContext(stormTopology, stormConf, taskToComponent, componentToSortedTasks, componentToStreamToFields, stormId, codeDir, pidDir, port, workerTasks, defaultResources, userResources);
    } catch (IOException e) {
        throw Utils.wrapInRuntime(e);
    }
}
Also used : StormTopology(org.apache.storm.generated.StormTopology) IOException(java.io.IOException) Fields(org.apache.storm.tuple.Fields) WorkerTopologyContext(org.apache.storm.task.WorkerTopologyContext) ArrayList(java.util.ArrayList) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) TreeMap(java.util.TreeMap)

Aggregations

StormTopology (org.apache.storm.generated.StormTopology)76 Test (org.junit.Test)45 Config (org.apache.storm.Config)38 HashMap (java.util.HashMap)31 ExecutionContext (org.apache.storm.flux.model.ExecutionContext)20 TopologyDef (org.apache.storm.flux.model.TopologyDef)20 Map (java.util.Map)17 ArrayList (java.util.ArrayList)14 Bolt (org.apache.storm.generated.Bolt)13 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)13 List (java.util.List)11 GlobalStreamId (org.apache.storm.generated.GlobalStreamId)11 Grouping (org.apache.storm.generated.Grouping)10 SpoutSpec (org.apache.storm.generated.SpoutSpec)10 TopologyDetails (org.apache.storm.scheduler.TopologyDetails)10 NullStruct (org.apache.storm.generated.NullStruct)9 INimbus (org.apache.storm.scheduler.INimbus)9 IRichBolt (org.apache.storm.topology.IRichBolt)9 Cluster (org.apache.storm.scheduler.Cluster)8 ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)8