Examples with IStormClusterState org.apache.storm.cluster.IStormClusterState used on opensource projects

Example 16 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method transition.

private void transition(String topoId, TopologyActions event, Object eventArg, boolean errorOnNoTransition) throws Exception {
    LOG.info("TRANSITION: {} {} {} {}", topoId, event, eventArg, errorOnNoTransition);
    assertIsLeader();
    synchronized (submitLock) {
        IStormClusterState clusterState = stormClusterState;
        StormBase base = clusterState.stormBase(topoId, null);
        if (base == null || base.get_status() == null) {
            LOG.info("Cannot apply event {} to {} because topology no longer exists", event, topoId);
        } else {
            TopologyStatus status = base.get_status();
            TopologyStateTransition transition = TOPO_STATE_TRANSITIONS.get(status).get(event);
            if (transition == null) {
                String message = "No transition for event: " + event + ", status: " + status + " storm-id: " + topoId;
                if (errorOnNoTransition) {
                    throw new RuntimeException(message);
                }
                if (TopologyActions.STARTUP != event) {
                    //STARTUP is a system event so don't log an issue
                    LOG.info(message);
                }
                transition = NOOP_TRANSITION;
            }
            StormBase updates = transition.transition(eventArg, this, topoId, base);
            if (updates != null) {
                clusterState.updateStorm(topoId, updates);
            }
        }
    }
}

Example 17 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method blobSync.

private void blobSync() throws Exception {
    if ("distributed".equals(conf.get(Config.STORM_CLUSTER_MODE))) {
        if (!isLeader()) {
            IStormClusterState state = stormClusterState;
            NimbusInfo nimbusInfo = nimbusHostPortInfo;
            BlobStore store = blobStore;
            Set<String> allKeys = new HashSet<>();
            for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
                allKeys.add(it.next());
            }
            Set<String> zkKeys = new HashSet<>(state.blobstore(() -> {
                try {
                    this.blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }));
            LOG.debug("blob-sync blob-store-keys {} zookeeper-keys {}", allKeys, zkKeys);
            BlobSynchronizer sync = new BlobSynchronizer(store, conf);
            sync.setNimbusInfo(nimbusInfo);
            sync.setBlobStoreKeySet(allKeys);
            sync.setZookeeperKeySet(zkKeys);
            sync.syncBlobs();
        }
    //else not leader (NOOP)
    }
//else local (NOOP)
}

Example 18 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method getTopologyInfoWithOpts.

@Override
public TopologyInfo getTopologyInfoWithOpts(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyInfoWithOptsCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
        if (common.base == null) {
            throw new NotAliveException(topoId);
        }
        IStormClusterState state = stormClusterState;
        NumErrorsChoice numErrChoice = OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
        Map<String, List<ErrorInfo>> errors = new HashMap<>();
        for (String component : common.allComponents) {
            switch(numErrChoice) {
                case NONE:
                    errors.put(component, Collections.emptyList());
                    break;
                case ONE:
                    List<ErrorInfo> errList = new ArrayList<>();
                    ErrorInfo info = state.lastError(topoId, component);
                    if (info != null) {
                        errList.add(info);
                    }
                    errors.put(component, errList);
                    break;
                case ALL:
                    errors.put(component, state.errors(topoId, component));
                    break;
                default:
                    LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
                    errors.put(component, state.errors(topoId, component));
                    break;
            }
        }
        List<ExecutorSummary> summaries = new ArrayList<>();
        if (common.assignment != null) {
            for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
                NodeInfo ni = entry.getValue();
                ExecutorInfo execInfo = toExecInfo(entry.getKey());
                String host = entry.getValue().get_node();
                Map<String, Object> heartbeat = common.beats.get(StatsUtil.convertExecutor(entry.getKey()));
                if (heartbeat == null) {
                    heartbeat = Collections.emptyMap();
                }
                ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), ni.get_node(), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
                //heartbeats "stats"
                Map<String, Object> hb = (Map<String, Object>) heartbeat.get("heartbeat");
                if (hb != null) {
                    Map ex = (Map) hb.get("stats");
                    if (ex != null) {
                        ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
                        summ.set_stats(stats);
                    }
                }
                summaries.add(summ);
            }
        }
        TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
        if (common.base.is_set_owner()) {
            topoInfo.set_owner(common.base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, common.base);
        if (resources != null) {
            topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoInfo.set_requested_cpu(resources.getRequestedCpu());
            topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        if (common.base.is_set_component_debug()) {
            topoInfo.set_component_debug(common.base.get_component_debug());
        }
        topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        return topoInfo;
    } catch (Exception e) {
        LOG.warn("Get topo info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 19 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method createStateInZookeeper.

@Override
public void createStateInZookeeper(String key) throws TException {
    try {
        IStormClusterState state = stormClusterState;
        BlobStore store = blobStore;
        NimbusInfo ni = nimbusHostPortInfo;
        if (store instanceof LocalFsBlobStore) {
            state.setupBlobstore(key, ni, getVersionForKey(key, ni, conf));
        }
        LOG.debug("Created state in zookeeper {} {} {}", state, store, ni);
    } catch (Exception e) {
        LOG.warn("Exception while creating state in zookeeper - key: " + key, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 20 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Worker method mkBackpressureHandler.

/**
     * make a handler that checks and updates worker's backpressure flag
     */
private WorkerBackpressureCallback mkBackpressureHandler() {
    final List<IRunningExecutor> executors = executorsAtom.get();
    return new WorkerBackpressureCallback() {

        @Override
        public void onEvent(Object obj) {
            String topologyId = workerState.topologyId;
            String assignmentId = workerState.assignmentId;
            int port = workerState.port;
            IStormClusterState stormClusterState = workerState.stormClusterState;
            boolean prevBackpressureFlag = workerState.backpressure.get();
            boolean currBackpressureFlag = prevBackpressureFlag;
            if (null != executors) {
                currBackpressureFlag = workerState.transferQueue.getThrottleOn() || (executors.stream().map(IRunningExecutor::getBackPressureFlag).reduce((op1, op2) -> (op1 || op2)).get());
            }
            if (currBackpressureFlag != prevBackpressureFlag) {
                try {
                    LOG.debug("worker backpressure flag changing from {} to {}", prevBackpressureFlag, currBackpressureFlag);
                    stormClusterState.workerBackpressure(topologyId, assignmentId, (long) port, currBackpressureFlag);
                    // doing the local reset after the zk update succeeds is very important to avoid a bad state upon zk exception
                    workerState.backpressure.set(currBackpressureFlag);
                } catch (Exception ex) {
                    LOG.error("workerBackpressure update failed when connecting to ZK ... will retry", ex);
                }
            }
        }
    };
}