Examples with NimbusInfo org.apache.storm.nimbus.NimbusInfo used on opensource projects

Example 1 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class BlobStoreUtils method getNimbodesWithLatestSequenceNumberOfBlob.

// Check for latest sequence number of a key inside zookeeper and return nimbodes containing the latest sequence number
public static Set<NimbusInfo> getNimbodesWithLatestSequenceNumberOfBlob(CuratorFramework zkClient, String key) throws Exception {
    List<String> stateInfoList;
    try {
        stateInfoList = zkClient.getChildren().forPath("/blobstore/" + key);
    } catch (KeeperException.NoNodeException e) {
        // this should be thrown to the caller to indicate that the key is invalid now
        throw new KeyNotFoundException(key);
    }
    Set<NimbusInfo> nimbusInfoSet = new HashSet<NimbusInfo>();
    int latestSeqNumber = getLatestSequenceNumber(stateInfoList);
    LOG.debug("getNimbodesWithLatestSequenceNumberOfBlob stateInfo {} version {}", stateInfoList, latestSeqNumber);
    // Get the nimbodes with the latest version
    for (String state : stateInfoList) {
        BlobKeySequenceInfo sequenceInfo = normalizeNimbusHostPortSequenceNumberInfo(state);
        if (latestSeqNumber == Integer.parseInt(sequenceInfo.getSequenceNumber())) {
            nimbusInfoSet.add(NimbusInfo.parse(sequenceInfo.getNimbusHostPort()));
        }
    }
    LOG.debug("nimbusInfoList {}", nimbusInfoSet);
    return nimbusInfoSet;
}

Example 2 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class Nimbus method getClusterInfoImpl.

private ClusterSummary getClusterInfoImpl() throws Exception {
    IStormClusterState state = stormClusterState;
    Map<String, SupervisorInfo> infos = state.allSupervisorInfo();
    List<SupervisorSummary> summaries = new ArrayList<>(infos.size());
    for (Entry<String, SupervisorInfo> entry : infos.entrySet()) {
        summaries.add(makeSupervisorSummary(entry.getKey(), entry.getValue()));
    }
    int uptime = this.uptime.upTime();
    Map<String, StormBase> bases = state.topologyBases();
    List<NimbusSummary> nimbuses = state.nimbuses();
    //update the isLeader field for each nimbus summary
    NimbusInfo leader = leaderElector.getLeader();
    for (NimbusSummary nimbusSummary : nimbuses) {
        nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs()));
        nimbusSummary.set_isLeader(leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port());
    }
    List<TopologySummary> topologySummaries = new ArrayList<>();
    for (Entry<String, StormBase> entry : bases.entrySet()) {
        StormBase base = entry.getValue();
        if (base == null) {
            continue;
        }
        String topoId = entry.getKey();
        Assignment assignment = state.assignmentInfo(topoId, null);
        int numTasks = 0;
        int numExecutors = 0;
        int numWorkers = 0;
        if (assignment != null && assignment.is_set_executor_node_port()) {
            for (List<Long> ids : assignment.get_executor_node_port().keySet()) {
                numTasks += StormCommon.executorIdToTasks(ids).size();
            }
            numExecutors = assignment.get_executor_node_port_size();
            numWorkers = new HashSet<>(assignment.get_executor_node_port().values()).size();
        }
        TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, numExecutors, numWorkers, Time.deltaSecs(base.get_launch_time_secs()), extractStatusStr(base));
        if (base.is_set_owner()) {
            summary.set_owner(base.get_owner());
        }
        String status = idToSchedStatus.get().get(topoId);
        if (status != null) {
            summary.set_sched_status(status);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            summary.set_requested_memonheap(resources.getRequestedMemOnHeap());
            summary.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            summary.set_requested_cpu(resources.getRequestedCpu());
            summary.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            summary.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            summary.set_assigned_cpu(resources.getAssignedCpu());
        }
        summary.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        topologySummaries.add(summary);
    }
    ClusterSummary ret = new ClusterSummary(summaries, topologySummaries, nimbuses);
    ret.set_nimbus_uptime_secs(uptime);
    return ret;
}

Example 3 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class Nimbus method setupBlobstore.

/**
     * Sets up blobstore state for all current keys.
     * @throws KeyNotFoundException 
     * @throws AuthorizationException 
     */
private void setupBlobstore() throws AuthorizationException, KeyNotFoundException {
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Set<String> localKeys = new HashSet<>();
    for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
        localKeys.add(it.next());
    }
    Set<String> activeKeys = new HashSet<>(state.activeKeys());
    Set<String> activeLocalKeys = new HashSet<>(localKeys);
    activeLocalKeys.retainAll(activeKeys);
    Set<String> keysToDelete = new HashSet<>(localKeys);
    keysToDelete.removeAll(activeKeys);
    NimbusInfo nimbusInfo = nimbusHostPortInfo;
    LOG.debug("Deleting keys not on the zookeeper {}", keysToDelete);
    for (String toDelete : keysToDelete) {
        store.deleteBlob(toDelete, NIMBUS_SUBJECT);
    }
    LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", activeKeys, activeLocalKeys);
    for (String key : activeLocalKeys) {
        try {
            state.setupBlobstore(key, nimbusInfo, getVersionForKey(key, nimbusInfo, conf));
        } catch (KeyNotFoundException e) {
            // invalid key, remove it from blobstore
            store.deleteBlob(key, NIMBUS_SUBJECT);
        }
    }
}

Example 4 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class Nimbus method setupStormCode.

private void setupStormCode(Map<String, Object> conf, String topoId, String tmpJarLocation, Map<String, Object> topoConf, StormTopology topology) throws Exception {
    Subject subject = getSubject();
    IStormClusterState clusterState = stormClusterState;
    BlobStore store = blobStore;
    String jarKey = ConfigUtils.masterStormJarKey(topoId);
    String codeKey = ConfigUtils.masterStormCodeKey(topoId);
    String confKey = ConfigUtils.masterStormConfKey(topoId);
    NimbusInfo hostPortInfo = nimbusHostPortInfo;
    if (tmpJarLocation != null) {
        //in local mode there is no jar
        try (FileInputStream fin = new FileInputStream(tmpJarLocation)) {
            store.createBlob(jarKey, fin, new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
        }
        if (store instanceof LocalFsBlobStore) {
            clusterState.setupBlobstore(jarKey, hostPortInfo, getVersionForKey(jarKey, hostPortInfo, conf));
        }
    }
    store.createBlob(confKey, Utils.toCompressedJsonConf(topoConf), new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
    if (store instanceof LocalFsBlobStore) {
        clusterState.setupBlobstore(confKey, hostPortInfo, getVersionForKey(confKey, hostPortInfo, conf));
    }
    store.createBlob(codeKey, Utils.serialize(topology), new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
    if (store instanceof LocalFsBlobStore) {
        clusterState.setupBlobstore(codeKey, hostPortInfo, getVersionForKey(codeKey, hostPortInfo, conf));
    }
}

Example 5 with NimbusInfo

use of org.apache.storm.nimbus.NimbusInfo in project storm by apache.

the class Nimbus method blobSync.

private void blobSync() throws Exception {
    if ("distributed".equals(conf.get(Config.STORM_CLUSTER_MODE))) {
        if (!isLeader()) {
            IStormClusterState state = stormClusterState;
            NimbusInfo nimbusInfo = nimbusHostPortInfo;
            BlobStore store = blobStore;
            Set<String> allKeys = new HashSet<>();
            for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
                allKeys.add(it.next());
            }
            Set<String> zkKeys = new HashSet<>(state.blobstore(() -> {
                try {
                    this.blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }));
            LOG.debug("blob-sync blob-store-keys {} zookeeper-keys {}", allKeys, zkKeys);
            BlobSynchronizer sync = new BlobSynchronizer(store, conf);
            sync.setNimbusInfo(nimbusInfo);
            sync.setBlobStoreKeySet(allKeys);
            sync.setZookeeperKeySet(zkKeys);
            sync.syncBlobs();
        }
    //else not leader (NOOP)
    }
//else local (NOOP)
}