Examples with BlobStore org.apache.storm.blobstore.BlobStore used on opensource projects

Example 11 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method getCommonTopoInfo.

private CommonTopoInfo getCommonTopoInfo(String topoId, String operation) throws NotAliveException, AuthorizationException, IOException, InvalidTopologyException {
    BlobStore store = blobStore;
    IStormClusterState state = stormClusterState;
    CommonTopoInfo ret = new CommonTopoInfo();
    ret.topoConf = tryReadTopoConf(topoId, store);
    ret.topoName = (String) ret.topoConf.get(Config.TOPOLOGY_NAME);
    checkAuthorization(ret.topoName, ret.topoConf, operation);
    ret.topology = tryReadTopology(topoId, store);
    ret.taskToComponent = StormCommon.stormTaskInfo(ret.topology, ret.topoConf);
    ret.base = state.stormBase(topoId, null);
    if (ret.base != null && ret.base.is_set_launch_time_secs()) {
        ret.launchTimeSecs = ret.base.get_launch_time_secs();
    } else {
        ret.launchTimeSecs = 0;
    }
    ret.assignment = state.assignmentInfo(topoId, null);
    ret.beats = OR(heartbeatsCache.get().get(topoId), Collections.<List<Integer>, Map<String, Object>>emptyMap());
    ret.allComponents = new HashSet<>(ret.taskToComponent.values());
    return ret;
}

Example 12 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method getTopologyHistory.

@SuppressWarnings("unchecked")
@Override
public TopologyHistoryInfo getTopologyHistory(String user) throws AuthorizationException, TException {
    try {
        List<String> adminUsers = (List<String>) conf.getOrDefault(Config.NIMBUS_ADMINS, Collections.emptyList());
        IStormClusterState state = stormClusterState;
        BlobStore store = blobStore;
        List<String> assignedIds = state.assignments(null);
        Set<String> ret = new HashSet<>();
        boolean isAdmin = adminUsers.contains(user);
        for (String topoId : assignedIds) {
            Map<String, Object> topoConf = tryReadTopoConf(topoId, store);
            List<String> groups = ConfigUtils.getTopoLogsGroups(topoConf);
            List<String> topoLogUsers = ConfigUtils.getTopoLogsUsers(topoConf);
            if (user == null || isAdmin || isUserPartOf(user, groups) || topoLogUsers.contains(user)) {
                ret.add(topoId);
            }
        }
        ret.addAll(readTopologyHistory(user, adminUsers));
        return new TopologyHistoryInfo(new ArrayList<>(ret));
    } catch (Exception e) {
        LOG.warn("Get topology history. (user='{}')", user, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 13 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method launchServer.

@VisibleForTesting
public void launchServer() throws Exception {
    try {
        BlobStore store = blobStore;
        IStormClusterState state = stormClusterState;
        NimbusInfo hpi = nimbusHostPortInfo;
        LOG.info("Starting Nimbus with conf {}", conf);
        validator.prepare(conf);
        //add to nimbuses
        state.addNimbusHost(hpi.getHost(), new NimbusSummary(hpi.getHost(), hpi.getPort(), Time.currentTimeSecs(), false, STORM_VERSION));
        leaderElector.addToLeaderLockQueue();
        if (store instanceof LocalFsBlobStore) {
            //register call back for blob-store
            state.blobstore(() -> {
                try {
                    blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
            setupBlobstore();
        }
        for (ClusterMetricsConsumerExecutor exec : clusterConsumerExceutors) {
            exec.prepare();
        }
        if (isLeader()) {
            for (String topoId : state.activeStorms()) {
                transition(topoId, TopologyActions.STARTUP, null);
            }
        }
        final boolean doNotReassign = (Boolean) conf.getOrDefault(ConfigUtils.NIMBUS_DO_NOT_REASSIGN, false);
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_MONITOR_FREQ_SECS)), () -> {
            try {
                if (!doNotReassign) {
                    mkAssignments();
                }
                doCleanup();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Schedule Nimbus inbox cleaner
        final int jarExpSecs = Utils.getInt(conf.get(Config.NIMBUS_INBOX_JAR_EXPIRATION_SECS));
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CLEANUP_INBOX_FREQ_SECS)), () -> {
            try {
                cleanInbox(getInbox(), jarExpSecs);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        //Schedule nimbus code sync thread to sync code from other nimbuses.
        if (store instanceof LocalFsBlobStore) {
            timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CODE_SYNC_FREQ_SECS)), () -> {
                try {
                    blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
        // Schedule topology history cleaner
        Integer interval = Utils.getInt(conf.get(Config.LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
        if (interval != null) {
            final int lvCleanupAgeMins = Utils.getInt(conf.get(Config.LOGVIEWER_CLEANUP_AGE_MINS));
            timer.scheduleRecurring(0, interval, () -> {
                try {
                    cleanTopologyHistory(lvCleanupAgeMins);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CREDENTIAL_RENEW_FREQ_SECS)), () -> {
            try {
                renewCredentials();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        StormMetricsRegistry.registerGauge("nimbus:num-supervisors", () -> state.supervisors(null));
        StormMetricsRegistry.startMetricsReporters(conf);
        if (clusterConsumerExceutors != null) {
            timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.STORM_CLUSTER_METRICS_CONSUMER_PUBLISH_INTERVAL_SECS)), () -> {
                try {
                    if (isLeader()) {
                        sendClusterMetricsToExecutors();
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
    } catch (Exception e) {
        if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e)) {
            throw e;
        }
        if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, e)) {
            throw e;
        }
        LOG.error("Error on initialization of nimbus", e);
        Utils.exitProcess(13, "Error on initialization of nimbus");
    }
}

Example 14 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method rmDependencyJarsInTopology.

@VisibleForTesting
public void rmDependencyJarsInTopology(String topoId) {
    try {
        BlobStore store = blobStore;
        IStormClusterState state = stormClusterState;
        StormTopology topo = readStormTopologyAsNimbus(topoId, store);
        List<String> dependencyJars = topo.get_dependency_jars();
        LOG.info("Removing dependency jars from blobs - {}", dependencyJars);
        if (dependencyJars != null && !dependencyJars.isEmpty()) {
            for (String key : dependencyJars) {
                rmBlobKey(store, key, state);
            }
        }
    } catch (Exception e) {
        //Yes eat the exception
        LOG.info("Exception {}", e);
    }
}

Example 15 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method renewCredentials.

private void renewCredentials() throws Exception {
    if (!isLeader()) {
        LOG.info("not a leader, skipping credential renewal.");
        return;
    }
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Collection<ICredentialsRenewer> renewers = credRenewers;
    Object lock = credUpdateLock;
    List<String> assignedIds = state.activeStorms();
    if (assignedIds != null) {
        for (String id : assignedIds) {
            Map<String, Object> topoConf = Collections.unmodifiableMap(tryReadTopoConf(id, store));
            synchronized (lock) {
                Credentials origCreds = state.credentials(id, null);
                if (origCreds != null) {
                    Map<String, String> orig = origCreds.get_creds();
                    Map<String, String> newCreds = new HashMap<>(orig);
                    for (ICredentialsRenewer renewer : renewers) {
                        LOG.info("Renewing Creds For {} with {}", id, renewer);
                        renewer.renew(newCreds, topoConf);
                    }
                    if (!newCreds.equals(origCreds)) {
                        state.setCredentials(id, new Credentials(newCreds), topoConf);
                    }
                }
            }
        }
    }
}