Examples with LocalFsBlobStore org.apache.storm.blobstore.LocalFsBlobStore used on opensource projects

Example 1 with LocalFsBlobStore

use of org.apache.storm.blobstore.LocalFsBlobStore in project storm by apache.

the class Nimbus method setupStormCode.

private void setupStormCode(Map<String, Object> conf, String topoId, String tmpJarLocation, Map<String, Object> topoConf, StormTopology topology) throws Exception {
    Subject subject = getSubject();
    IStormClusterState clusterState = stormClusterState;
    BlobStore store = blobStore;
    String jarKey = ConfigUtils.masterStormJarKey(topoId);
    String codeKey = ConfigUtils.masterStormCodeKey(topoId);
    String confKey = ConfigUtils.masterStormConfKey(topoId);
    NimbusInfo hostPortInfo = nimbusHostPortInfo;
    if (tmpJarLocation != null) {
        //in local mode there is no jar
        try (FileInputStream fin = new FileInputStream(tmpJarLocation)) {
            store.createBlob(jarKey, fin, new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
        }
        if (store instanceof LocalFsBlobStore) {
            clusterState.setupBlobstore(jarKey, hostPortInfo, getVersionForKey(jarKey, hostPortInfo, conf));
        }
    }
    store.createBlob(confKey, Utils.toCompressedJsonConf(topoConf), new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
    if (store instanceof LocalFsBlobStore) {
        clusterState.setupBlobstore(confKey, hostPortInfo, getVersionForKey(confKey, hostPortInfo, conf));
    }
    store.createBlob(codeKey, Utils.serialize(topology), new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
    if (store instanceof LocalFsBlobStore) {
        clusterState.setupBlobstore(codeKey, hostPortInfo, getVersionForKey(codeKey, hostPortInfo, conf));
    }
}

Example 2 with LocalFsBlobStore

use of org.apache.storm.blobstore.LocalFsBlobStore in project storm by apache.

the class Nimbus method createStateInZookeeper.

@Override
public void createStateInZookeeper(String key) throws TException {
    try {
        IStormClusterState state = stormClusterState;
        BlobStore store = blobStore;
        NimbusInfo ni = nimbusHostPortInfo;
        if (store instanceof LocalFsBlobStore) {
            state.setupBlobstore(key, ni, getVersionForKey(key, ni, conf));
        }
        LOG.debug("Created state in zookeeper {} {} {}", state, store, ni);
    } catch (Exception e) {
        LOG.warn("Exception while creating state in zookeeper - key: " + key, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 3 with LocalFsBlobStore

use of org.apache.storm.blobstore.LocalFsBlobStore in project storm by apache.

the class Nimbus method launchServer.

@VisibleForTesting
public void launchServer() throws Exception {
    try {
        BlobStore store = blobStore;
        IStormClusterState state = stormClusterState;
        NimbusInfo hpi = nimbusHostPortInfo;
        LOG.info("Starting Nimbus with conf {}", conf);
        validator.prepare(conf);
        //add to nimbuses
        state.addNimbusHost(hpi.getHost(), new NimbusSummary(hpi.getHost(), hpi.getPort(), Time.currentTimeSecs(), false, STORM_VERSION));
        leaderElector.addToLeaderLockQueue();
        if (store instanceof LocalFsBlobStore) {
            //register call back for blob-store
            state.blobstore(() -> {
                try {
                    blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
            setupBlobstore();
        }
        for (ClusterMetricsConsumerExecutor exec : clusterConsumerExceutors) {
            exec.prepare();
        }
        if (isLeader()) {
            for (String topoId : state.activeStorms()) {
                transition(topoId, TopologyActions.STARTUP, null);
            }
        }
        final boolean doNotReassign = (Boolean) conf.getOrDefault(ConfigUtils.NIMBUS_DO_NOT_REASSIGN, false);
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_MONITOR_FREQ_SECS)), () -> {
            try {
                if (!doNotReassign) {
                    mkAssignments();
                }
                doCleanup();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Schedule Nimbus inbox cleaner
        final int jarExpSecs = Utils.getInt(conf.get(Config.NIMBUS_INBOX_JAR_EXPIRATION_SECS));
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CLEANUP_INBOX_FREQ_SECS)), () -> {
            try {
                cleanInbox(getInbox(), jarExpSecs);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        //Schedule nimbus code sync thread to sync code from other nimbuses.
        if (store instanceof LocalFsBlobStore) {
            timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CODE_SYNC_FREQ_SECS)), () -> {
                try {
                    blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
        // Schedule topology history cleaner
        Integer interval = Utils.getInt(conf.get(Config.LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
        if (interval != null) {
            final int lvCleanupAgeMins = Utils.getInt(conf.get(Config.LOGVIEWER_CLEANUP_AGE_MINS));
            timer.scheduleRecurring(0, interval, () -> {
                try {
                    cleanTopologyHistory(lvCleanupAgeMins);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CREDENTIAL_RENEW_FREQ_SECS)), () -> {
            try {
                renewCredentials();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        StormMetricsRegistry.registerGauge("nimbus:num-supervisors", () -> state.supervisors(null));
        StormMetricsRegistry.startMetricsReporters(conf);
        if (clusterConsumerExceutors != null) {
            timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.STORM_CLUSTER_METRICS_CONSUMER_PUBLISH_INTERVAL_SECS)), () -> {
                try {
                    if (isLeader()) {
                        sendClusterMetricsToExecutors();
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
    } catch (Exception e) {
        if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e)) {
            throw e;
        }
        if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, e)) {
            throw e;
        }
        LOG.error("Error on initialization of nimbus", e);
        Utils.exitProcess(13, "Error on initialization of nimbus");
    }
}

Example 4 with LocalFsBlobStore

use of org.apache.storm.blobstore.LocalFsBlobStore in project storm by apache.

the class LocalizerTest method testKeyNotFoundException.

@Test(expected = KeyNotFoundException.class)
public void testKeyNotFoundException() throws Exception {
    Map conf = Utils.readStormConfig();
    String key1 = "key1";
    conf.put(Config.STORM_LOCAL_DIR, "target");
    LocalFsBlobStore bs = new LocalFsBlobStore();
    LocalFsBlobStore spy = spy(bs);
    Mockito.doReturn(true).when(spy).checkForBlobOrDownload(key1);
    Mockito.doNothing().when(spy).checkForBlobUpdate(key1);
    spy.prepare(conf, null, null);
    spy.getBlob(key1, null);
}