Examples with BlobStore org.apache.storm.blobstore.BlobStore used on opensource projects

Example 1 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method computeExecutorToComponent.

private Map<List<Integer>, String> computeExecutorToComponent(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
    BlobStore store = blobStore;
    List<List<Integer>> executors = computeExecutors(topoId, base);
    StormTopology topology = readStormTopologyAsNimbus(topoId, store);
    Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
    Map<Integer, String> taskToComponent = StormCommon.stormTaskInfo(topology, topoConf);
    Map<List<Integer>, String> ret = new HashMap<>();
    for (List<Integer> executor : executors) {
        ret.put(executor, taskToComponent.get(executor.get(0)));
    }
    return ret;
}

Example 2 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method setupBlobstore.

/**
     * Sets up blobstore state for all current keys.
     * @throws KeyNotFoundException 
     * @throws AuthorizationException 
     */
private void setupBlobstore() throws AuthorizationException, KeyNotFoundException {
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Set<String> localKeys = new HashSet<>();
    for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
        localKeys.add(it.next());
    }
    Set<String> activeKeys = new HashSet<>(state.activeKeys());
    Set<String> activeLocalKeys = new HashSet<>(localKeys);
    activeLocalKeys.retainAll(activeKeys);
    Set<String> keysToDelete = new HashSet<>(localKeys);
    keysToDelete.removeAll(activeKeys);
    NimbusInfo nimbusInfo = nimbusHostPortInfo;
    LOG.debug("Deleting keys not on the zookeeper {}", keysToDelete);
    for (String toDelete : keysToDelete) {
        store.deleteBlob(toDelete, NIMBUS_SUBJECT);
    }
    LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", activeKeys, activeLocalKeys);
    for (String key : activeLocalKeys) {
        try {
            state.setupBlobstore(key, nimbusInfo, getVersionForKey(key, nimbusInfo, conf));
        } catch (KeyNotFoundException e) {
            // invalid key, remove it from blobstore
            store.deleteBlob(key, NIMBUS_SUBJECT);
        }
    }
}

Example 3 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class Nimbus method blobSync.

private void blobSync() throws Exception {
    if ("distributed".equals(conf.get(Config.STORM_CLUSTER_MODE))) {
        if (!isLeader()) {
            IStormClusterState state = stormClusterState;
            NimbusInfo nimbusInfo = nimbusHostPortInfo;
            BlobStore store = blobStore;
            Set<String> allKeys = new HashSet<>();
            for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
                allKeys.add(it.next());
            }
            Set<String> zkKeys = new HashSet<>(state.blobstore(() -> {
                try {
                    this.blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }));
            LOG.debug("blob-sync blob-store-keys {} zookeeper-keys {}", allKeys, zkKeys);
            BlobSynchronizer sync = new BlobSynchronizer(store, conf);
            sync.setNimbusInfo(nimbusInfo);
            sync.setBlobStoreKeySet(allKeys);
            sync.setZookeeperKeySet(zkKeys);
            sync.syncBlobs();
        }
    //else not leader (NOOP)
    }
//else local (NOOP)
}

Example 4 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class BlobStoreTest method testWithAuthenticationUpdate.

@Test
void testWithAuthenticationUpdate() throws Exception {
    try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-update")) {
        BlobStore store = container.blobStore;
        Subject who = getSubject("test_subject");
        assertStoreHasExactly(store);
        SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
        try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
            out.write(1);
        }
        assertStoreHasExactly(store, "test");
        readAssertEqualsWithAuth(store, who, "test", 1);
        try (AtomicOutputStream out = store.updateBlob("test", who)) {
            out.write(2);
        }
        assertStoreHasExactly(store, "test");
        readAssertEqualsWithAuth(store, who, "test", 2);
        try (AtomicOutputStream out = store.updateBlob("test", who)) {
            out.write(3);
        }
        assertStoreHasExactly(store, "test");
        readAssertEqualsWithAuth(store, who, "test", 3);
        LOG.info("Deleting test");
        store.deleteBlob("test", who);
        assertStoreHasExactly(store);
    }
}

Example 5 with BlobStore

use of org.apache.storm.blobstore.BlobStore in project storm by apache.

the class BlobStoreTest method testWithAuthentication.

@ParameterizedTest
@EnumSource(value = AuthenticationTestSubject.class)
void testWithAuthentication(AuthenticationTestSubject testSubject) throws Exception {
    try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-" + testSubject.name())) {
        BlobStore store = container.blobStore;
        assertStoreHasExactly(store);
        SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
        try (AtomicOutputStream out = store.createBlob("test", metadata, testSubject.subject)) {
            assertStoreHasExactly(store, "test");
            out.write(1);
        }
        store.deleteBlob("test", testSubject.subject);
    }
}