Examples with ClientBlobStore org.apache.storm.blobstore.ClientBlobStore used on opensource projects

Example 1 with ClientBlobStore

use of org.apache.storm.blobstore.ClientBlobStore in project storm by apache.

the class Blobstore method setAclCli.

private static void setAclCli(String[] args) throws Exception {
    Map<String, Object> cl = CLI.opt("s", "set", Collections.emptyList(), new AsAclParser()).arg("key", CLI.FIRST_WINS).parse(args);
    final String key = (String) cl.get("key");
    final List<AccessControl> setAcl = (List<AccessControl>) cl.get("s");
    ClientBlobStore.withConfiguredClient(new ClientBlobStore.WithBlobstore() {

        @Override
        public void run(ClientBlobStore blobStore) throws Exception {
            ReadableBlobMeta meta = blobStore.getBlobMeta(key);
            List<AccessControl> acl = meta.get_settable().get_acl();
            List<AccessControl> newAcl;
            if (setAcl != null && !setAcl.isEmpty()) {
                newAcl = setAcl;
            } else {
                newAcl = acl;
            }
            SettableBlobMeta newMeta = new SettableBlobMeta(newAcl);
            LOG.info("Setting ACL for {} to {}", key, generateAccessControlsInfo(newAcl));
            blobStore.setBlobMeta(key, newMeta);
        }
    });
}

Example 2 with ClientBlobStore

use of org.apache.storm.blobstore.ClientBlobStore in project storm by apache.

the class Localizer method updateBlobs.

/**
   * This function updates blobs on the supervisor. It uses a separate thread pool and runs
   * asynchronously of the download and delete.
   */
public List<LocalizedResource> updateBlobs(List<LocalResource> localResources, String user) throws AuthorizationException, KeyNotFoundException, IOException {
    LocalizedResourceSet lrsrcSet = _userRsrc.get(user);
    ArrayList<LocalizedResource> results = new ArrayList<>();
    ArrayList<Callable<LocalizedResource>> updates = new ArrayList<>();
    if (lrsrcSet == null) {
        // resource set must have been removed
        return results;
    }
    ClientBlobStore blobstore = null;
    try {
        blobstore = getClientBlobStore();
        for (LocalResource localResource : localResources) {
            String key = localResource.getBlobName();
            LocalizedResource lrsrc = lrsrcSet.get(key, localResource.shouldUncompress());
            if (lrsrc == null) {
                LOG.warn("blob requested for update doesn't exist: {}", key);
                continue;
            } else {
                // update it if either the version isn't the latest or if any local blob files are missing
                if (!isLocalizedResourceUpToDate(lrsrc, blobstore) || !isLocalizedResourceDownloaded(lrsrc)) {
                    LOG.debug("updating blob: {}", key);
                    updates.add(new DownloadBlob(this, _conf, key, new File(lrsrc.getFilePath()), user, lrsrc.isUncompressed(), true));
                }
            }
        }
    } finally {
        if (blobstore != null) {
            blobstore.shutdown();
        }
    }
    try {
        List<Future<LocalizedResource>> futures = _updateExecService.invokeAll(updates);
        for (Future<LocalizedResource> futureRsrc : futures) {
            try {
                LocalizedResource lrsrc = futureRsrc.get();
                // put the resource just in case it was removed at same time by the cleaner
                LocalizedResourceSet newSet = new LocalizedResourceSet(user);
                LocalizedResourceSet newlrsrcSet = _userRsrc.putIfAbsent(user, newSet);
                if (newlrsrcSet == null) {
                    newlrsrcSet = newSet;
                }
                newlrsrcSet.putIfAbsent(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
                results.add(lrsrc);
            } catch (ExecutionException e) {
                LOG.error("Error updating blob: ", e);
                if (e.getCause() instanceof AuthorizationException) {
                    throw (AuthorizationException) e.getCause();
                }
                if (e.getCause() instanceof KeyNotFoundException) {
                    throw (KeyNotFoundException) e.getCause();
                }
            }
        }
    } catch (RejectedExecutionException re) {
        LOG.error("Error updating blobs : ", re);
    } catch (InterruptedException ie) {
        throw new IOException("Interrupted Exception", ie);
    }
    return results;
}

Example 3 with ClientBlobStore

use of org.apache.storm.blobstore.ClientBlobStore in project storm by apache.

the class Utils method getClientBlobStoreForSupervisor.

public static ClientBlobStore getClientBlobStoreForSupervisor(Map conf) {
    ClientBlobStore store = (ClientBlobStore) newInstance((String) conf.get(Config.SUPERVISOR_BLOBSTORE));
    store.prepare(conf);
    return store;
}

Example 4 with ClientBlobStore

use of org.apache.storm.blobstore.ClientBlobStore in project storm by apache.

the class AsyncLocalizerTest method testRequestDownloadBaseTopologyBlobs.

@Test
public void testRequestDownloadBaseTopologyBlobs() throws Exception {
    final String topoId = "TOPO";
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    ExecutorInfo ei = new ExecutorInfo();
    ei.set_task_start(1);
    ei.set_task_end(1);
    la.add_to_executors(ei);
    final int port = 8080;
    final String jarKey = topoId + "-stormjar.jar";
    final String codeKey = topoId + "-stormcode.ser";
    final String confKey = topoId + "-stormconf.ser";
    final String stormLocal = "/tmp/storm-local/";
    final String stormRoot = stormLocal + topoId + "/";
    final File fStormRoot = new File(stormRoot);
    ClientBlobStore blobStore = mock(ClientBlobStore.class);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.SUPERVISOR_BLOBSTORE, ClientBlobStore.class.getName());
    conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, DefaultPrincipalToLocal.class.getName());
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    conf.put(Config.STORM_LOCAL_DIR, stormLocal);
    Localizer localizer = mock(Localizer.class);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    ConfigUtils mockedCU = mock(ConfigUtils.class);
    Utils mockedU = mock(Utils.class);
    Map<String, Object> topoConf = new HashMap<>(conf);
    AsyncLocalizer al = new AsyncLocalizer(conf, localizer, ops);
    ConfigUtils orig = ConfigUtils.setInstance(mockedCU);
    Utils origUtils = Utils.setInstance(mockedU);
    try {
        when(mockedCU.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot);
        when(mockedCU.supervisorLocalDirImpl(conf)).thenReturn(stormLocal);
        when(mockedU.newInstanceImpl(ClientBlobStore.class)).thenReturn(blobStore);
        when(mockedCU.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
        Future<Void> f = al.requestDownloadBaseTopologyBlobs(la, port);
        f.get(20, TimeUnit.SECONDS);
        // We should be done now...
        verify(blobStore).prepare(conf);
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(jarKey), startsWith(stormLocal), eq(blobStore));
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(codeKey), startsWith(stormLocal), eq(blobStore));
        verify(mockedU).downloadResourcesAsSupervisorImpl(eq(confKey), startsWith(stormLocal), eq(blobStore));
        verify(blobStore).shutdown();
        //Extracting the dir from the jar
        verify(mockedU).extractDirFromJarImpl(endsWith("stormjar.jar"), eq("resources"), any(File.class));
        verify(ops).moveDirectoryPreferAtomic(any(File.class), eq(fStormRoot));
        verify(ops).setupStormCodeDir(topoConf, fStormRoot);
        verify(ops, never()).deleteIfExists(any(File.class));
    } finally {
        al.shutdown();
        ConfigUtils.setInstance(orig);
        Utils.setInstance(origUtils);
    }
}

Example 5 with ClientBlobStore

use of org.apache.storm.blobstore.ClientBlobStore in project storm by apache.

the class Localizer method getBlobs.

/**
   * This function either returns the blobs in the existing cache or if they don't exist in the
   * cache, it downloads them in parallel (up to SUPERVISOR_BLOBSTORE_DOWNLOAD_THREAD_COUNT)
   * and will block until all of them have been downloaded
   */
public synchronized List<LocalizedResource> getBlobs(List<LocalResource> localResources, String user, String topo, File userFileDir) throws AuthorizationException, KeyNotFoundException, IOException {
    LocalizedResourceSet newSet = new LocalizedResourceSet(user);
    LocalizedResourceSet lrsrcSet = _userRsrc.putIfAbsent(user, newSet);
    if (lrsrcSet == null) {
        lrsrcSet = newSet;
    }
    ArrayList<LocalizedResource> results = new ArrayList<>();
    ArrayList<Callable<LocalizedResource>> downloads = new ArrayList<>();
    ClientBlobStore blobstore = null;
    try {
        blobstore = getClientBlobStore();
        for (LocalResource localResource : localResources) {
            String key = localResource.getBlobName();
            boolean uncompress = localResource.shouldUncompress();
            LocalizedResource lrsrc = lrsrcSet.get(key, localResource.shouldUncompress());
            boolean isUpdate = false;
            if ((lrsrc != null) && (lrsrc.isUncompressed() == localResource.shouldUncompress()) && (isLocalizedResourceDownloaded(lrsrc))) {
                if (isLocalizedResourceUpToDate(lrsrc, blobstore)) {
                    LOG.debug("blob already exists: {}", key);
                    lrsrc.addReference(topo);
                    results.add(lrsrc);
                    continue;
                }
                LOG.debug("blob exists but isn't up to date: {}", key);
                isUpdate = true;
            }
            // go off to blobstore and get it
            // assume dir passed in exists and has correct permission
            LOG.debug("fetching blob: {}", key);
            File downloadDir = getCacheDirForFiles(userFileDir);
            File localFile = new File(downloadDir, key);
            if (uncompress) {
                // for compressed file, download to archives dir
                downloadDir = getCacheDirForArchives(userFileDir);
                localFile = new File(downloadDir, key);
            }
            downloadDir.mkdir();
            downloads.add(new DownloadBlob(this, _conf, key, localFile, user, uncompress, isUpdate));
        }
    } finally {
        if (blobstore != null) {
            blobstore.shutdown();
        }
    }
    try {
        List<Future<LocalizedResource>> futures = _execService.invokeAll(downloads);
        for (Future<LocalizedResource> futureRsrc : futures) {
            LocalizedResource lrsrc = futureRsrc.get();
            lrsrc.addReference(topo);
            lrsrcSet.add(lrsrc.getKey(), lrsrc, lrsrc.isUncompressed());
            results.add(lrsrc);
        }
    } catch (ExecutionException e) {
        if (e.getCause() instanceof AuthorizationException)
            throw (AuthorizationException) e.getCause();
        else if (e.getCause() instanceof KeyNotFoundException) {
            throw (KeyNotFoundException) e.getCause();
        } else {
            throw new IOException("Error getting blobs", e);
        }
    } catch (RejectedExecutionException re) {
        throw new IOException("RejectedExecutionException: ", re);
    } catch (InterruptedException ie) {
        throw new IOException("Interrupted Exception", ie);
    }
    return results;
}