Examples with ReadableBlobMeta org.apache.storm.generated.ReadableBlobMeta used on opensource projects

Example 1 with ReadableBlobMeta

use of org.apache.storm.generated.ReadableBlobMeta in project storm by apache.

the class LocalFsBlobStore method getBlobMeta.

@Override
public ReadableBlobMeta getBlobMeta(String key, Subject who) throws AuthorizationException, KeyNotFoundException {
    validateKey(key);
    if (!checkForBlobOrDownload(key)) {
        checkForBlobUpdate(key);
    }
    SettableBlobMeta meta = getStoredBlobMeta(key);
    _aclHandler.validateUserCanReadMeta(meta.get_acl(), who, key);
    ReadableBlobMeta rbm = new ReadableBlobMeta();
    rbm.set_settable(meta);
    try {
        LocalFsBlobStoreFile pf = fbs.read(DATA_PREFIX + key);
        rbm.set_version(pf.getModTime());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return rbm;
}

Example 2 with ReadableBlobMeta

use of org.apache.storm.generated.ReadableBlobMeta in project storm by apache.

the class Blobstore method setAclCli.

private static void setAclCli(String[] args) throws Exception {
    Map<String, Object> cl = CLI.opt("s", "set", Collections.emptyList(), new AsAclParser()).arg("key", CLI.FIRST_WINS).parse(args);
    final String key = (String) cl.get("key");
    final List<AccessControl> setAcl = (List<AccessControl>) cl.get("s");
    ClientBlobStore.withConfiguredClient(new ClientBlobStore.WithBlobstore() {

        @Override
        public void run(ClientBlobStore blobStore) throws Exception {
            ReadableBlobMeta meta = blobStore.getBlobMeta(key);
            List<AccessControl> acl = meta.get_settable().get_acl();
            List<AccessControl> newAcl;
            if (setAcl != null && !setAcl.isEmpty()) {
                newAcl = setAcl;
            } else {
                newAcl = acl;
            }
            SettableBlobMeta newMeta = new SettableBlobMeta(newAcl);
            LOG.info("Setting ACL for {} to {}", key, generateAccessControlsInfo(newAcl));
            blobStore.setBlobMeta(key, newMeta);
        }
    });
}

Example 3 with ReadableBlobMeta

use of org.apache.storm.generated.ReadableBlobMeta in project storm by apache.

the class LocalizerTest method testMultipleUsers.

@Test
public void testMultipleUsers() throws Exception {
    Map conf = new HashMap();
    // set clean time really high so doesn't kick in
    conf.put(Config.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000);
    String topo1 = "topo1";
    String topo2 = "topo2";
    String topo3 = "topo3";
    String key1 = "key1";
    String key2 = "key2";
    String key3 = "key3";
    Localizer localizer = new TestLocalizer(conf, baseDir.toString());
    // set to keep 2 blobs (each of size 34)
    localizer.setTargetCacheSize(68);
    ReadableBlobMeta rbm = new ReadableBlobMeta();
    rbm.set_settable(new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING));
    when(mockblobstore.getBlobMeta(anyString())).thenReturn(rbm);
    when(mockblobstore.getBlob(key1)).thenReturn(new TestInputStreamWithMeta());
    when(mockblobstore.getBlob(key2)).thenReturn(new TestInputStreamWithMeta());
    when(mockblobstore.getBlob(key3)).thenReturn(new TestInputStreamWithMeta());
    File user1Dir = localizer.getLocalUserFileCacheDir(user1);
    assertTrue("failed to create user dir", user1Dir.mkdirs());
    File user2Dir = localizer.getLocalUserFileCacheDir(user2);
    assertTrue("failed to create user dir", user2Dir.mkdirs());
    File user3Dir = localizer.getLocalUserFileCacheDir(user3);
    assertTrue("failed to create user dir", user3Dir.mkdirs());
    LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false), user1, topo1, user1Dir);
    LocalizedResource lrsrc2 = localizer.getBlob(new LocalResource(key2, false), user2, topo2, user2Dir);
    LocalizedResource lrsrc3 = localizer.getBlob(new LocalResource(key3, false), user3, topo3, user3Dir);
    // make sure we support different user reading same blob
    LocalizedResource lrsrc1_user3 = localizer.getBlob(new LocalResource(key1, false), user3, topo3, user3Dir);
    String expectedUserDir1 = joinPath(baseDir.toString(), Localizer.USERCACHE, user1);
    String expectedFileDirUser1 = joinPath(expectedUserDir1, Localizer.FILECACHE, Localizer.FILESDIR);
    String expectedFileDirUser2 = joinPath(baseDir.toString(), Localizer.USERCACHE, user2, Localizer.FILECACHE, Localizer.FILESDIR);
    String expectedFileDirUser3 = joinPath(baseDir.toString(), Localizer.USERCACHE, user3, Localizer.FILECACHE, Localizer.FILESDIR);
    assertTrue("user filecache dir user1 not created", new File(expectedFileDirUser1).exists());
    assertTrue("user filecache dir user2 not created", new File(expectedFileDirUser2).exists());
    assertTrue("user filecache dir user3 not created", new File(expectedFileDirUser3).exists());
    File keyFile = new File(expectedFileDirUser1, key1 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    File keyFile2 = new File(expectedFileDirUser2, key2 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    File keyFile3 = new File(expectedFileDirUser3, key3 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    File keyFile1user3 = new File(expectedFileDirUser3, key1 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    assertTrue("blob not created", keyFile.exists());
    assertTrue("blob not created", keyFile2.exists());
    assertTrue("blob not created", keyFile3.exists());
    assertTrue("blob not created", keyFile1user3.exists());
    LocalizedResourceSet lrsrcSet = localizer.getUserResources().get(user1);
    assertEquals("local resource set size wrong", 1, lrsrcSet.getSize());
    LocalizedResourceSet lrsrcSet2 = localizer.getUserResources().get(user2);
    assertEquals("local resource set size wrong", 1, lrsrcSet2.getSize());
    LocalizedResourceSet lrsrcSet3 = localizer.getUserResources().get(user3);
    assertEquals("local resource set size wrong", 2, lrsrcSet3.getSize());
    localizer.removeBlobReference(lrsrc.getKey(), user1, topo1, false);
    // should remove key1
    localizer.handleCacheCleanup();
    lrsrcSet = localizer.getUserResources().get(user1);
    lrsrcSet3 = localizer.getUserResources().get(user3);
    assertNull("user set should be null", lrsrcSet);
    assertFalse("blob dir not deleted", new File(expectedFileDirUser1).exists());
    assertFalse("blob dir not deleted", new File(expectedUserDir1).exists());
    assertEquals("local resource set size wrong", 2, lrsrcSet3.getSize());
    assertTrue("blob deleted", keyFile2.exists());
    assertFalse("blob not deleted", keyFile.exists());
    assertTrue("blob deleted", keyFile3.exists());
    assertTrue("blob deleted", keyFile1user3.exists());
}

Example 4 with ReadableBlobMeta

use of org.apache.storm.generated.ReadableBlobMeta in project storm by apache.

the class LocalizerTest method testMultipleKeysOneUser.

@Test
public void testMultipleKeysOneUser() throws Exception {
    Map conf = new HashMap();
    // set clean time really high so doesn't kick in
    conf.put(Config.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000);
    String key1 = "key1";
    String topo1 = "topo1";
    String key2 = "key2";
    String key3 = "key3";
    Localizer localizer = new TestLocalizer(conf, baseDir.toString());
    // set to keep 2 blobs (each of size 34)
    localizer.setTargetCacheSize(68);
    ReadableBlobMeta rbm = new ReadableBlobMeta();
    rbm.set_settable(new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING));
    when(mockblobstore.getBlobMeta(anyString())).thenReturn(rbm);
    when(mockblobstore.getBlob(key1)).thenReturn(new TestInputStreamWithMeta());
    when(mockblobstore.getBlob(key2)).thenReturn(new TestInputStreamWithMeta());
    when(mockblobstore.getBlob(key3)).thenReturn(new TestInputStreamWithMeta());
    List<LocalResource> keys = Arrays.asList(new LocalResource[] { new LocalResource(key1, false), new LocalResource(key2, false), new LocalResource(key3, false) });
    File user1Dir = localizer.getLocalUserFileCacheDir(user1);
    assertTrue("failed to create user dir", user1Dir.mkdirs());
    List<LocalizedResource> lrsrcs = localizer.getBlobs(keys, user1, topo1, user1Dir);
    LocalizedResource lrsrc = lrsrcs.get(0);
    LocalizedResource lrsrc2 = lrsrcs.get(1);
    LocalizedResource lrsrc3 = lrsrcs.get(2);
    String expectedFileDir = joinPath(baseDir.toString(), Localizer.USERCACHE, user1, Localizer.FILECACHE, Localizer.FILESDIR);
    assertTrue("user filecache dir not created", new File(expectedFileDir).exists());
    File keyFile = new File(expectedFileDir, key1 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    File keyFile2 = new File(expectedFileDir, key2 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    File keyFile3 = new File(expectedFileDir, key3 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    assertTrue("blob not created", keyFile.exists());
    assertTrue("blob not created", keyFile2.exists());
    assertTrue("blob not created", keyFile3.exists());
    assertEquals("size doesn't match", 34, keyFile.length());
    assertEquals("size doesn't match", 34, keyFile2.length());
    assertEquals("size doesn't match", 34, keyFile3.length());
    assertEquals("size doesn't match", 34, lrsrc.getSize());
    assertEquals("size doesn't match", 34, lrsrc3.getSize());
    assertEquals("size doesn't match", 34, lrsrc2.getSize());
    LocalizedResourceSet lrsrcSet = localizer.getUserResources().get(user1);
    assertEquals("local resource set size wrong", 3, lrsrcSet.getSize());
    assertEquals("user doesn't match", user1, lrsrcSet.getUser());
    long timeBefore = System.nanoTime();
    localizer.removeBlobReference(lrsrc.getKey(), user1, topo1, false);
    localizer.removeBlobReference(lrsrc2.getKey(), user1, topo1, false);
    localizer.removeBlobReference(lrsrc3.getKey(), user1, topo1, false);
    long timeAfter = System.nanoTime();
    // add reference to one and then remove reference again so it has newer timestamp
    lrsrc = localizer.getBlob(new LocalResource(key1, false), user1, topo1, user1Dir);
    assertTrue("timestamp not within range", (lrsrc.getLastAccessTime() >= timeBefore && lrsrc.getLastAccessTime() <= timeAfter));
    localizer.removeBlobReference(lrsrc.getKey(), user1, topo1, false);
    // should remove the second blob first
    localizer.handleCacheCleanup();
    lrsrcSet = localizer.getUserResources().get(user1);
    assertEquals("local resource set size wrong", 2, lrsrcSet.getSize());
    long end = System.currentTimeMillis() + 100;
    while ((end - System.currentTimeMillis()) >= 0 && keyFile2.exists()) {
        Thread.sleep(1);
    }
    assertFalse("blob not deleted", keyFile2.exists());
    assertTrue("blob deleted", keyFile.exists());
    assertTrue("blob deleted", keyFile3.exists());
    // set size to cleanup another one
    localizer.setTargetCacheSize(34);
    // should remove the third blob
    localizer.handleCacheCleanup();
    lrsrcSet = localizer.getUserResources().get(user1);
    assertEquals("local resource set size wrong", 1, lrsrcSet.getSize());
    assertTrue("blob deleted", keyFile.exists());
    assertFalse("blob not deleted", keyFile3.exists());
}

Example 5 with ReadableBlobMeta

use of org.apache.storm.generated.ReadableBlobMeta in project storm by apache.

the class LocalizerTest method testBasic.

@Test
public void testBasic() throws Exception {
    Map conf = new HashMap();
    // set clean time really high so doesn't kick in
    conf.put(Config.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000);
    String key1 = "key1";
    String topo1 = "topo1";
    Localizer localizer = new TestLocalizer(conf, baseDir.toString());
    // set really small so will do cleanup
    localizer.setTargetCacheSize(1);
    ReadableBlobMeta rbm = new ReadableBlobMeta();
    rbm.set_settable(new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING));
    when(mockblobstore.getBlobMeta(key1)).thenReturn(rbm);
    when(mockblobstore.getBlob(key1)).thenReturn(new TestInputStreamWithMeta());
    long timeBefore = System.nanoTime();
    File user1Dir = localizer.getLocalUserFileCacheDir(user1);
    assertTrue("failed to create user dir", user1Dir.mkdirs());
    LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, false), user1, topo1, user1Dir);
    long timeAfter = System.nanoTime();
    String expectedUserDir = joinPath(baseDir.toString(), Localizer.USERCACHE, user1);
    String expectedFileDir = joinPath(expectedUserDir, Localizer.FILECACHE, Localizer.FILESDIR);
    assertTrue("user filecache dir not created", new File(expectedFileDir).exists());
    File keyFile = new File(expectedFileDir, key1);
    File keyFileCurrentSymlink = new File(expectedFileDir, key1 + Utils.DEFAULT_CURRENT_BLOB_SUFFIX);
    assertTrue("blob not created", keyFileCurrentSymlink.exists());
    LocalizedResourceSet lrsrcSet = localizer.getUserResources().get(user1);
    assertEquals("local resource set size wrong", 1, lrsrcSet.getSize());
    assertEquals("user doesn't match", user1, lrsrcSet.getUser());
    LocalizedResource key1rsrc = lrsrcSet.get(key1, false);
    assertNotNull("Local resource doesn't exist but should", key1rsrc);
    assertEquals("key doesn't match", key1, key1rsrc.getKey());
    assertEquals("refcount doesn't match", 1, key1rsrc.getRefCount());
    assertEquals("file path doesn't match", keyFile.toString(), key1rsrc.getFilePath());
    assertEquals("size doesn't match", 34, key1rsrc.getSize());
    assertTrue("timestamp not within range", (key1rsrc.getLastAccessTime() >= timeBefore && key1rsrc.getLastAccessTime() <= timeAfter));
    timeBefore = System.nanoTime();
    localizer.removeBlobReference(lrsrc.getKey(), user1, topo1, false);
    timeAfter = System.nanoTime();
    lrsrcSet = localizer.getUserResources().get(user1);
    assertEquals("local resource set size wrong", 1, lrsrcSet.getSize());
    key1rsrc = lrsrcSet.get(key1, false);
    assertNotNull("Local resource doesn't exist but should", key1rsrc);
    assertEquals("refcount doesn't match", 0, key1rsrc.getRefCount());
    assertTrue("timestamp not within range", (key1rsrc.getLastAccessTime() >= timeBefore && key1rsrc.getLastAccessTime() <= timeAfter));
    // should remove the blob since cache size set really small
    localizer.handleCacheCleanup();
    lrsrcSet = localizer.getUserResources().get(user1);
    assertNull("user set should be null", lrsrcSet);
    assertFalse("blob not deleted", keyFile.exists());
    assertFalse("blob dir not deleted", new File(expectedFileDir).exists());
    assertFalse("blob dir not deleted", new File(expectedUserDir).exists());
}