Examples with SettableBlobMeta org.apache.storm.generated.SettableBlobMeta used on opensource projects

Example 56 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class ClientBlobStoreTest method testGoodACLsForSetBlobMeta.

@Test
public void testGoodACLsForSetBlobMeta() throws Exception {
    String testKey = "testBlobKey";
    SettableBlobMeta meta = new SettableBlobMeta();
    createTestBlob(testKey, meta);
    meta.add_to_acl(BlobStoreAclHandler.parseAccessControl("u:nextuser:r--"));
    client.setBlobMeta(testKey, meta);
    validatedBlobAcls(testKey);
}

Example 57 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class ClientBlobStoreTest method testGoodACLsForCreate.

@Test
public void testGoodACLsForCreate() throws Exception {
    SettableBlobMeta meta = new SettableBlobMeta();
    AccessControl submitterAcl = BlobStoreAclHandler.parseAccessControl("u:tester:rwa");
    meta.add_to_acl(submitterAcl);
    String testKey = "testBlobKey";
    client.createBlob(testKey, meta);
    validatedBlobAcls(testKey);
}

Example 58 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class AsyncLocalizerTest method testMultipleKeysOneUser.

@Test
public void testMultipleKeysOneUser() throws Exception {
    try (Time.SimulatedTime st = new Time.SimulatedTime();
        TmpPath tmp = new TmpPath()) {
        Map<String, Object> conf = new HashMap<>();
        // set clean time really high so doesn't kick in
        conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1_000);
        String key1 = "key1";
        String topo1 = "topo1";
        String key2 = "key2";
        String key3 = "key3";
        TestLocalizer localizer = new TestLocalizer(conf, tmp.getPath());
        // set to keep 2 blobs (each of size 34)
        localizer.setTargetCacheSize(68);
        ReadableBlobMeta rbm = new ReadableBlobMeta();
        rbm.set_settable(new SettableBlobMeta(WORLD_EVERYTHING));
        when(mockBlobStore.getBlobMeta(anyString())).thenReturn(rbm);
        when(mockBlobStore.isRemoteBlobExists(anyString())).thenReturn(true);
        when(mockBlobStore.getBlob(key1)).thenReturn(new TestInputStreamWithMeta(0));
        when(mockBlobStore.getBlob(key2)).thenReturn(new TestInputStreamWithMeta(0));
        when(mockBlobStore.getBlob(key3)).thenReturn(new TestInputStreamWithMeta(0));
        List<LocalResource> keys = Arrays.asList(new LocalResource(key1, false, false), new LocalResource(key2, false, false), new LocalResource(key3, false, false));
        File user1Dir = localizer.getLocalUserFileCacheDir(user1);
        assertTrue("failed to create user dir", user1Dir.mkdirs());
        LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList());
        PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment);
        List<LocalizedResource> lrsrcs = localizer.getBlobs(keys, topo1Pna, null);
        LocalizedResource lrsrc = lrsrcs.get(0);
        LocalizedResource lrsrc2 = lrsrcs.get(1);
        LocalizedResource lrsrc3 = lrsrcs.get(2);
        String expectedFileDir = joinPath(tmp.getPath(), USERCACHE, user1, LocalizedResource.FILECACHE, LocalizedResource.FILESDIR);
        assertTrue("user filecache dir not created", new File(expectedFileDir).exists());
        File keyFile = new File(expectedFileDir, key1 + LocalizedResource.CURRENT_BLOB_SUFFIX);
        File keyFile2 = new File(expectedFileDir, key2 + LocalizedResource.CURRENT_BLOB_SUFFIX);
        File keyFile3 = new File(expectedFileDir, key3 + LocalizedResource.CURRENT_BLOB_SUFFIX);
        assertTrue("blob not created", keyFile.exists());
        assertTrue("blob not created", keyFile2.exists());
        assertTrue("blob not created", keyFile3.exists());
        assertEquals("size doesn't match", 34, keyFile.length());
        assertEquals("size doesn't match", 34, keyFile2.length());
        assertEquals("size doesn't match", 34, keyFile3.length());
        assertEquals("size doesn't match", 34, lrsrc.getSizeOnDisk());
        assertEquals("size doesn't match", 34, lrsrc3.getSizeOnDisk());
        assertEquals("size doesn't match", 34, lrsrc2.getSizeOnDisk());
        ConcurrentMap<String, LocalizedResource> lrsrcSet = localizer.getUserFiles().get(user1);
        assertEquals("local resource set size wrong", 3, lrsrcSet.size());
        LOG.info("Removing blob references...");
        long timeBefore = Time.nanoTime();
        Time.advanceTime(10);
        localizer.removeBlobReference(lrsrc.getKey(), topo1Pna, false);
        Time.advanceTime(10);
        localizer.removeBlobReference(lrsrc2.getKey(), topo1Pna, false);
        Time.advanceTime(10);
        localizer.removeBlobReference(lrsrc3.getKey(), topo1Pna, false);
        Time.advanceTime(10);
        long timeAfter = Time.nanoTime();
        LOG.info("Done removing blob references...");
        // add reference to one and then remove reference again so it has newer timestamp
        LOG.info("Get Blob...");
        lrsrc = localizer.getBlob(new LocalResource(key1, false, false), topo1Pna, null);
        LOG.info("Got Blob...");
        assertTrue("timestamp not within range " + timeBefore + " <= " + lrsrc.getLastUsed() + " <= " + timeAfter, (lrsrc.getLastUsed() >= timeBefore && lrsrc.getLastUsed() <= timeAfter));
        // Resets the last access time for key1
        localizer.removeBlobReference(lrsrc.getKey(), topo1Pna, false);
        // should remove the second blob first
        localizer.cleanup();
        lrsrcSet = localizer.getUserFiles().get(user1);
        assertEquals("local resource set size wrong", 2, lrsrcSet.size());
        long end = System.currentTimeMillis() + 100;
        while ((end - System.currentTimeMillis()) >= 0 && keyFile2.exists()) {
            Thread.sleep(1);
        }
        assertTrue("blob deleted", keyFile.exists());
        assertFalse("blob not deleted", keyFile2.exists());
        assertTrue("blob deleted", keyFile3.exists());
        // set size to cleanup another one
        localizer.setTargetCacheSize(34);
        // should remove the third blob, because the first has the reset timestamp
        localizer.cleanup();
        lrsrcSet = localizer.getUserFiles().get(user1);
        assertEquals("local resource set size wrong", 1, lrsrcSet.size());
        assertTrue("blob deleted", keyFile.exists());
        assertFalse("blob not deleted", keyFile2.exists());
        assertFalse("blob not deleted", keyFile3.exists());
    }
}

Example 59 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class AsyncLocalizerTest method testArchives.

// archive passed in must contain symlink named tmptestsymlink if not a zip file
public void testArchives(File archiveFile, boolean supportSymlinks, int size) throws Exception {
    if (Utils.isOnWindows()) {
        // Windows should set this to false cause symlink in compressed file doesn't work properly.
        supportSymlinks = false;
    }
    try (Time.SimulatedTime st = new Time.SimulatedTime();
        TmpPath tmp = new TmpPath()) {
        Map<String, Object> conf = new HashMap<>();
        // set clean time really high so doesn't kick in
        conf.put(DaemonConfig.SUPERVISOR_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, 60 * 60 * 1000);
        String key1 = archiveFile.getName();
        String topo1 = "topo1";
        LOG.info("About to create new AsyncLocalizer...");
        TestLocalizer localizer = new TestLocalizer(conf, tmp.getPath());
        // set really small so will do cleanup
        localizer.setTargetCacheSize(1);
        LOG.info("created AsyncLocalizer...");
        ReadableBlobMeta rbm = new ReadableBlobMeta();
        rbm.set_settable(new SettableBlobMeta(WORLD_EVERYTHING));
        when(mockBlobStore.getBlobMeta(key1)).thenReturn(rbm);
        when(mockBlobStore.getBlob(key1)).thenReturn(new TestInputStreamWithMeta(new FileInputStream(archiveFile.getAbsolutePath()), 0, archiveFile.length()));
        long timeBefore = Time.currentTimeMillis();
        Time.advanceTime(10);
        File user1Dir = localizer.getLocalUserFileCacheDir(user1);
        assertTrue("failed to create user dir", user1Dir.mkdirs());
        LocalAssignment topo1Assignment = constructLocalAssignment(topo1, user1, Collections.emptyList());
        PortAndAssignment topo1Pna = new PortAndAssignmentImpl(1, topo1Assignment);
        LocalizedResource lrsrc = localizer.getBlob(new LocalResource(key1, true, false), topo1Pna, null);
        Time.advanceTime(10);
        long timeAfter = Time.currentTimeMillis();
        Time.advanceTime(10);
        String expectedUserDir = joinPath(tmp.getPath(), USERCACHE, user1);
        String expectedFileDir = joinPath(expectedUserDir, LocalizedResource.FILECACHE, LocalizedResource.ARCHIVESDIR);
        assertTrue("user filecache dir not created", new File(expectedFileDir).exists());
        File keyFile = new File(expectedFileDir, key1 + ".0");
        assertTrue("blob not created " + keyFile, keyFile.exists());
        assertTrue("blob is not uncompressed", keyFile.isDirectory());
        File symlinkFile = new File(keyFile, "tmptestsymlink");
        if (supportSymlinks) {
            assertTrue("blob uncompressed doesn't contain symlink", Files.isSymbolicLink(symlinkFile.toPath()));
        } else {
            assertTrue("blob symlink file doesn't exist", symlinkFile.exists());
        }
        ConcurrentMap<String, LocalizedResource> lrsrcSet = localizer.getUserArchives().get(user1);
        assertEquals("local resource set size wrong", 1, lrsrcSet.size());
        LocalizedResource key1rsrc = lrsrcSet.get(key1);
        assertNotNull("Local resource doesn't exist but should", key1rsrc);
        assertEquals("key doesn't match", key1, key1rsrc.getKey());
        assertEquals("refcount doesn't match " + key1rsrc.getDependencies(), true, key1rsrc.isUsed());
        assertEquals("file path doesn't match", keyFile.toPath(), key1rsrc.getFilePathWithVersion());
        assertEquals("size doesn't match", size, key1rsrc.getSizeOnDisk());
        assertTrue("timestamp not within range", (key1rsrc.getLastUsed() >= timeBefore && key1rsrc.getLastUsed() <= timeAfter));
        timeBefore = Time.currentTimeMillis();
        Time.advanceTime(10);
        localizer.removeBlobReference(lrsrc.getKey(), topo1Pna, true);
        Time.advanceTime(10);
        timeAfter = Time.currentTimeMillis();
        Time.advanceTime(10);
        lrsrcSet = localizer.getUserArchives().get(user1);
        assertEquals("local resource set size wrong", 1, lrsrcSet.size());
        key1rsrc = lrsrcSet.get(key1);
        assertNotNull("Local resource doesn't exist but should", key1rsrc);
        assertEquals("refcount doesn't match " + key1rsrc.getDependencies(), false, key1rsrc.isUsed());
        assertTrue("timestamp not within range", (key1rsrc.getLastUsed() >= timeBefore && key1rsrc.getLastUsed() <= timeAfter));
        // should remove the blob since cache size set really small
        localizer.cleanup();
        lrsrcSet = localizer.getUserArchives().get(user1);
        assertFalse("blob contents not deleted", symlinkFile.exists());
        assertFalse("blob not deleted", keyFile.exists());
        assertFalse("blob file dir not deleted", new File(expectedFileDir).exists());
        assertFalse("blob dir not deleted", new File(expectedUserDir).exists());
        assertNull("user set should be null", lrsrcSet);
    }
}

Example 60 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class LocalFsBlobStore method setBlobMeta.

@Override
public void setBlobMeta(String key, SettableBlobMeta meta, Subject who) throws AuthorizationException, KeyNotFoundException {
    validateKey(key);
    checkForBlobOrDownload(key);
    aclHandler.normalizeSettableBlobMeta(key, meta, who, ADMIN);
    BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl());
    SettableBlobMeta orig = getStoredBlobMeta(key);
    aclHandler.hasPermissions(orig.get_acl(), ADMIN, who, key);
    BlobStoreFileOutputStream outputStream = null;
    try {
        outputStream = new BlobStoreFileOutputStream(fbs.write(META_PREFIX + key, false));
        outputStream.write(Utils.thriftSerialize(meta));
        outputStream.close();
        outputStream = null;
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (outputStream != null) {
            try {
                outputStream.cancel();
            } catch (IOException e) {
            // Ignored
            }
        }
    }
}