Examples with BlobStoreFile org.apache.storm.blobstore.BlobStoreFile used on opensource projects

Example 6 with BlobStoreFile

use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.

the class HdfsBlobStore method getBlobMeta.

@Override
public ReadableBlobMeta getBlobMeta(String key, Subject who) throws AuthorizationException, KeyNotFoundException {
    who = checkAndGetSubject(who);
    validateKey(key);
    SettableBlobMeta meta = getStoredBlobMeta(key);
    _aclHandler.validateUserCanReadMeta(meta.get_acl(), who, key);
    ReadableBlobMeta rbm = new ReadableBlobMeta();
    rbm.set_settable(meta);
    try {
        BlobStoreFile pf = _hbs.read(DATA_PREFIX + key);
        rbm.set_version(pf.getModTime());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return rbm;
}

Example 7 with BlobStoreFile

use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.

the class HdfsBlobStoreImpl method fullCleanup.

public void fullCleanup(long age) throws IOException {
    long cleanUpIfBefore = System.currentTimeMillis() - age;
    Iterator<String> keys = new KeyInHashDirIterator();
    while (keys.hasNext()) {
        String key = keys.next();
        Path keyDir = getKeyDir(key);
        Iterator<BlobStoreFile> i = listBlobStoreFiles(keyDir);
        if (!i.hasNext()) {
            //The dir is empty, so try to delete it, may fail, but that is OK
            try {
                _fs.delete(keyDir, true);
            } catch (Exception e) {
                LOG.warn("Could not delete " + keyDir + " will try again later");
            }
        }
        while (i.hasNext()) {
            BlobStoreFile f = i.next();
            if (f.isTmp()) {
                if (f.getModTime() <= cleanUpIfBefore) {
                    f.delete();
                }
            }
        }
    }
}

Example 8 with BlobStoreFile

use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.

the class HdfsBlobStoreImplTest method testMultiple.

// Be careful about adding additional tests as the dfscluster will be shared
@Test
public void testMultiple() throws Exception {
    String testString = "testingblob";
    String validKey = "validkeyBasic";
    FileSystem fs = dfscluster.getFileSystem();
    Map conf = new HashMap();
    TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
    // should have created blobDir
    assertTrue("BlobStore dir wasn't created", fs.exists(blobDir));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir).getPermission());
    // test exist with non-existent key
    assertFalse("file exists but shouldn't", hbs.exists("bogus"));
    // test write
    BlobStoreFile pfile = hbs.write(validKey, false);
    // Adding metadata to avoid null pointer exception
    SettableBlobMeta meta = new SettableBlobMeta();
    meta.set_replication_factor(1);
    pfile.setMetadata(meta);
    OutputStream ios = pfile.getOutputStream();
    ios.write(testString.getBytes(Charset.forName("UTF-8")));
    ios.close();
    // test commit creates properly
    assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
    pfile.commit();
    Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA);
    assertTrue("blob data not committed", fs.exists(dataFile));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
    assertTrue("key doesn't exist but should", hbs.exists(validKey));
    // test read
    BlobStoreFile readpFile = hbs.read(validKey);
    String readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
    assertEquals("string read from blob doesn't match", testString, readString);
    // test listkeys
    Iterator<String> keys = hbs.listKeys();
    assertTrue("blob has one key", keys.hasNext());
    assertEquals("one key in blobstore", validKey, keys.next());
    // delete
    hbs.deleteKey(validKey);
    assertFalse("key not deleted", fs.exists(dataFile));
    assertFalse("key not deleted", hbs.exists(validKey));
    // Now do multiple
    String testString2 = "testingblob2";
    String validKey2 = "validkey2";
    // test write
    pfile = hbs.write(validKey, false);
    pfile.setMetadata(meta);
    ios = pfile.getOutputStream();
    ios.write(testString.getBytes(Charset.forName("UTF-8")));
    ios.close();
    // test commit creates properly
    assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
    pfile.commit();
    assertTrue("blob data not committed", fs.exists(dataFile));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
    assertTrue("key doesn't exist but should", hbs.exists(validKey));
    // test write again
    pfile = hbs.write(validKey2, false);
    pfile.setMetadata(meta);
    OutputStream ios2 = pfile.getOutputStream();
    ios2.write(testString2.getBytes(Charset.forName("UTF-8")));
    ios2.close();
    // test commit second creates properly
    pfile.commit();
    Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA);
    assertTrue("blob data not committed", fs.exists(dataFile2));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2).getPermission());
    assertTrue("key doesn't exist but should", hbs.exists(validKey2));
    // test listkeys
    keys = hbs.listKeys();
    int total = 0;
    boolean key1Found = false;
    boolean key2Found = false;
    while (keys.hasNext()) {
        total++;
        String key = keys.next();
        if (key.equals(validKey)) {
            key1Found = true;
        } else if (key.equals(validKey2)) {
            key2Found = true;
        } else {
            fail("Found key that wasn't expected: " + key);
        }
    }
    assertEquals("number of keys is wrong", 2, total);
    assertTrue("blobstore missing key1", key1Found);
    assertTrue("blobstore missing key2", key2Found);
    // test read
    readpFile = hbs.read(validKey);
    readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
    assertEquals("string read from blob doesn't match", testString, readString);
    // test read
    readpFile = hbs.read(validKey2);
    readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
    assertEquals("string read from blob doesn't match", testString2, readString);
    hbs.deleteKey(validKey);
    assertFalse("key not deleted", hbs.exists(validKey));
    hbs.deleteKey(validKey2);
    assertFalse("key not deleted", hbs.exists(validKey2));
}

Example 9 with BlobStoreFile

use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.

the class HdfsBlobStoreImplTest method testGetFileLength.

@Test
public void testGetFileLength() throws IOException {
    FileSystem fs = dfscluster.getFileSystem();
    Map conf = new HashMap();
    String validKey = "validkeyBasic";
    String testString = "testingblob";
    TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
    BlobStoreFile pfile = hbs.write(validKey, false);
    // Adding metadata to avoid null pointer exception
    SettableBlobMeta meta = new SettableBlobMeta();
    meta.set_replication_factor(1);
    pfile.setMetadata(meta);
    OutputStream ios = pfile.getOutputStream();
    ios.write(testString.getBytes(Charset.forName("UTF-8")));
    ios.close();
    assertEquals(testString.getBytes(Charset.forName("UTF-8")).length, pfile.getFileLength());
}