use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.
the class HdfsBlobStore method getBlobMeta.
@Override
public ReadableBlobMeta getBlobMeta(String key, Subject who) throws AuthorizationException, KeyNotFoundException {
who = checkAndGetSubject(who);
validateKey(key);
SettableBlobMeta meta = getStoredBlobMeta(key);
_aclHandler.validateUserCanReadMeta(meta.get_acl(), who, key);
ReadableBlobMeta rbm = new ReadableBlobMeta();
rbm.set_settable(meta);
try {
BlobStoreFile pf = _hbs.read(DATA_PREFIX + key);
rbm.set_version(pf.getModTime());
} catch (IOException e) {
throw new RuntimeException(e);
}
return rbm;
}
use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.
the class HdfsBlobStoreImpl method fullCleanup.
public void fullCleanup(long age) throws IOException {
long cleanUpIfBefore = System.currentTimeMillis() - age;
Iterator<String> keys = new KeyInHashDirIterator();
while (keys.hasNext()) {
String key = keys.next();
Path keyDir = getKeyDir(key);
Iterator<BlobStoreFile> i = listBlobStoreFiles(keyDir);
if (!i.hasNext()) {
//The dir is empty, so try to delete it, may fail, but that is OK
try {
_fs.delete(keyDir, true);
} catch (Exception e) {
LOG.warn("Could not delete " + keyDir + " will try again later");
}
}
while (i.hasNext()) {
BlobStoreFile f = i.next();
if (f.isTmp()) {
if (f.getModTime() <= cleanUpIfBefore) {
f.delete();
}
}
}
}
}
use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.
the class HdfsBlobStoreImplTest method testMultiple.
// Be careful about adding additional tests as the dfscluster will be shared
@Test
public void testMultiple() throws Exception {
String testString = "testingblob";
String validKey = "validkeyBasic";
FileSystem fs = dfscluster.getFileSystem();
Map conf = new HashMap();
TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
// should have created blobDir
assertTrue("BlobStore dir wasn't created", fs.exists(blobDir));
assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir).getPermission());
// test exist with non-existent key
assertFalse("file exists but shouldn't", hbs.exists("bogus"));
// test write
BlobStoreFile pfile = hbs.write(validKey, false);
// Adding metadata to avoid null pointer exception
SettableBlobMeta meta = new SettableBlobMeta();
meta.set_replication_factor(1);
pfile.setMetadata(meta);
OutputStream ios = pfile.getOutputStream();
ios.write(testString.getBytes(Charset.forName("UTF-8")));
ios.close();
// test commit creates properly
assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
pfile.commit();
Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA);
assertTrue("blob data not committed", fs.exists(dataFile));
assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
assertTrue("key doesn't exist but should", hbs.exists(validKey));
// test read
BlobStoreFile readpFile = hbs.read(validKey);
String readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
assertEquals("string read from blob doesn't match", testString, readString);
// test listkeys
Iterator<String> keys = hbs.listKeys();
assertTrue("blob has one key", keys.hasNext());
assertEquals("one key in blobstore", validKey, keys.next());
// delete
hbs.deleteKey(validKey);
assertFalse("key not deleted", fs.exists(dataFile));
assertFalse("key not deleted", hbs.exists(validKey));
// Now do multiple
String testString2 = "testingblob2";
String validKey2 = "validkey2";
// test write
pfile = hbs.write(validKey, false);
pfile.setMetadata(meta);
ios = pfile.getOutputStream();
ios.write(testString.getBytes(Charset.forName("UTF-8")));
ios.close();
// test commit creates properly
assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
pfile.commit();
assertTrue("blob data not committed", fs.exists(dataFile));
assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
assertTrue("key doesn't exist but should", hbs.exists(validKey));
// test write again
pfile = hbs.write(validKey2, false);
pfile.setMetadata(meta);
OutputStream ios2 = pfile.getOutputStream();
ios2.write(testString2.getBytes(Charset.forName("UTF-8")));
ios2.close();
// test commit second creates properly
pfile.commit();
Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA);
assertTrue("blob data not committed", fs.exists(dataFile2));
assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2).getPermission());
assertTrue("key doesn't exist but should", hbs.exists(validKey2));
// test listkeys
keys = hbs.listKeys();
int total = 0;
boolean key1Found = false;
boolean key2Found = false;
while (keys.hasNext()) {
total++;
String key = keys.next();
if (key.equals(validKey)) {
key1Found = true;
} else if (key.equals(validKey2)) {
key2Found = true;
} else {
fail("Found key that wasn't expected: " + key);
}
}
assertEquals("number of keys is wrong", 2, total);
assertTrue("blobstore missing key1", key1Found);
assertTrue("blobstore missing key2", key2Found);
// test read
readpFile = hbs.read(validKey);
readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
assertEquals("string read from blob doesn't match", testString, readString);
// test read
readpFile = hbs.read(validKey2);
readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
assertEquals("string read from blob doesn't match", testString2, readString);
hbs.deleteKey(validKey);
assertFalse("key not deleted", hbs.exists(validKey));
hbs.deleteKey(validKey2);
assertFalse("key not deleted", hbs.exists(validKey2));
}
use of org.apache.storm.blobstore.BlobStoreFile in project storm by apache.
the class HdfsBlobStoreImplTest method testGetFileLength.
@Test
public void testGetFileLength() throws IOException {
FileSystem fs = dfscluster.getFileSystem();
Map conf = new HashMap();
String validKey = "validkeyBasic";
String testString = "testingblob";
TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
BlobStoreFile pfile = hbs.write(validKey, false);
// Adding metadata to avoid null pointer exception
SettableBlobMeta meta = new SettableBlobMeta();
meta.set_replication_factor(1);
pfile.setMetadata(meta);
OutputStream ios = pfile.getOutputStream();
ios.write(testString.getBytes(Charset.forName("UTF-8")));
ios.close();
assertEquals(testString.getBytes(Charset.forName("UTF-8")).length, pfile.getFileLength());
}
Aggregations