Examples with SettableBlobMeta org.apache.storm.generated.SettableBlobMeta used on opensource projects

Example 26 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class BlobStoreTest method testBasic.

public void testBasic(BlobStore store) throws Exception {
    assertStoreHasExactly(store);
    LOG.info("Creating test");
    // Tests for case when subject == null (security turned off) and
    // acls for the blob are set to WORLD_EVERYTHING
    SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    AtomicOutputStream out = store.createBlob("test", metadata, null);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEquals(store, "test", 1);
    LOG.info("Deleting test");
    store.deleteBlob("test", null);
    assertStoreHasExactly(store);
    // The following tests are run for both hdfs and local store to test the
    // update blob interface
    metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    LOG.info("Creating test again");
    out = store.createBlob("test", metadata, null);
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 2);
    LOG.info("Updating test");
    out = store.updateBlob("test", null);
    out.write(3);
    out.close();
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    LOG.info("Updating test again");
    out = store.updateBlob("test", null);
    out.write(4);
    out.flush();
    LOG.info("SLEEPING");
    Thread.sleep(2);
    if (store instanceof HdfsBlobStore) {
        ((HdfsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
    try {
        out.close();
    } catch (IOException e) {
    //This is likely to happen when we try to commit something that
    // was cleaned up.  This is expected and acceptable.
    }
}

Example 27 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class HdfsBlobStoreImplTest method testMultiple.

// Be careful about adding additional tests as the dfscluster will be shared
@Test
public void testMultiple() throws Exception {
    String testString = "testingblob";
    String validKey = "validkeyBasic";
    FileSystem fs = dfscluster.getFileSystem();
    Map conf = new HashMap();
    TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
    // should have created blobDir
    assertTrue("BlobStore dir wasn't created", fs.exists(blobDir));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir).getPermission());
    // test exist with non-existent key
    assertFalse("file exists but shouldn't", hbs.exists("bogus"));
    // test write
    BlobStoreFile pfile = hbs.write(validKey, false);
    // Adding metadata to avoid null pointer exception
    SettableBlobMeta meta = new SettableBlobMeta();
    meta.set_replication_factor(1);
    pfile.setMetadata(meta);
    OutputStream ios = pfile.getOutputStream();
    ios.write(testString.getBytes(Charset.forName("UTF-8")));
    ios.close();
    // test commit creates properly
    assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
    pfile.commit();
    Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA);
    assertTrue("blob data not committed", fs.exists(dataFile));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
    assertTrue("key doesn't exist but should", hbs.exists(validKey));
    // test read
    BlobStoreFile readpFile = hbs.read(validKey);
    String readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
    assertEquals("string read from blob doesn't match", testString, readString);
    // test listkeys
    Iterator<String> keys = hbs.listKeys();
    assertTrue("blob has one key", keys.hasNext());
    assertEquals("one key in blobstore", validKey, keys.next());
    // delete
    hbs.deleteKey(validKey);
    assertFalse("key not deleted", fs.exists(dataFile));
    assertFalse("key not deleted", hbs.exists(validKey));
    // Now do multiple
    String testString2 = "testingblob2";
    String validKey2 = "validkey2";
    // test write
    pfile = hbs.write(validKey, false);
    pfile.setMetadata(meta);
    ios = pfile.getOutputStream();
    ios.write(testString.getBytes(Charset.forName("UTF-8")));
    ios.close();
    // test commit creates properly
    assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
    pfile.commit();
    assertTrue("blob data not committed", fs.exists(dataFile));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
    assertTrue("key doesn't exist but should", hbs.exists(validKey));
    // test write again
    pfile = hbs.write(validKey2, false);
    pfile.setMetadata(meta);
    OutputStream ios2 = pfile.getOutputStream();
    ios2.write(testString2.getBytes(Charset.forName("UTF-8")));
    ios2.close();
    // test commit second creates properly
    pfile.commit();
    Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA);
    assertTrue("blob data not committed", fs.exists(dataFile2));
    assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2).getPermission());
    assertTrue("key doesn't exist but should", hbs.exists(validKey2));
    // test listkeys
    keys = hbs.listKeys();
    int total = 0;
    boolean key1Found = false;
    boolean key2Found = false;
    while (keys.hasNext()) {
        total++;
        String key = keys.next();
        if (key.equals(validKey)) {
            key1Found = true;
        } else if (key.equals(validKey2)) {
            key2Found = true;
        } else {
            fail("Found key that wasn't expected: " + key);
        }
    }
    assertEquals("number of keys is wrong", 2, total);
    assertTrue("blobstore missing key1", key1Found);
    assertTrue("blobstore missing key2", key2Found);
    // test read
    readpFile = hbs.read(validKey);
    readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
    assertEquals("string read from blob doesn't match", testString, readString);
    // test read
    readpFile = hbs.read(validKey2);
    readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
    assertEquals("string read from blob doesn't match", testString2, readString);
    hbs.deleteKey(validKey);
    assertFalse("key not deleted", hbs.exists(validKey));
    hbs.deleteKey(validKey2);
    assertFalse("key not deleted", hbs.exists(validKey2));
}

Example 28 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class HdfsBlobStoreImplTest method testGetFileLength.

@Test
public void testGetFileLength() throws IOException {
    FileSystem fs = dfscluster.getFileSystem();
    Map conf = new HashMap();
    String validKey = "validkeyBasic";
    String testString = "testingblob";
    TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
    BlobStoreFile pfile = hbs.write(validKey, false);
    // Adding metadata to avoid null pointer exception
    SettableBlobMeta meta = new SettableBlobMeta();
    meta.set_replication_factor(1);
    pfile.setMetadata(meta);
    OutputStream ios = pfile.getOutputStream();
    ios.write(testString.getBytes(Charset.forName("UTF-8")));
    ios.close();
    assertEquals(testString.getBytes(Charset.forName("UTF-8")).length, pfile.getFileLength());
}

Example 29 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class Blobstore method createCli.

private static void createCli(String[] args) throws Exception {
    Map<String, Object> cl = CLI.opt("f", "file", null, CLI.AS_STRING).opt("a", "acl", Collections.emptyList(), new AsAclParser()).opt("r", "replication-factor", -1, CLI.AS_INT).arg("key", CLI.FIRST_WINS).parse(args);
    final String key = (String) cl.get("key");
    final String file = (String) cl.get("f");
    final List<AccessControl> acl = (List<AccessControl>) cl.get("a");
    final Integer replicationFactor = (Integer) cl.get("r");
    SettableBlobMeta meta = new SettableBlobMeta(acl);
    meta.set_replication_factor(replicationFactor);
    Utils.validateKeyName(key);
    LOG.info("Creating {} with ACL {}", key, generateAccessControlsInfo(acl));
    if (StringUtils.isNotEmpty(file)) {
        try (BufferedInputStream f = new BufferedInputStream(new FileInputStream(file))) {
            BlobStoreSupport.createBlobFromStream(key, f, meta);
        }
    } else {
        BlobStoreSupport.createBlobFromStream(key, System.in, meta);
    }
    LOG.info("Successfully created {}", key);
}

Example 30 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class LocalFsBlobStore method updateBlob.

@Override
public AtomicOutputStream updateBlob(String key, Subject who) throws AuthorizationException, KeyNotFoundException {
    validateKey(key);
    checkForBlobOrDownload(key);
    SettableBlobMeta meta = getStoredBlobMeta(key);
    _aclHandler.hasPermissions(meta.get_acl(), WRITE, who, key);
    try {
        return new BlobStoreFileOutputStream(fbs.write(DATA_PREFIX + key, false));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}