Examples with AtomicOutputStream org.apache.storm.blobstore.AtomicOutputStream used on opensource projects

Example 1 with AtomicOutputStream

use of org.apache.storm.blobstore.AtomicOutputStream in project storm by apache.

the class BlobStoreAPIWordCountTopology method updateBlobWithContent.

// Equivalent update command on command line
// storm blobstore update --file blacklist.txt key
private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file) throws KeyNotFoundException, AuthorizationException, IOException {
    AtomicOutputStream blobOutputStream = clientBlobStore.updateBlob(blobKey);
    blobOutputStream.write(readFile(file).toString().getBytes());
    blobOutputStream.close();
}

Example 2 with AtomicOutputStream

use of org.apache.storm.blobstore.AtomicOutputStream in project storm by apache.

the class BlobStoreAPIWordCountTopology method createBlobWithContent.

// Equivalent create command on command line
// storm blobstore create --file blacklist.txt --acl o::rwa key
private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file) throws AuthorizationException, KeyAlreadyExistsException, IOException, KeyNotFoundException {
    String stringBlobACL = "o::rwa";
    AccessControl blobACL = BlobStoreAclHandler.parseAccessControl(stringBlobACL);
    List<AccessControl> acls = new LinkedList<AccessControl>();
    // more ACLs can be added here
    acls.add(blobACL);
    SettableBlobMeta settableBlobMeta = new SettableBlobMeta(acls);
    AtomicOutputStream blobStream = clientBlobStore.createBlob(blobKey, settableBlobMeta);
    blobStream.write(readFile(file).toString().getBytes());
    blobStream.close();
}

Example 3 with AtomicOutputStream

use of org.apache.storm.blobstore.AtomicOutputStream in project storm by apache.

the class BlobStoreTest method testMultiple.

public void testMultiple(BlobStore store) throws Exception {
    assertStoreHasExactly(store);
    LOG.info("Creating test");
    AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 1);
    LOG.info("Creating other");
    out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null);
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 1);
    readAssertEquals(store, "other", 2);
    LOG.info("Updating other");
    out = store.updateBlob("other", null);
    out.write(5);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 1);
    readAssertEquals(store, "other", 5);
    LOG.info("Deleting test");
    store.deleteBlob("test", null);
    assertStoreHasExactly(store, "other");
    readAssertEquals(store, "other", 5);
    LOG.info("Creating test again");
    out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null);
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 2);
    readAssertEquals(store, "other", 5);
    LOG.info("Updating test");
    out = store.updateBlob("test", null);
    out.write(3);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 3);
    readAssertEquals(store, "other", 5);
    LOG.info("Deleting other");
    store.deleteBlob("other", null);
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    LOG.info("Updating test again");
    out = store.updateBlob("test", null);
    out.write(4);
    out.flush();
    LOG.info("SLEEPING");
    Thread.sleep(2);
    if (store instanceof HdfsBlobStore) {
        ((HdfsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    try {
        out.close();
    } catch (IOException e) {
    //This is likely to happen when we try to commit something that
    // was cleaned up.  This is expected and acceptable.
    }
}

Example 4 with AtomicOutputStream

use of org.apache.storm.blobstore.AtomicOutputStream in project storm by apache.

the class BlobStoreTest method testWithAuthentication.

// Check for Blobstore with authentication
public void testWithAuthentication(BlobStore store) throws Exception {
    //Test for Nimbus Admin
    Subject admin = getSubject("admin");
    assertStoreHasExactly(store);
    SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    AtomicOutputStream out = store.createBlob("test", metadata, admin);
    assertStoreHasExactly(store, "test");
    out.write(1);
    out.close();
    store.deleteBlob("test", admin);
    //Test for Supervisor Admin
    Subject supervisor = getSubject("supervisor");
    assertStoreHasExactly(store);
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    out = store.createBlob("test", metadata, supervisor);
    assertStoreHasExactly(store, "test");
    out.write(1);
    out.close();
    store.deleteBlob("test", supervisor);
    //Test for Nimbus itself as a user
    Subject nimbus = getNimbusSubject();
    assertStoreHasExactly(store);
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    out = store.createBlob("test", metadata, nimbus);
    assertStoreHasExactly(store, "test");
    out.write(1);
    out.close();
    store.deleteBlob("test", nimbus);
    // Test with a dummy test_subject for cases where subject !=null (security turned on)
    Subject who = getSubject("test_subject");
    assertStoreHasExactly(store);
    // Tests for case when subject != null (security turned on) and
    // acls for the blob are set to WORLD_EVERYTHING
    metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    out = store.createBlob("test", metadata, who);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test", 1);
    LOG.info("Deleting test");
    store.deleteBlob("test", who);
    assertStoreHasExactly(store);
    // Tests for case when subject != null (security turned on) and
    // acls are not set for the blob (DEFAULT)
    LOG.info("Creating test again");
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    out = store.createBlob("test", metadata, who);
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test");
    // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
    // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
    // complete access to the blob
    assertTrue("ACL does not contain WORLD_EVERYTHING", !metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test", 2);
    LOG.info("Updating test");
    out = store.updateBlob("test", who);
    out.write(3);
    out.close();
    assertStoreHasExactly(store, "test");
    readAssertEqualsWithAuth(store, who, "test", 3);
    LOG.info("Updating test again");
    out = store.updateBlob("test", who);
    out.write(4);
    out.flush();
    LOG.info("SLEEPING");
    Thread.sleep(2);
    assertStoreHasExactly(store, "test");
    readAssertEqualsWithAuth(store, who, "test", 3);
    //Test for subject with no principals and acls set to WORLD_EVERYTHING
    who = new Subject();
    metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    LOG.info("Creating test");
    out = store.createBlob("test-empty-subject-WE", metadata, who);
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test-empty-subject-WE", "test");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2);
    //Test for subject with no principals and acls set to DEFAULT
    who = new Subject();
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    LOG.info("Creating other");
    out = store.createBlob("test-empty-subject-DEF", metadata, who);
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2);
    if (store instanceof HdfsBlobStore) {
        ((HdfsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
    try {
        out.close();
    } catch (IOException e) {
    //This is likely to happen when we try to commit something that
    // was cleaned up.  This is expected and acceptable.
    }
}

Example 5 with AtomicOutputStream

use of org.apache.storm.blobstore.AtomicOutputStream in project storm by apache.

the class BlobStoreTest method testReplication.

// Test for replication.
public void testReplication(String path, BlobStore store) throws Exception {
    SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    metadata.set_replication_factor(4);
    AtomicOutputStream out = store.createBlob("test", metadata, null);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    assertEquals("Blobstore replication not matching", store.getBlobReplication("test", null), 4);
    store.deleteBlob("test", null);
    //Test for replication with NIMBUS as user
    Subject admin = getSubject("admin");
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    metadata.set_replication_factor(4);
    out = store.createBlob("test", metadata, admin);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    assertEquals("Blobstore replication not matching", store.getBlobReplication("test", admin), 4);
    store.updateBlobReplication("test", 5, admin);
    assertEquals("Blobstore replication not matching", store.getBlobReplication("test", admin), 5);
    store.deleteBlob("test", admin);
    //Test for replication using SUPERVISOR access
    Subject supervisor = getSubject("supervisor");
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    metadata.set_replication_factor(4);
    out = store.createBlob("test", metadata, supervisor);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    assertEquals("Blobstore replication not matching", store.getBlobReplication("test", supervisor), 4);
    store.updateBlobReplication("test", 5, supervisor);
    assertEquals("Blobstore replication not matching", store.getBlobReplication("test", supervisor), 5);
    store.deleteBlob("test", supervisor);
    //Test for a user having read or write or admin access to read replication for a blob
    String createSubject = "createSubject";
    String writeSubject = "writeSubject";
    String adminSubject = "adminSubject";
    Subject who = getSubject(createSubject);
    AccessControl writeAccess = new AccessControl(AccessControlType.USER, READ);
    AccessControl adminAccess = new AccessControl(AccessControlType.USER, ADMIN);
    writeAccess.set_name(writeSubject);
    adminAccess.set_name(adminSubject);
    List<AccessControl> acl = Arrays.asList(writeAccess, adminAccess);
    metadata = new SettableBlobMeta(acl);
    metadata.set_replication_factor(4);
    out = store.createBlob("test", metadata, who);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    who = getSubject(writeSubject);
    assertEquals("Blobstore replication not matching", store.getBlobReplication("test", who), 4);
    //Test for a user having WRITE or ADMIN privileges to change replication of a blob
    who = getSubject(adminSubject);
    store.updateBlobReplication("test", 5, who);
    assertEquals("Blobstore replication not matching", store.getBlobReplication("test", who), 5);
    store.deleteBlob("test", getSubject(createSubject));
}