Examples with SettableBlobMeta backtype.storm.generated.SettableBlobMeta used on opensource projects

Example 6 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class ServiceHandler method createOrUpdateBlob.

private void createOrUpdateBlob(String key, InputStream stream, boolean update, String topologyId) throws Exception {
    StormClusterState clusterState = data.getStormClusterState();
    BlobStore blobStore = data.getBlobStore();
    NimbusInfo nimbusInfo = data.getNimbusHostPortInfo();
    if (update) {
        BlobStoreUtils.updateBlob(blobStore, key, stream);
        LOG.info("Successfully updated blobstore for topology:{}, key:{}", topologyId, key);
    } else {
        blobStore.createBlob(key, stream, new SettableBlobMeta());
        LOG.info("Successfully created blobstore for topology:{}, key:{}", topologyId, key);
    }
    if (blobStore instanceof LocalFsBlobStore) {
        clusterState.setup_blobstore(key, nimbusInfo, BlobStoreUtils.getVersionForKey(key, nimbusInfo, conf));
    }
}

Example 7 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class BlobStoreTest method testGetFileLength.

@Test
public void testGetFileLength() throws AuthorizationException, KeyNotFoundException, KeyAlreadyExistsException, IOException {
    LocalFsBlobStore store = initLocalFs();
    AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta());
    out.write(1);
    out.close();
    assertEquals(1, store.getBlob("test").getFileLength());
}

Example 8 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class BlobStoreTest method testBasic.

// Check for Blobstore with authentication
// public void testWithAuthentication(BlobStore store) throws Exception {
// //Test for Nimbus Admin
// Subject admin = getSubject("admin");
// assertStoreHasExactly(store);
// SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
// AtomicOutputStream out = store.createBlob("test", metadata, admin);
// assertStoreHasExactly(store, "test");
// out.write(1);
// out.close();
// store.deleteBlob("test", admin);
// 
// //Test for Supervisor Admin
// Subject supervisor = getSubject("supervisor");
// assertStoreHasExactly(store);
// metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
// out = store.createBlob("test", metadata, supervisor);
// assertStoreHasExactly(store, "test");
// out.write(1);
// out.close();
// store.deleteBlob("test", supervisor);
// 
// //Test for Nimbus itself as a user
// Subject nimbus = getNimbusSubject();
// assertStoreHasExactly(store);
// metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
// out = store.createBlob("test", metadata, nimbus);
// assertStoreHasExactly(store, "test");
// out.write(1);
// out.close();
// store.deleteBlob("test", nimbus);
// 
// // Test with a dummy test_subject for cases where subject !=null (security turned on)
// Subject who = getSubject("test_subject");
// assertStoreHasExactly(store);
// 
// // Tests for case when subject != null (security turned on) and
// // acls for the blob are set to WORLD_EVERYTHING
// metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
// out = store.createBlob("test", metadata, who);
// out.write(1);
// out.close();
// assertStoreHasExactly(store, "test");
// // Testing whether acls are set to WORLD_EVERYTHING
// assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
// readAssertEqualsWithAuth(store, who, "test", 1);
// 
// LOG.info("Deleting test");
// store.deleteBlob("test", who);
// assertStoreHasExactly(store);
// 
// // Tests for case when subject != null (security turned on) and
// // acls are not set for the blob (DEFAULT)
// LOG.info("Creating test again");
// metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
// out = store.createBlob("test", metadata, who);
// out.write(2);
// out.close();
// assertStoreHasExactly(store, "test");
// // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
// // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
// // complete access to the blob
// assertTrue("ACL does not contain WORLD_EVERYTHING", !metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
// readAssertEqualsWithAuth(store, who, "test", 2);
// 
// LOG.info("Updating test");
// out = store.updateBlob("test", who);
// out.write(3);
// out.close();
// assertStoreHasExactly(store, "test");
// readAssertEqualsWithAuth(store, who, "test", 3);
// 
// LOG.info("Updating test again");
// out = store.updateBlob("test", who);
// out.write(4);
// out.flush();
// LOG.info("SLEEPING");
// Thread.sleep(2);
// assertStoreHasExactly(store, "test");
// readAssertEqualsWithAuth(store, who, "test", 3);
// 
// // Test for subject with no principals and acls set to WORLD_EVERYTHING
// who = new Subject();
// metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
// LOG.info("Creating test");
// out = store.createBlob("test-empty-subject-WE", metadata, who);
// out.write(2);
// out.close();
// assertStoreHasExactly(store, "test-empty-subject-WE", "test");
// // Testing whether acls are set to WORLD_EVERYTHING
// assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
// readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2);
// 
// // Test for subject with no principals and acls set to DEFAULT
// who = new Subject();
// metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
// LOG.info("Creating other");
// out = store.createBlob("test-empty-subject-DEF", metadata, who);
// out.write(2);
// out.close();
// assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE");
// // Testing whether acls are set to WORLD_EVERYTHING
// assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
// readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2);
// 
// if (store instanceof LocalFsBlobStore) {
// ((LocalFsBlobStore) store).fullCleanup(1);
// } else {
// fail("Error the blobstore is of unknowntype");
// }
// try {
// out.close();
// } catch (IOException e) {
// // This is likely to happen when we try to commit something that
// // was cleaned up.  This is expected and acceptable.
// }
// }
public void testBasic(BlobStore store) throws Exception {
    assertStoreHasExactly(store);
    LOG.info("Creating test");
    // Tests for case when subject == null (security turned off) and
    // acls for the blob are set to WORLD_EVERYTHING
    SettableBlobMeta metadata = new SettableBlobMeta();
    AtomicOutputStream out = store.createBlob("test", metadata);
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    // Testing whether acls are set to WORLD_EVERYTHING
    // assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEquals(store, "test", 1);
    LOG.info("Deleting test");
    store.deleteBlob("test");
    assertStoreHasExactly(store);
    // The following tests are run for both hdfs and local store to test the
    // update blob interface
    metadata = new SettableBlobMeta();
    LOG.info("Creating test again");
    out = store.createBlob("test", metadata);
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test");
    // if (store instanceof LocalFsBlobStore) {
    // assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    // }
    readAssertEquals(store, "test", 2);
    LOG.info("Updating test");
    out = store.updateBlob("test");
    out.write(3);
    out.close();
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    LOG.info("Updating test again");
    out = store.updateBlob("test");
    out.write(4);
    out.flush();
    LOG.info("SLEEPING");
    Thread.sleep(2);
    // acls for the blob are set to DEFAULT (Empty ACL List) only for LocalFsBlobstore
    if (store instanceof LocalFsBlobStore) {
        metadata = new SettableBlobMeta();
        LOG.info("Creating test for empty acls when security is off");
        out = store.createBlob("test-empty-acls", metadata);
        LOG.info("metadata {}", metadata);
        out.write(2);
        out.close();
        assertStoreHasExactly(store, "test-empty-acls", "test");
        // Testing whether acls are set to WORLD_EVERYTHING, Here we are testing only for LocalFsBlobstore
        // as the HdfsBlobstore gets the subject information of the local system user and behaves as it is
        // always authenticated.
        // assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.get_acl().toString().contains("OTHER"));
        LOG.info("Deleting test-empty-acls");
        store.deleteBlob("test-empty-acls");
    }
    if (store instanceof LocalFsBlobStore) {
        ((LocalFsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
    try {
        out.close();
    } catch (IOException e) {
    // This is likely to happen when we try to commit something that
    // was cleaned up.  This is expected and acceptable.
    }
}

Example 9 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class LocalFsBlobStore method setBlobMeta.

@Override
public void setBlobMeta(String key, SettableBlobMeta meta) throws KeyNotFoundException {
    validateKey(key);
    checkForBlobOrDownload(key);
    SettableBlobMeta orig = getStoredBlobMeta(key);
    BlobStoreFileOutputStream mOut = null;
    try {
        mOut = new BlobStoreFileOutputStream(fbs.write(META_PREFIX + key, false));
        mOut.write(JStormUtils.thriftSerialize(meta));
        mOut.close();
        mOut = null;
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (mOut != null) {
            try {
                mOut.cancel();
            } catch (IOException e) {
            //Ignored
            }
        }
    }
}

Example 10 with SettableBlobMeta

use of backtype.storm.generated.SettableBlobMeta in project jstorm by alibaba.

the class BlobStoreTest method testMultiple.

public void testMultiple(BlobStore store) throws Exception {
    assertStoreHasExactly(store);
    LOG.info("Creating test");
    AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta());
    out.write(1);
    out.close();
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 1);
    LOG.info("Creating other");
    out = store.createBlob("other", new SettableBlobMeta());
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 1);
    readAssertEquals(store, "other", 2);
    LOG.info("Updating other");
    out = store.updateBlob("other");
    out.write(5);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 1);
    readAssertEquals(store, "other", 5);
    LOG.info("Deleting test");
    store.deleteBlob("test");
    assertStoreHasExactly(store, "other");
    readAssertEquals(store, "other", 5);
    LOG.info("Creating test again");
    out = store.createBlob("test", new SettableBlobMeta());
    out.write(2);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 2);
    readAssertEquals(store, "other", 5);
    LOG.info("Updating test");
    out = store.updateBlob("test");
    out.write(3);
    out.close();
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 3);
    readAssertEquals(store, "other", 5);
    LOG.info("Deleting other");
    store.deleteBlob("other");
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    LOG.info("Updating test again");
    out = store.updateBlob("test");
    out.write(4);
    out.flush();
    LOG.info("SLEEPING");
    Thread.sleep(2);
    if (store instanceof LocalFsBlobStore) {
        ((LocalFsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    try {
        out.close();
    } catch (IOException e) {
    // This is likely to happen when we try to commit something that
    // was cleaned up.  This is expected and acceptable.
    }
}