Examples with SettableBlobMeta org.apache.storm.generated.SettableBlobMeta used on opensource projects

Example 41 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class BlobStoreTest method testMultiple.

public void testMultiple(BlobStore store) throws Exception {
    assertStoreHasExactly(store);
    LOG.info("Creating test");
    try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) {
        out.write(1);
    }
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 1);
    LOG.info("Creating other");
    try (AtomicOutputStream out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) {
        out.write(2);
    }
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 1);
    readAssertEquals(store, "other", 2);
    LOG.info("Updating other");
    try (AtomicOutputStream out = store.updateBlob("other", null)) {
        out.write(5);
    }
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 1);
    readAssertEquals(store, "other", 5);
    LOG.info("Deleting test");
    store.deleteBlob("test", null);
    assertStoreHasExactly(store, "other");
    readAssertEquals(store, "other", 5);
    LOG.info("Creating test again");
    try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) {
        out.write(2);
    }
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 2);
    readAssertEquals(store, "other", 5);
    LOG.info("Updating test");
    try (AtomicOutputStream out = store.updateBlob("test", null)) {
        out.write(3);
    }
    assertStoreHasExactly(store, "test", "other");
    readAssertEquals(store, "test", 3);
    readAssertEquals(store, "other", 5);
    LOG.info("Deleting other");
    store.deleteBlob("other", null);
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    LOG.info("Updating test again");
    // intended to not guarding with try-with-resource since otherwise test will fail
    AtomicOutputStream out = store.updateBlob("test", null);
    out.write(4);
    out.flush();
    LOG.info("SLEEPING");
    Thread.sleep(2);
    if (store instanceof LocalFsBlobStore) {
        ((LocalFsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    try {
        out.close();
    } catch (IOException e) {
    // This is likely to happen when we try to commit something that
    // was cleaned up.  This is expected and acceptable.
    }
}

Example 42 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class BlobStoreTest method testBasic.

public void testBasic(BlobStore store) throws Exception {
    assertStoreHasExactly(store);
    LOG.info("Creating test");
    // Tests for case when subject == null (security turned off) and
    // acls for the blob are set to WORLD_EVERYTHING
    SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
        out.write(1);
    }
    assertStoreHasExactly(store, "test");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEquals(store, "test", 1);
    LOG.info("Deleting test");
    store.deleteBlob("test", null);
    assertStoreHasExactly(store);
    // The following tests are run for both hdfs and local store to test the
    // update blob interface
    metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    LOG.info("Creating test again");
    try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
        out.write(2);
    }
    assertStoreHasExactly(store, "test");
    if (store instanceof LocalFsBlobStore) {
        assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    }
    readAssertEquals(store, "test", 2);
    LOG.info("Updating test");
    try (AtomicOutputStream out = store.updateBlob("test", null)) {
        out.write(3);
    }
    assertStoreHasExactly(store, "test");
    readAssertEquals(store, "test", 3);
    LOG.info("Updating test again");
    try (AtomicOutputStream out = store.updateBlob("test", null)) {
        out.write(4);
        out.flush();
        LOG.info("SLEEPING");
        Thread.sleep(2);
    }
    // acls for the blob are set to DEFAULT (Empty ACL List) only for LocalFsBlobstore
    if (store instanceof LocalFsBlobStore) {
        metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
        LOG.info("Creating test for empty acls when security is off");
        try (AtomicOutputStream out = store.createBlob("test-empty-acls", metadata, null)) {
            LOG.info("metadata {}", metadata);
            out.write(2);
        }
        assertStoreHasExactly(store, "test-empty-acls", "test");
        // Testing whether acls are set to WORLD_EVERYTHING, Here we are testing only for LocalFsBlobstore
        // as the HdfsBlobstore gets the subject information of the local system user and behaves as it is
        // always authenticated.
        assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.get_acl().toString().contains("OTHER"));
        LOG.info("Deleting test-empty-acls");
        store.deleteBlob("test-empty-acls", null);
    }
    if (store instanceof LocalFsBlobStore) {
        ((LocalFsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
}

Example 43 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class BlobStoreTest method testWithAuthentication.

// Check for Blobstore with authentication
public void testWithAuthentication(BlobStore store) throws Exception {
    //Test for Nimbus Admin
    Subject admin = getSubject("admin");
    assertStoreHasExactly(store);
    SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    try (AtomicOutputStream out = store.createBlob("test", metadata, admin)) {
        assertStoreHasExactly(store, "test");
        out.write(1);
    }
    store.deleteBlob("test", admin);
    //Test for Supervisor Admin
    Subject supervisor = getSubject("supervisor");
    assertStoreHasExactly(store);
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    try (AtomicOutputStream out = store.createBlob("test", metadata, supervisor)) {
        assertStoreHasExactly(store, "test");
        out.write(1);
    }
    store.deleteBlob("test", supervisor);
    //Test for Nimbus itself as a user
    Subject nimbus = getNimbusSubject();
    assertStoreHasExactly(store);
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    try (AtomicOutputStream out = store.createBlob("test", metadata, nimbus)) {
        assertStoreHasExactly(store, "test");
        out.write(1);
    }
    store.deleteBlob("test", nimbus);
    // Test with a dummy test_subject for cases where subject !=null (security turned on)
    Subject who = getSubject("test_subject");
    assertStoreHasExactly(store);
    // Tests for case when subject != null (security turned on) and
    // acls for the blob are set to WORLD_EVERYTHING
    metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
        out.write(1);
    }
    assertStoreHasExactly(store, "test");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test", 1);
    LOG.info("Deleting test");
    store.deleteBlob("test", who);
    assertStoreHasExactly(store);
    // Tests for case when subject != null (security turned on) and
    // acls are not set for the blob (DEFAULT)
    LOG.info("Creating test again");
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
        out.write(2);
    }
    assertStoreHasExactly(store, "test");
    // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
    // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
    // complete access to the blob
    assertTrue("ACL does not contain WORLD_EVERYTHING", !metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test", 2);
    LOG.info("Updating test");
    try (AtomicOutputStream out = store.updateBlob("test", who)) {
        out.write(3);
    }
    assertStoreHasExactly(store, "test");
    readAssertEqualsWithAuth(store, who, "test", 3);
    LOG.info("Updating test again");
    try (AtomicOutputStream out = store.updateBlob("test", who)) {
        out.write(4);
        out.flush();
        LOG.info("SLEEPING");
        Thread.sleep(2);
        assertStoreHasExactly(store, "test");
        readAssertEqualsWithAuth(store, who, "test", 3);
    }
    // Test for subject with no principals and acls set to WORLD_EVERYTHING
    who = new Subject();
    metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
    LOG.info("Creating test");
    try (AtomicOutputStream out = store.createBlob("test-empty-subject-WE", metadata, who)) {
        out.write(2);
    }
    assertStoreHasExactly(store, "test-empty-subject-WE", "test");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2);
    // Test for subject with no principals and acls set to DEFAULT
    who = new Subject();
    metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
    LOG.info("Creating other");
    try (AtomicOutputStream out = store.createBlob("test-empty-subject-DEF", metadata, who)) {
        out.write(2);
    }
    assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE");
    // Testing whether acls are set to WORLD_EVERYTHING
    assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
    readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2);
    if (store instanceof LocalFsBlobStore) {
        ((LocalFsBlobStore) store).fullCleanup(1);
    } else {
        fail("Error the blobstore is of unknowntype");
    }
}

Example 44 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class BlobStoreTest method testWithAuthenticationNoPrincipal.

@ParameterizedTest
@ValueSource(booleans = { true, false })
void testWithAuthenticationNoPrincipal(boolean securityEnabled) throws Exception {
    try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore-auth-no-principal-sec-" + securityEnabled)) {
        BlobStore store = container.blobStore;
        // Test for subject with no principals
        Subject who = new Subject();
        assertStoreHasExactly(store);
        // Tests for case when subject != null (security turned on) and
        // acls for the blob are set to WORLD_EVERYTHING
        SettableBlobMeta metadata = new SettableBlobMeta(securityEnabled ? BlobStoreAclHandler.DEFAULT : BlobStoreAclHandler.WORLD_EVERYTHING);
        try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
            out.write(1);
        }
        assertStoreHasExactly(store, "test");
        // With no principals in the subject ACL should always be set to WORLD_EVERYTHING
        assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
        readAssertEqualsWithAuth(store, who, "test", 1);
    }
}

Example 45 with SettableBlobMeta

use of org.apache.storm.generated.SettableBlobMeta in project storm by apache.

the class HdfsBlobStoreImplTest method testMultiple.

// Be careful about adding additional tests as the dfscluster will be shared
@Test
public void testMultiple() throws Exception {
    String testString = "testingblob";
    String validKey = "validkeyBasic";
    // Will be closed automatically when shutting down the DFS cluster
    FileSystem fs = DFS_CLUSTER_RULE.getDfscluster().getFileSystem();
    Map<String, Object> conf = new HashMap<>();
    try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, DFS_CLUSTER_RULE.getHadoopConf())) {
        // should have created blobDir
        assertTrue("BlobStore dir wasn't created", fs.exists(blobDir));
        assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir).getPermission());
        // test exist with non-existent key
        assertFalse("file exists but shouldn't", hbs.exists("bogus"));
        // test write
        BlobStoreFile pfile = hbs.write(validKey, false);
        // Adding metadata to avoid null pointer exception
        SettableBlobMeta meta = new SettableBlobMeta();
        meta.set_replication_factor(1);
        pfile.setMetadata(meta);
        try (OutputStream ios = pfile.getOutputStream()) {
            ios.write(testString.getBytes(StandardCharsets.UTF_8));
        }
        // test modTime can change
        Long initialModTime = pfile.getModTime();
        try (OutputStream ios = pfile.getOutputStream()) {
            ios.write(testString.getBytes(StandardCharsets.UTF_8));
        }
        Long nextModTime = pfile.getModTime();
        assertTrue(nextModTime > initialModTime);
        // test commit creates properly
        assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
        pfile.commit();
        Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA);
        assertTrue("blob data not committed", fs.exists(dataFile));
        assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
        assertTrue("key doesn't exist but should", hbs.exists(validKey));
        // test read
        BlobStoreFile readpFile = hbs.read(validKey);
        try (InputStream inStream = readpFile.getInputStream()) {
            String readString = IOUtils.toString(inStream, StandardCharsets.UTF_8);
            assertEquals("string read from blob doesn't match", testString, readString);
        }
        // test listkeys
        Iterator<String> keys = hbs.listKeys();
        assertTrue("blob has one key", keys.hasNext());
        assertEquals("one key in blobstore", validKey, keys.next());
        // delete
        hbs.deleteKey(validKey);
        assertFalse("key not deleted", fs.exists(dataFile));
        assertFalse("key not deleted", hbs.exists(validKey));
        // Now do multiple
        String testString2 = "testingblob2";
        String validKey2 = "validkey2";
        // test write
        pfile = hbs.write(validKey, false);
        pfile.setMetadata(meta);
        try (OutputStream ios = pfile.getOutputStream()) {
            ios.write(testString.getBytes(StandardCharsets.UTF_8));
        }
        // test commit creates properly
        assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
        pfile.commit();
        assertTrue("blob data not committed", fs.exists(dataFile));
        assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
        assertTrue("key doesn't exist but should", hbs.exists(validKey));
        // test write again
        pfile = hbs.write(validKey2, false);
        pfile.setMetadata(meta);
        try (OutputStream ios2 = pfile.getOutputStream()) {
            ios2.write(testString2.getBytes(StandardCharsets.UTF_8));
        }
        // test commit second creates properly
        pfile.commit();
        Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA);
        assertTrue("blob data not committed", fs.exists(dataFile2));
        assertEquals("BlobStore dir was created with wrong permissions", HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2).getPermission());
        assertTrue("key doesn't exist but should", hbs.exists(validKey2));
        // test listkeys
        keys = hbs.listKeys();
        int total = 0;
        boolean key1Found = false;
        boolean key2Found = false;
        while (keys.hasNext()) {
            total++;
            String key = keys.next();
            if (key.equals(validKey)) {
                key1Found = true;
            } else if (key.equals(validKey2)) {
                key2Found = true;
            } else {
                fail("Found key that wasn't expected: " + key);
            }
        }
        assertEquals("number of keys is wrong", 2, total);
        assertTrue("blobstore missing key1", key1Found);
        assertTrue("blobstore missing key2", key2Found);
        // test read
        readpFile = hbs.read(validKey);
        try (InputStream inStream = readpFile.getInputStream()) {
            String readString = IOUtils.toString(inStream, StandardCharsets.UTF_8);
            assertEquals("string read from blob doesn't match", testString, readString);
        }
        // test read
        readpFile = hbs.read(validKey2);
        try (InputStream inStream = readpFile.getInputStream()) {
            String readString = IOUtils.toString(inStream, StandardCharsets.UTF_8);
            assertEquals("string read from blob doesn't match", testString2, readString);
        }
        hbs.deleteKey(validKey);
        assertFalse("key not deleted", hbs.exists(validKey));
        hbs.deleteKey(validKey2);
        assertFalse("key not deleted", hbs.exists(validKey2));
    }
}