Examples with BlobContainer org.elasticsearch.common.blobstore.BlobContainer used on opensource projects

Example 11 with BlobContainer

use of org.elasticsearch.common.blobstore.BlobContainer in project elasticsearch by elastic.

the class ESBlobStoreTestCase method testContainerCreationAndDeletion.

public void testContainerCreationAndDeletion() throws IOException {
    try (BlobStore store = newBlobStore()) {
        final BlobContainer containerFoo = store.blobContainer(new BlobPath().add("foo"));
        final BlobContainer containerBar = store.blobContainer(new BlobPath().add("bar"));
        byte[] data1 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        byte[] data2 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        writeBlob(containerFoo, "test", new BytesArray(data1));
        writeBlob(containerBar, "test", new BytesArray(data2));
        assertArrayEquals(readBlobFully(containerFoo, "test", data1.length), data1);
        assertArrayEquals(readBlobFully(containerBar, "test", data2.length), data2);
        assertTrue(containerFoo.blobExists("test"));
        assertTrue(containerBar.blobExists("test"));
        store.delete(new BlobPath());
        assertFalse(containerFoo.blobExists("test"));
        assertFalse(containerBar.blobExists("test"));
    }
}

Example 12 with BlobContainer

use of org.elasticsearch.common.blobstore.BlobContainer in project elasticsearch by elastic.

the class ESBlobStoreRepositoryIntegTestCase method testIndicesDeletedFromRepository.

public void testIndicesDeletedFromRepository() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    final String repoName = "test-repo";
    createTestRepository(repoName);
    createIndex("test-idx-1", "test-idx-2", "test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 20; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    refresh();
    logger.info("--> take a snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap").setWaitForCompletion(true).get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> indexing more data");
    for (int i = 20; i < 40; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    logger.info("--> take another snapshot with only 2 of the 3 indices");
    createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap2").setWaitForCompletion(true).setIndices("test-idx-1", "test-idx-2").get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> delete a snapshot");
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, "test-snap").get());
    logger.info("--> verify index folder deleted from blob container");
    RepositoriesService repositoriesSvc = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    @SuppressWarnings("unchecked") BlobStoreRepository repository = (BlobStoreRepository) repositoriesSvc.repository(repoName);
    BlobContainer indicesBlobContainer = repository.blobStore().blobContainer(repository.basePath().add("indices"));
    RepositoryData repositoryData = repository.getRepositoryData();
    for (IndexId indexId : repositoryData.getIndices().values()) {
        if (indexId.getName().equals("test-idx-3")) {
            // deleted index
            assertFalse(indicesBlobContainer.blobExists(indexId.getId()));
        }
    }
}

Example 13 with BlobContainer

use of org.elasticsearch.common.blobstore.BlobContainer in project elasticsearch by elastic.

the class BlobStoreFormatIT method testCompressionIsApplied.

public void testCompressionIsApplied() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    StringBuilder veryRedundantText = new StringBuilder();
    for (int i = 0; i < randomIntBetween(100, 300); i++) {
        veryRedundantText.append("Blah ");
    }
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    BlobObj blobObj = new BlobObj(veryRedundantText.toString());
    checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
    checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
    Map<String, BlobMetaData> blobs = blobContainer.listBlobsByPrefix("blob-");
    assertEquals(blobs.size(), 2);
    assertThat(blobs.get("blob-not-comp").length(), greaterThan(blobs.get("blob-comp").length()));
}

Example 14 with BlobContainer

use of org.elasticsearch.common.blobstore.BlobContainer in project elasticsearch by elastic.

the class BlobStoreFormatIT method testAtomicWrite.

public void testAtomicWrite() throws Exception {
    final BlobStore blobStore = createTestBlobStore();
    final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    String testString = randomAsciiOfLength(randomInt(10000));
    final CountDownLatch block = new CountDownLatch(1);
    final CountDownLatch unblock = new CountDownLatch(1);
    final BlobObj blobObj = new BlobObj(testString) {

        @Override
        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
            super.toXContent(builder, params);
            // Block before finishing writing
            try {
                block.countDown();
                unblock.await(5, TimeUnit.SECONDS);
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
            }
            return builder;
        }
    };
    final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ExecutorService threadPool = Executors.newFixedThreadPool(1);
    try {
        Future<Void> future = threadPool.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                checksumFormat.writeAtomic(blobObj, blobContainer, "test-blob");
                return null;
            }
        });
        block.await(5, TimeUnit.SECONDS);
        assertFalse(blobContainer.blobExists("test-blob"));
        unblock.countDown();
        future.get();
        assertTrue(blobContainer.blobExists("test-blob"));
    } finally {
        threadPool.shutdown();
    }
}

Example 15 with BlobContainer

use of org.elasticsearch.common.blobstore.BlobContainer in project elasticsearch by elastic.

the class BlobStoreFormatIT method testBlobCorruption.

public void testBlobCorruption() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    String testString = randomAsciiOfLength(randomInt(10000));
    BlobObj blobObj = new BlobObj(testString);
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    checksumFormat.write(blobObj, blobContainer, "test-path");
    assertEquals(checksumFormat.read(blobContainer, "test-path").getText(), testString);
    randomCorruption(blobContainer, "test-path");
    try {
        checksumFormat.read(blobContainer, "test-path");
        fail("Should have failed due to corruption");
    } catch (ElasticsearchCorruptionException ex) {
        assertThat(ex.getMessage(), containsString("test-path"));
    } catch (EOFException ex) {
    // This can happen if corrupt the byte length
    }
}