Examples with ChecksumBlobStoreFormat org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat used on opensource projects

Example 1 with ChecksumBlobStoreFormat

use of org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat in project elasticsearch by elastic.

the class BlobStoreFormatIT method testCompressionIsApplied.

public void testCompressionIsApplied() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    StringBuilder veryRedundantText = new StringBuilder();
    for (int i = 0; i < randomIntBetween(100, 300); i++) {
        veryRedundantText.append("Blah ");
    }
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    BlobObj blobObj = new BlobObj(veryRedundantText.toString());
    checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
    checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
    Map<String, BlobMetaData> blobs = blobContainer.listBlobsByPrefix("blob-");
    assertEquals(blobs.size(), 2);
    assertThat(blobs.get("blob-not-comp").length(), greaterThan(blobs.get("blob-comp").length()));
}

Example 2 with ChecksumBlobStoreFormat

use of org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat in project elasticsearch by elastic.

the class BlobStoreFormatIT method testAtomicWrite.

public void testAtomicWrite() throws Exception {
    final BlobStore blobStore = createTestBlobStore();
    final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    String testString = randomAsciiOfLength(randomInt(10000));
    final CountDownLatch block = new CountDownLatch(1);
    final CountDownLatch unblock = new CountDownLatch(1);
    final BlobObj blobObj = new BlobObj(testString) {

        @Override
        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
            super.toXContent(builder, params);
            // Block before finishing writing
            try {
                block.countDown();
                unblock.await(5, TimeUnit.SECONDS);
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
            }
            return builder;
        }
    };
    final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ExecutorService threadPool = Executors.newFixedThreadPool(1);
    try {
        Future<Void> future = threadPool.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                checksumFormat.writeAtomic(blobObj, blobContainer, "test-blob");
                return null;
            }
        });
        block.await(5, TimeUnit.SECONDS);
        assertFalse(blobContainer.blobExists("test-blob"));
        unblock.countDown();
        future.get();
        assertTrue(blobContainer.blobExists("test-blob"));
    } finally {
        threadPool.shutdown();
    }
}

Example 3 with ChecksumBlobStoreFormat

use of org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat in project elasticsearch by elastic.

the class BlobStoreFormatIT method testBlobCorruption.

public void testBlobCorruption() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    String testString = randomAsciiOfLength(randomInt(10000));
    BlobObj blobObj = new BlobObj(testString);
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    checksumFormat.write(blobObj, blobContainer, "test-path");
    assertEquals(checksumFormat.read(blobContainer, "test-path").getText(), testString);
    randomCorruption(blobContainer, "test-path");
    try {
        checksumFormat.read(blobContainer, "test-path");
        fail("Should have failed due to corruption");
    } catch (ElasticsearchCorruptionException ex) {
        assertThat(ex.getMessage(), containsString("test-path"));
    } catch (EOFException ex) {
    // This can happen if corrupt the byte length
    }
}

Example 4 with ChecksumBlobStoreFormat

use of org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat in project elasticsearch by elastic.

the class BlobStoreFormatIT method testBlobStoreOperations.

public void testBlobStoreOperations() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), false, XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), false, XContentType.SMILE);
    ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent, xContentRegistry(), true, XContentType.SMILE);
    // Write blobs in different formats
    checksumJSON.write(new BlobObj("checksum json"), blobContainer, "check-json");
    checksumSMILE.write(new BlobObj("checksum smile"), blobContainer, "check-smile");
    checksumSMILECompressed.write(new BlobObj("checksum smile compressed"), blobContainer, "check-smile-comp");
    // Assert that all checksum blobs can be read by all formats
    assertEquals(checksumJSON.read(blobContainer, "check-json").getText(), "checksum json");
    assertEquals(checksumSMILE.read(blobContainer, "check-json").getText(), "checksum json");
    assertEquals(checksumJSON.read(blobContainer, "check-smile").getText(), "checksum smile");
    assertEquals(checksumSMILE.read(blobContainer, "check-smile").getText(), "checksum smile");
    assertEquals(checksumJSON.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
    assertEquals(checksumSMILE.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
}