Examples with BytesReference org.opensearch.common.bytes.BytesReference used on opensource projects

Example 11 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class SnapshotCustomPluginStateIT method testIncludeGlobalState.

public void testIncludeGlobalState() throws Exception {
    createRepository("test-repo", "fs");
    boolean testTemplate = randomBoolean();
    boolean testPipeline = randomBoolean();
    // At least something should be stored
    boolean testScript = (testTemplate == false && testPipeline == false) || randomBoolean();
    if (testTemplate) {
        logger.info("-->  creating test template");
        assertThat(client().admin().indices().preparePutTemplate("test-template").setPatterns(Collections.singletonList("te*")).setMapping(XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("field1").field("type", "text").field("store", true).endObject().startObject("field2").field("type", "keyword").field("store", true).endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
    }
    if (testPipeline) {
        logger.info("-->  creating test pipeline");
        BytesReference pipelineSource = BytesReference.bytes(jsonBuilder().startObject().field("description", "my_pipeline").startArray("processors").startObject().startObject("test").endObject().endObject().endArray().endObject());
        assertAcked(clusterAdmin().preparePutPipeline("barbaz", pipelineSource, XContentType.JSON).get());
    }
    if (testScript) {
        logger.info("-->  creating test script");
        assertAcked(clusterAdmin().preparePutStoredScript().setId("foobar").setContent(new BytesArray("{\"script\": { \"lang\": \"" + MockScriptEngine.NAME + "\", \"source\": \"1\"} }"), XContentType.JSON));
    }
    logger.info("--> snapshot without global state");
    CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap-no-global-state").setIndices().setIncludeGlobalState(false).setWaitForCompletion(true).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
    assertThat(getSnapshot("test-repo", "test-snap-no-global-state").state(), equalTo(SnapshotState.SUCCESS));
    SnapshotsStatusResponse snapshotsStatusResponse = clusterAdmin().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-no-global-state").get();
    assertThat(snapshotsStatusResponse.getSnapshots().size(), equalTo(1));
    SnapshotStatus snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
    assertThat(snapshotStatus.includeGlobalState(), equalTo(false));
    logger.info("--> snapshot with global state");
    createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap-with-global-state").setIndices().setIncludeGlobalState(true).setWaitForCompletion(true).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
    assertThat(getSnapshot("test-repo", "test-snap-with-global-state").state(), equalTo(SnapshotState.SUCCESS));
    snapshotsStatusResponse = clusterAdmin().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-with-global-state").get();
    assertThat(snapshotsStatusResponse.getSnapshots().size(), equalTo(1));
    snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
    assertThat(snapshotStatus.includeGlobalState(), equalTo(true));
    if (testTemplate) {
        logger.info("-->  delete test template");
        cluster().wipeTemplates("test-template");
        GetIndexTemplatesResponse getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
        assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
    }
    if (testPipeline) {
        logger.info("-->  delete test pipeline");
        assertAcked(clusterAdmin().deletePipeline(new DeletePipelineRequest("barbaz")).get());
    }
    if (testScript) {
        logger.info("-->  delete test script");
        assertAcked(clusterAdmin().prepareDeleteStoredScript("foobar").get());
    }
    logger.info("--> try restoring cluster state from snapshot without global state");
    RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap-no-global-state").setWaitForCompletion(true).setRestoreGlobalState(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(0));
    logger.info("--> check that template wasn't restored");
    GetIndexTemplatesResponse getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
    assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
    logger.info("--> restore cluster state");
    restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap-with-global-state").setWaitForCompletion(true).setRestoreGlobalState(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(0));
    if (testTemplate) {
        logger.info("--> check that template is restored");
        getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
        assertIndexTemplateExists(getIndexTemplatesResponse, "test-template");
    }
    if (testPipeline) {
        logger.info("--> check that pipeline is restored");
        GetPipelineResponse getPipelineResponse = clusterAdmin().prepareGetPipeline("barbaz").get();
        assertTrue(getPipelineResponse.isFound());
    }
    if (testScript) {
        logger.info("--> check that script is restored");
        GetStoredScriptResponse getStoredScriptResponse = clusterAdmin().prepareGetStoredScript("foobar").get();
        assertNotNull(getStoredScriptResponse.getSource());
    }
    createIndexWithRandomDocs("test-idx", 100);
    logger.info("--> snapshot without global state but with indices");
    createSnapshotResponse = clusterAdmin().prepareCreateSnapshot("test-repo", "test-snap-no-global-state-with-index").setIndices("test-idx").setIncludeGlobalState(false).setWaitForCompletion(true).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(getSnapshot("test-repo", "test-snap-no-global-state-with-index").state(), equalTo(SnapshotState.SUCCESS));
    logger.info("-->  delete global state and index ");
    cluster().wipeIndices("test-idx");
    if (testTemplate) {
        cluster().wipeTemplates("test-template");
    }
    if (testPipeline) {
        assertAcked(clusterAdmin().deletePipeline(new DeletePipelineRequest("barbaz")).get());
    }
    if (testScript) {
        assertAcked(clusterAdmin().prepareDeleteStoredScript("foobar").get());
    }
    getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
    assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
    logger.info("--> try restoring index and cluster state from snapshot without global state");
    restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "test-snap-no-global-state-with-index").setWaitForCompletion(true).setRestoreGlobalState(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
    logger.info("--> check that global state wasn't restored but index was");
    getIndexTemplatesResponse = client().admin().indices().prepareGetTemplates().get();
    assertIndexTemplateMissing(getIndexTemplatesResponse, "test-template");
    assertFalse(clusterAdmin().prepareGetPipeline("barbaz").get().isFound());
    assertNull(clusterAdmin().prepareGetStoredScript("foobar").get().getSource());
    assertDocCount("test-idx", 100L);
}

Example 12 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class IndexRequest method source.

/**
 * Sets the content source to index.
 * <p>
 * <b>Note: the number of objects passed to this method as varargs must be an even
 * number. Also the first argument in each pair (the field name) must have a
 * valid String representation.</b>
 * </p>
 */
public IndexRequest source(XContentType xContentType, Object... source) {
    if (source.length % 2 != 0) {
        throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]");
    }
    if (source.length == 2 && source[0] instanceof BytesReference && source[1] instanceof Boolean) {
        throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag" + " was removed, please just use source(BytesReference)");
    }
    try {
        XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
        builder.startObject();
        for (int i = 0; i < source.length; i++) {
            builder.field(source[i++].toString(), source[i]);
        }
        builder.endObject();
        return source(builder);
    } catch (IOException e) {
        throw new OpenSearchGenerationException("Failed to generate", e);
    }
}

Example 13 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class StreamOutput method writeText.

public void writeText(Text text) throws IOException {
    if (!text.hasBytes()) {
        final String string = text.string();
        spare.copyChars(string);
        writeInt(spare.length());
        write(spare.bytes(), 0, spare.length());
    } else {
        BytesReference bytes = text.bytes();
        writeInt(bytes.length());
        bytes.writeTo(this);
    }
}

Example 14 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class IndicesService method cacheShardLevelResult.

/**
 * Cache something calculated at the shard level.
 * @param shard the shard this item is part of
 * @param reader a reader for this shard. Used to invalidate the cache when there are changes.
 * @param cacheKey key for the thing being cached within this shard
 * @param loader loads the data into the cache if needed
 * @return the contents of the cache or the result of calling the loader
 */
private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey, CheckedConsumer<StreamOutput, IOException> loader) throws Exception {
    IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard);
    CheckedSupplier<BytesReference, IOException> supplier = () -> {
        /* BytesStreamOutput allows to pass the expected size but by default uses
             * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie.
             * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful
             * since we don't shrink to the actual size once we are done serializing.
             * By passing 512 as the expected size we will resize the byte array in the stream
             * slowly until we hit the page size and don't waste too much memory for small query
             * results.*/
        final int expectedSizeInBytes = 512;
        try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) {
            loader.accept(out);
            // the memory properly paged instead of having varied sized bytes
            return out.bytes();
        }
    };
    return indicesRequestCache.getOrCompute(cacheEntity, supplier, reader, cacheKey);
}

Example 15 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class IndicesRequestCache method getOrCompute.

BytesReference getOrCompute(CacheEntity cacheEntity, CheckedSupplier<BytesReference, IOException> loader, DirectoryReader reader, BytesReference cacheKey) throws Exception {
    assert reader.getReaderCacheHelper() != null;
    final Key key = new Key(cacheEntity, reader.getReaderCacheHelper().getKey(), cacheKey);
    Loader cacheLoader = new Loader(cacheEntity, loader);
    BytesReference value = cache.computeIfAbsent(key, cacheLoader);
    if (cacheLoader.isLoaded()) {
        key.entity.onMiss();
        // see if its the first time we see this reader, and make sure to register a cleanup key
        CleanupKey cleanupKey = new CleanupKey(cacheEntity, reader.getReaderCacheHelper().getKey());
        if (!registeredClosedListeners.containsKey(cleanupKey)) {
            Boolean previous = registeredClosedListeners.putIfAbsent(cleanupKey, Boolean.TRUE);
            if (previous == null) {
                OpenSearchDirectoryReader.addReaderCloseListener(reader, cleanupKey);
            }
        }
    } else {
        key.entity.onHit();
    }
    return value;
}