Examples with ShadowIndexShard org.elasticsearch.index.shard.ShadowIndexShard used on opensource projects

Example 1 with ShadowIndexShard

use of org.elasticsearch.index.shard.ShadowIndexShard in project elasticsearch by elastic.

the class IndexWithShadowReplicasIT method testRestoreToShadow.

/**
     * Tests the case where we create an index without shadow replicas, snapshot it and then restore into
     * an index with shadow replicas enabled.
     */
public void testRestoreToShadow() throws ExecutionException, InterruptedException {
    final Path dataPath = createTempDir();
    Settings nodeSettings = nodeSettings(dataPath);
    internalCluster().startNodes(3, nodeSettings);
    Settings idxSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
    assertAcked(prepareCreate("foo").setSettings(idxSettings));
    ensureGreen();
    final int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        client().prepareIndex("foo", "doc", "" + i).setSource("foo", "bar").get();
    }
    assertNoFailures(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
    assertAcked(client().admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())));
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("foo").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    Settings shadowSettings = Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString()).put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2).build();
    logger.info("--> restore the index into shadow replica index");
    RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setIndexSettings(shadowSettings).setWaitForCompletion(true).setRenamePattern("(.+)").setRenameReplacement("$1-copy").execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    refresh();
    Index index = resolveIndex("foo-copy");
    for (IndicesService service : internalCluster().getDataNodeInstances(IndicesService.class)) {
        if (service.hasIndex(index)) {
            IndexShard shard = service.indexServiceSafe(index).getShardOrNull(0);
            if (shard.routingEntry().primary()) {
                assertFalse(shard instanceof ShadowIndexShard);
            } else {
                assertTrue(shard instanceof ShadowIndexShard);
            }
        }
    }
    logger.info("--> performing query");
    SearchResponse resp = client().prepareSearch("foo-copy").setQuery(matchAllQuery()).get();
    assertHitCount(resp, numDocs);
}

Example 2 with ShadowIndexShard

use of org.elasticsearch.index.shard.ShadowIndexShard in project elasticsearch by elastic.

the class IndexWithShadowReplicasIT method testIndexWithFewDocuments.

@TestLogging("org.elasticsearch.gateway:TRACE")
public void testIndexWithFewDocuments() throws Exception {
    final Path dataPath = createTempDir();
    Settings nodeSettings = nodeSettings(dataPath);
    internalCluster().startNodes(3, nodeSettings);
    final String IDX = "test";
    Settings idxSettings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2).put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)).put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString()).put(IndexMetaData.SETTING_SHADOW_REPLICAS, true).put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true).build();
    prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
    ensureGreen(IDX);
    // So basically, the primary should fail and the replica will need to
    // replay the translog, this is what this tests
    client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();
    IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(IDX).clear().setTranslog(true).get();
    assertEquals(2, indicesStatsResponse.getIndex(IDX).getPrimaries().getTranslog().estimatedNumberOfOperations());
    assertEquals(2, indicesStatsResponse.getIndex(IDX).getTotal().getTranslog().estimatedNumberOfOperations());
    Index index = resolveIndex(IDX);
    for (IndicesService service : internalCluster().getInstances(IndicesService.class)) {
        IndexService indexService = service.indexService(index);
        if (indexService != null) {
            IndexShard shard = indexService.getShard(0);
            TranslogStats translogStats = shard.translogStats();
            assertTrue(translogStats != null || shard instanceof ShadowIndexShard);
            if (translogStats != null) {
                assertEquals(2, translogStats.estimatedNumberOfOperations());
            }
        }
    }
    // Check that we can get doc 1 and 2, because we are doing realtime
    // gets and getting from the primary
    GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
    GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
    flushAndRefresh(IDX);
    client().prepareIndex(IDX, "doc", "3").setSource("foo", "bar").get();
    client().prepareIndex(IDX, "doc", "4").setSource("foo", "bar").get();
    refresh();
    // Check that we can get doc 1 and 2 without realtime
    gResp1 = client().prepareGet(IDX, "doc", "1").setRealtime(false).get();
    gResp2 = client().prepareGet(IDX, "doc", "2").setRealtime(false).get();
    assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
    assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
    logger.info("--> restarting all nodes");
    if (randomBoolean()) {
        logger.info("--> rolling restart");
        internalCluster().rollingRestart();
    } else {
        logger.info("--> full restart");
        internalCluster().fullRestart();
    }
    client().admin().cluster().prepareHealth().setWaitForNodes("3").get();
    ensureGreen(IDX);
    flushAndRefresh(IDX);
    logger.info("--> performing query");
    SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
    assertHitCount(resp, 4);
    logger.info("--> deleting index");
    assertAcked(client().admin().indices().prepareDelete(IDX));
}

Example 3 with ShadowIndexShard

use of org.elasticsearch.index.shard.ShadowIndexShard in project elasticsearch by elastic.

the class IndexService method createShard.

public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
    final boolean primary = routing.primary();
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
        final boolean canDeleteShardContent = this.indexSettings.isOnSharedFilesystem() == false || (primary && this.indexSettings.isOnSharedFilesystem());
        final Engine.Warmer engineWarmer = (searcher) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
                warmer.warm(searcher, shard, IndexService.this.indexSettings);
            }
        };
        store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
        if (useShadowEngine(primary, this.indexSettings)) {
            indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners);
        // no indexing listeners - shadow  engines don't index
        } else {
            indexShard = new IndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, () -> globalCheckpointSyncer.accept(shardId), searchOperationListeners, indexingOperationListeners);
        }
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}