Examples with IndexShard org.elasticsearch.index.shard.IndexShard used on opensource projects

Example 81 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class RetentionLeaseIT method testRetentionLeasesSyncOnRecovery.

@Test
public void testRetentionLeasesSyncOnRecovery() throws Exception {
    final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2);
    internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas);
    /*
         * We effectively disable the background sync to ensure that the retention leases are not synced in the background so that the only
         * source of retention leases on the replicas would be from recovery.
         */
    execute("create table doc.tbl (x int) clustered into 1 shards " + "with (" + "   number_of_replicas = 0, " + "   \"soft_deletes.enabled\" = true, " + "   \"soft_deletes.retention_lease.sync_interval\" = ?)", new Object[] { TimeValue.timeValueHours(24).getStringRep() });
    allowNodes("tbl", 1);
    ensureYellow("tbl");
    execute("alter table doc.tbl set (number_of_replicas = ?)", new Object[] { numberOfReplicas });
    final String primaryShardNodeId = clusterService().state().routingTable().index("tbl").shard(0).primaryShard().currentNodeId();
    final String primaryShardNodeName = clusterService().state().nodes().get(primaryShardNodeId).getName();
    final IndexShard primary = internalCluster().getInstance(IndicesService.class, primaryShardNodeName).getShardOrNull(new ShardId(resolveIndex("tbl"), 0));
    final int length = randomIntBetween(1, 8);
    final Map<String, RetentionLease> currentRetentionLeases = new HashMap<>();
    for (int i = 0; i < length; i++) {
        final String id = randomValueOtherThanMany(currentRetentionLeases.keySet()::contains, () -> randomAlphaOfLength(8));
        final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE);
        final String source = randomAlphaOfLength(8);
        final CountDownLatch latch = new CountDownLatch(1);
        final ActionListener<ReplicationResponse> listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString()));
        currentRetentionLeases.put(id, primary.addRetentionLease(id, retainingSequenceNumber, source, listener));
        latch.await();
    }
    // Cause some recoveries to fail to ensure that retention leases are handled properly when retrying a recovery
    // 
    execute("set global persistent \"indices.recovery.retry_delay_network\" = '100ms'");
    final Semaphore recoveriesToDisrupt = new Semaphore(scaledRandomIntBetween(0, 4));
    final MockTransportService primaryTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, primaryShardNodeName);
    primaryTransportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE) && recoveriesToDisrupt.tryAcquire()) {
            if (randomBoolean()) {
                // return a ConnectTransportException to the START_RECOVERY action
                final TransportService replicaTransportService = internalCluster().getInstance(TransportService.class, connection.getNode().getName());
                final DiscoveryNode primaryNode = primaryTransportService.getLocalNode();
                replicaTransportService.disconnectFromNode(primaryNode);
                AbstractSimpleTransportTestCase.connectToNode(replicaTransportService, primaryNode);
            } else {
                // return an exception to the FINALIZE action
                throw new ElasticsearchException("failing recovery for test purposes");
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    // now allow the replicas to be allocated and wait for recovery to finalize
    allowNodes("tbl", 1 + numberOfReplicas);
    ensureGreen("tbl");
    // check current retention leases have been synced to all replicas
    for (final ShardRouting replicaShard : clusterService().state().routingTable().index("tbl").shard(0).replicaShards()) {
        final String replicaShardNodeId = replicaShard.currentNodeId();
        final String replicaShardNodeName = clusterService().state().nodes().get(replicaShardNodeId).getName();
        final IndexShard replica = internalCluster().getInstance(IndicesService.class, replicaShardNodeName).getShardOrNull(new ShardId(resolveIndex("tbl"), 0));
        final Map<String, RetentionLease> retentionLeasesOnReplica = RetentionLeaseUtils.toMapExcludingPeerRecoveryRetentionLeases(replica.getRetentionLeases());
        assertThat(retentionLeasesOnReplica, equalTo(currentRetentionLeases));
        // check retention leases have been written on the replica; see RecoveryTarget#finalizeRecovery
        assertThat(currentRetentionLeases, equalTo(RetentionLeaseUtils.toMapExcludingPeerRecoveryRetentionLeases(replica.loadRetentionLeases())));
    }
}

Example 82 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class RetentionLeaseIT method testRetentionLeasesSyncOnExpiration.

@Test
public void testRetentionLeasesSyncOnExpiration() throws Exception {
    final int numberOfReplicas = 2 - scaledRandomIntBetween(0, 2);
    internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas);
    final long estimatedTimeIntervalMillis = ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.get(Settings.EMPTY).millis();
    final TimeValue retentionLeaseTimeToLive = TimeValue.timeValueMillis(randomLongBetween(estimatedTimeIntervalMillis, 2 * estimatedTimeIntervalMillis));
    execute("create table doc.tbl (x int) clustered into 1 shards " + "with (" + "   number_of_replicas = ?, " + "   \"soft_deletes.enabled\" = true, " + "   \"soft_deletes.retention_lease.sync_interval\" = ?)", new Object[] { numberOfReplicas, retentionLeaseTimeToLive.getStringRep() });
    ensureGreen("tbl");
    final String primaryShardNodeId = clusterService().state().routingTable().index("tbl").shard(0).primaryShard().currentNodeId();
    final String primaryShardNodeName = clusterService().state().nodes().get(primaryShardNodeId).getName();
    final IndexShard primary = internalCluster().getInstance(IndicesService.class, primaryShardNodeName).getShardOrNull(new ShardId(resolveIndex("tbl"), 0));
    // we will add multiple retention leases, wait for some to expire, and assert a consistent view between the primary and the replicas
    final int length = randomIntBetween(1, 8);
    for (int i = 0; i < length; i++) {
        // update the index for retention leases to live a long time
        execute("alter table doc.tbl reset (\"soft_deletes.retention_lease.period\")");
        final String id = randomAlphaOfLength(8);
        final long retainingSequenceNumber = randomLongBetween(0, Long.MAX_VALUE);
        final String source = randomAlphaOfLength(8);
        final CountDownLatch latch = new CountDownLatch(1);
        final ActionListener<ReplicationResponse> listener = ActionListener.wrap(r -> latch.countDown(), e -> fail(e.toString()));
        final RetentionLease currentRetentionLease = primary.addRetentionLease(id, retainingSequenceNumber, source, listener);
        final long now = System.nanoTime();
        latch.await();
        // check current retention leases have been synced to all replicas
        for (final ShardRouting replicaShard : clusterService().state().routingTable().index("tbl").shard(0).replicaShards()) {
            final String replicaShardNodeId = replicaShard.currentNodeId();
            final String replicaShardNodeName = clusterService().state().nodes().get(replicaShardNodeId).getName();
            final IndexShard replica = internalCluster().getInstance(IndicesService.class, replicaShardNodeName).getShardOrNull(new ShardId(resolveIndex("tbl"), 0));
            assertThat(RetentionLeaseUtils.toMapExcludingPeerRecoveryRetentionLeases(replica.getRetentionLeases()).values(), anyOf(empty(), contains(currentRetentionLease)));
        }
        // update the index for retention leases to short a long time, to force expiration
        execute("alter table doc.tbl set (\"soft_deletes.retention_lease.period\" = ?)", new Object[] { retentionLeaseTimeToLive.getStringRep() });
        // sleep long enough that the current retention lease has expired
        final long later = System.nanoTime();
        Thread.sleep(Math.max(0, retentionLeaseTimeToLive.millis() - TimeUnit.NANOSECONDS.toMillis(later - now)));
        assertBusy(() -> assertThat(RetentionLeaseUtils.toMapExcludingPeerRecoveryRetentionLeases(primary.getRetentionLeases()).entrySet(), empty()));
        // now that all retention leases are expired should have been synced to all replicas
        assertBusy(() -> {
            for (final ShardRouting replicaShard : clusterService().state().routingTable().index("tbl").shard(0).replicaShards()) {
                final String replicaShardNodeId = replicaShard.currentNodeId();
                final String replicaShardNodeName = clusterService().state().nodes().get(replicaShardNodeId).getName();
                final IndexShard replica = internalCluster().getInstance(IndicesService.class, replicaShardNodeName).getShardOrNull(new ShardId(resolveIndex("tbl"), 0));
                assertThat(RetentionLeaseUtils.toMapExcludingPeerRecoveryRetentionLeases(replica.getRetentionLeases()).entrySet(), empty());
            }
        });
    }
}

Example 83 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class IndexRecoveryIT method testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint.

@Test
public void testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    String indexName = "test";
    execute("CREATE TABLE doc.test (num INT)" + " CLUSTERED INTO 1 SHARDS" + " WITH (" + "  number_of_replicas = 1," + "  \"unassigned.node_left.delayed_timeout\"='12h'," + "  \"soft_deletes.enabled\"=true" + " )");
    int numDocs = randomIntBetween(1, 100);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO doc.test (num) VALUES (?)", args);
    ensureGreen(indexName);
    final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
    final DiscoveryNodes discoveryNodes = clusterService().state().nodes();
    final IndexShardRoutingTable indexShardRoutingTable = clusterService().state().routingTable().shardRoutingTable(shardId);
    final IndexShard primary = internalCluster().getInstance(IndicesService.class, discoveryNodes.get(indexShardRoutingTable.primaryShard().currentNodeId()).getName()).getShardOrNull(shardId);
    final ShardRouting replicaShardRouting = indexShardRoutingTable.replicaShards().get(0);
    internalCluster().restartNode(discoveryNodes.get(replicaShardRouting.currentNodeId()).getName(), new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(discoveryNodes.getSize() - 1)).setWaitForEvents(Priority.LANGUID).get().isTimedOut());
            execute("INSERT INTO doc.test (num) VALUES (?)", args);
            // We do not guarantee that the replica can recover locally all the way to its own global checkpoint before starting
            // to recover from the primary, so we must be careful not to perform an operations-based recovery if this would require
            // some operations that are not being retained. Emulate this by advancing the lease ahead of the replica's GCP:
            primary.renewRetentionLease(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replicaShardRouting), primary.seqNoStats().getMaxSeqNo() + 1, ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE);
            return super.onNodeStopped(nodeName);
        }
    });
    ensureGreen(indexName);
    // noinspection OptionalGetWithoutIsPresent because it fails the test if absent
    final var recoveryState = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).get().shardRecoveryStates().get(indexName).stream().filter(rs -> rs.getPrimary() == false).findFirst().get();
    assertThat(recoveryState.getIndex().totalFileCount(), greaterThan(0));
}

Example 84 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class IndexRecoveryIT method testDoesNotCopyOperationsInSafeCommit.

@Test
public void testDoesNotCopyOperationsInSafeCommit() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    String indexName = "test";
    execute("CREATE TABLE doc.test (num INT)" + " CLUSTERED INTO 1 SHARDS" + " WITH (" + "  number_of_replicas = 0," + "  \"soft_deletes.enabled\"=true" + " )");
    int numDocs = randomIntBetween(1, 100);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO doc.test (num) VALUES (?)", args);
    final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
    final DiscoveryNodes discoveryNodes = clusterService().state().nodes();
    final IndexShardRoutingTable indexShardRoutingTable = clusterService().state().routingTable().shardRoutingTable(shardId);
    final IndexShard primary = internalCluster().getInstance(IndicesService.class, discoveryNodes.get(indexShardRoutingTable.primaryShard().currentNodeId()).getName()).getShardOrNull(shardId);
    final long maxSeqNoBeforeRecovery = primary.seqNoStats().getMaxSeqNo();
    assertBusy(() -> assertThat(primary.getLastSyncedGlobalCheckpoint(), equalTo(maxSeqNoBeforeRecovery)));
    // makes a safe commit
    execute("OPTIMIZE TABLE doc.test");
    execute("INSERT INTO doc.test (num) VALUES (?)", args);
    execute("ALTER TABLE doc.test SET (number_of_replicas = 1)");
    ensureGreen(indexName);
    final long maxSeqNoAfterRecovery = primary.seqNoStats().getMaxSeqNo();
    // noinspection OptionalGetWithoutIsPresent because it fails the test if absent
    final var recoveryState = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).get().shardRecoveryStates().get(indexName).stream().filter(rs -> rs.getPrimary() == false).findFirst().get();
    assertThat((long) recoveryState.getTranslog().recoveredOperations(), lessThanOrEqualTo(maxSeqNoAfterRecovery - maxSeqNoBeforeRecovery));
}

Example 85 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class IndexingMemoryControllerTests method testShardAdditionAndRemoval.

public void testShardAdditionAndRemoval() throws IOException {
    MockController controller = new MockController(Settings.builder().put("indices.memory.index_buffer_size", "4mb").build());
    IndexShard shard0 = newStartedShard();
    controller.simulateIndexing(shard0);
    controller.assertBuffer(shard0, 1);
    // add another shard
    IndexShard shard1 = newStartedShard();
    controller.simulateIndexing(shard1);
    controller.assertBuffer(shard0, 1);
    controller.assertBuffer(shard1, 1);
    // remove first shard
    controller.deleteShard(shard0);
    controller.forceCheck();
    controller.assertBuffer(shard1, 1);
    // remove second shard
    controller.deleteShard(shard1);
    controller.forceCheck();
    // add a new one
    IndexShard shard2 = newStartedShard();
    controller.simulateIndexing(shard2);
    controller.assertBuffer(shard2, 1);
    closeShards(shard0, shard1, shard2);
}