Examples with IndexShard org.elasticsearch.index.shard.IndexShard used on opensource projects

Example 61 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class IndexService method createShard.

public synchronized IndexShard createShard(final ShardRouting routing, final Consumer<ShardId> globalCheckpointSyncer, final RetentionLeaseSyncer retentionLeaseSyncer) throws IOException {
    Objects.requireNonNull(retentionLeaseSyncer);
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, "starting shard", TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        Directory directory = directoryFactory.newDirectory(this.indexSettings, path);
        store = new Store(shardId, this.indexSettings, directory, lock, new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId)));
        eventListener.onStoreCreated(shardId);
        indexShard = new IndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, engineFactory, eventListener, threadPool, bigArrays, indexingOperationListeners, () -> globalCheckpointSyncer.accept(shardId), retentionLeaseSyncer, circuitBreakerService);
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}

Example 62 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class RecoverySourceHandlerTests method testSendOperationsConcurrently.

public void testSendOperationsConcurrently() throws Throwable {
    final IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    Set<Long> receivedSeqNos = ConcurrentCollections.newConcurrentSet();
    long maxSeenAutoIdTimestamp = randomBoolean() ? -1 : randomNonNegativeLong();
    long maxSeqNoOfUpdatesOrDeletes = randomBoolean() ? -1 : randomNonNegativeLong();
    RetentionLeases retentionLeases = new RetentionLeases(randomNonNegativeLong(), randomNonNegativeLong(), Collections.emptySet());
    long mappingVersion = randomNonNegativeLong();
    AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    int numOps = randomIntBetween(0, 1000);
    AtomicBoolean received = new AtomicBoolean();
    RecoveryTargetHandler target = new TestRecoveryTargetHandler() {

        @Override
        public void indexTranslogOperations(List<Translog.Operation> operations, int receivedTotalOps, long receivedMaxSeenAutoIdTimestamp, long receivedMaxSeqNoOfUpdatesOrDeletes, RetentionLeases receivedRetentionLease, long receivedMappingVersion, ActionListener<Long> listener) {
            received.set(true);
            assertThat(receivedMaxSeenAutoIdTimestamp, equalTo(maxSeenAutoIdTimestamp));
            assertThat(receivedMaxSeqNoOfUpdatesOrDeletes, equalTo(maxSeqNoOfUpdatesOrDeletes));
            assertThat(receivedRetentionLease, equalTo(retentionLeases));
            assertThat(receivedMappingVersion, equalTo(mappingVersion));
            assertThat(receivedTotalOps, equalTo(numOps));
            for (Translog.Operation operation : operations) {
                receivedSeqNos.add(operation.seqNo());
            }
            if (randomBoolean()) {
                localCheckpoint.addAndGet(randomIntBetween(1, 100));
            }
            listener.onResponse(localCheckpoint.get());
        }
    };
    PlainActionFuture<RecoverySourceHandler.SendSnapshotResult> sendFuture = new PlainActionFuture<>();
    long startingSeqNo = randomIntBetween(0, 1000);
    long endingSeqNo = startingSeqNo + randomIntBetween(0, 10000);
    List<Translog.Operation> operations = generateOperations(numOps);
    Randomness.shuffle(operations);
    List<Translog.Operation> skipOperations = randomSubsetOf(operations);
    Translog.Snapshot snapshot = newTranslogSnapshot(operations, skipOperations);
    RecoverySourceHandler handler = new RecoverySourceHandler(shard, new AsyncRecoveryTarget(target, recoveryExecutor), threadPool, getStartRecoveryRequest(), between(1, 10 * 1024), between(1, 5), between(1, 5));
    handler.phase2(startingSeqNo, endingSeqNo, snapshot, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersion, sendFuture);
    RecoverySourceHandler.SendSnapshotResult sendSnapshotResult = sendFuture.actionGet();
    assertTrue(received.get());
    assertThat(sendSnapshotResult.targetLocalCheckpoint, equalTo(localCheckpoint.get()));
    assertThat(sendSnapshotResult.sentOperations, equalTo(receivedSeqNos.size()));
    Set<Long> sentSeqNos = new HashSet<>();
    for (Translog.Operation op : operations) {
        if (startingSeqNo <= op.seqNo() && op.seqNo() <= endingSeqNo && skipOperations.contains(op) == false) {
            sentSeqNos.add(op.seqNo());
        }
    }
    assertThat(receivedSeqNos, equalTo(sentSeqNos));
}

Example 63 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class RecoverySourceHandlerTests method testCancellationsDoesNotLeakPrimaryPermits.

@Test
public void testCancellationsDoesNotLeakPrimaryPermits() throws Exception {
    final CancellableThreads cancellableThreads = new CancellableThreads();
    final IndexShard shard = mock(IndexShard.class);
    final AtomicBoolean freed = new AtomicBoolean(true);
    when(shard.isRelocatedPrimary()).thenReturn(false);
    doAnswer(invocation -> {
        freed.set(false);
        ((ActionListener<Releasable>) invocation.getArguments()[0]).onResponse(() -> freed.set(true));
        return null;
    }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), anyObject());
    Thread cancelingThread = new Thread(() -> cancellableThreads.cancel("test"));
    cancelingThread.start();
    try {
        RecoverySourceHandler.runUnderPrimaryPermit(() -> {
        }, "test", shard, cancellableThreads, logger);
    } catch (CancellableThreads.ExecutionCancelledException e) {
    // expected.
    }
    cancelingThread.join();
    // we have to use assert busy as we may be interrupted while acquiring the permit, if so we want to check
    // that the permit is released.
    assertBusy(() -> assertTrue(freed.get()));
}

Example 64 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class RecoverySourceHandlerTests method testVerifySeqNoStatsWhenRecoverWithSyncId.

public void testVerifySeqNoStatsWhenRecoverWithSyncId() throws Exception {
    IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    RecoverySourceHandler handler = new RecoverySourceHandler(shard, new TestRecoveryTargetHandler(), threadPool, getStartRecoveryRequest(), between(1, 16), between(1, 4), between(1, 4));
    String syncId = UUIDs.randomBase64UUID();
    int numDocs = between(0, 1000);
    long localCheckpoint = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
    long maxSeqNo = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
    assertTrue(handler.canSkipPhase1(newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs), newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs)));
    AssertionError error = expectThrows(AssertionError.class, () -> {
        long localCheckpointOnTarget = randomValueOtherThan(localCheckpoint, () -> randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE));
        long maxSeqNoOnTarget = randomValueOtherThan(maxSeqNo, () -> randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE));
        handler.canSkipPhase1(newMetadataSnapshot(syncId, Long.toString(localCheckpoint), Long.toString(maxSeqNo), numDocs), newMetadataSnapshot(syncId, Long.toString(localCheckpointOnTarget), Long.toString(maxSeqNoOnTarget), numDocs));
    });
    assertThat(error.getMessage(), containsString("try to recover [index][1] with sync id but seq_no stats are mismatched:"));
}

Example 65 with IndexShard

use of org.elasticsearch.index.shard.IndexShard in project crate by crate.

the class RecoverySourceHandlerTests method testSendSnapshotStopOnError.

@Test
public void testSendSnapshotStopOnError() throws Exception {
    final int fileChunkSizeInBytes = between(1, 10 * 1024);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    final IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    final List<Translog.Operation> ops = new ArrayList<>();
    for (int numOps = between(1, 256), i = 0; i < numOps; i++) {
        final Engine.Index index = getIndex(Integer.toString(i));
        ops.add(new Translog.Index(index, new Engine.IndexResult(1, 1, i, true)));
    }
    final AtomicBoolean wasFailed = new AtomicBoolean();
    RecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {

        @Override
        public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps, long timestamp, long msu, RetentionLeases retentionLeases, long mappingVersion, ActionListener<Long> listener) {
            if (randomBoolean()) {
                listener.onResponse(SequenceNumbers.NO_OPS_PERFORMED);
            } else {
                listener.onFailure(new RuntimeException("test - failed to index"));
                wasFailed.set(true);
            }
        }
    };
    RecoverySourceHandler handler = new RecoverySourceHandler(shard, new AsyncRecoveryTarget(recoveryTarget, threadPool.generic()), threadPool, request, fileChunkSizeInBytes, between(1, 10), between(1, 10));
    PlainActionFuture<RecoverySourceHandler.SendSnapshotResult> future = new PlainActionFuture<>();
    final long startingSeqNo = randomLongBetween(0, ops.size() - 1L);
    final long endingSeqNo = randomLongBetween(startingSeqNo, ops.size() - 1L);
    handler.phase2(startingSeqNo, endingSeqNo, newTranslogSnapshot(ops, Collections.emptyList()), randomNonNegativeLong(), randomNonNegativeLong(), RetentionLeases.EMPTY, randomNonNegativeLong(), future);
    if (wasFailed.get()) {
        final RecoveryEngineException error = expectThrows(RecoveryEngineException.class, future::actionGet);
        assertThat(error.getMessage(), equalTo("Phase[2] failed to send/replay operations"));
        assertThat(error.getCause().getMessage(), equalTo("test - failed to index"));
    }
}