Examples with Version org.opensearch.Version used on opensource projects

Example 61 with Version

use of org.opensearch.Version in project OpenSearch by opensearch-project.

the class SnapshotsService method createSnapshot.

/**
 * Initializes the snapshotting process.
 * <p>
 * This method is used by clients to start snapshot. It makes sure that there is no snapshots are currently running and
 * creates a snapshot record in cluster state metadata.
 *
 * @param request  snapshot request
 * @param listener snapshot creation listener
 */
public void createSnapshot(final CreateSnapshotRequest request, final ActionListener<Snapshot> listener) {
    final String repositoryName = request.repository();
    final String snapshotName = indexNameExpressionResolver.resolveDateMathExpression(request.snapshot());
    validate(repositoryName, snapshotName);
    // TODO: create snapshot UUID in CreateSnapshotRequest and make this operation idempotent to cleanly deal with transport layer
    // retries
    // new UUID for the snapshot
    final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID());
    Repository repository = repositoriesService.repository(request.repository());
    if (repository.isReadOnly()) {
        listener.onFailure(new RepositoryException(repository.getMetadata().name(), "cannot create snapshot in a readonly repository"));
        return;
    }
    final Snapshot snapshot = new Snapshot(repositoryName, snapshotId);
    final Map<String, Object> userMeta = repository.adaptUserMetadata(request.userMetadata());
    repository.executeConsistentStateUpdate(repositoryData -> new ClusterStateUpdateTask() {

        private SnapshotsInProgress.Entry newEntry;

        @Override
        public ClusterState execute(ClusterState currentState) {
            ensureSnapshotNameAvailableInRepo(repositoryData, snapshotName, repository);
            final SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
            final List<SnapshotsInProgress.Entry> runningSnapshots = snapshots.entries();
            ensureSnapshotNameNotRunning(runningSnapshots, repositoryName, snapshotName);
            validate(repositoryName, snapshotName, currentState);
            final boolean concurrentOperationsAllowed = currentState.nodes().getMinNodeVersion().onOrAfter(FULL_CONCURRENCY_VERSION);
            final SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress.EMPTY);
            if (deletionsInProgress.hasDeletionsInProgress() && concurrentOperationsAllowed == false) {
                throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "cannot snapshot while a snapshot deletion is in-progress in [" + deletionsInProgress + "]");
            }
            final RepositoryCleanupInProgress repositoryCleanupInProgress = currentState.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY);
            if (repositoryCleanupInProgress.hasCleanupInProgress()) {
                throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "cannot snapshot while a repository cleanup is in-progress in [" + repositoryCleanupInProgress + "]");
            }
            // cluster state anyway in #applyClusterState.
            if (concurrentOperationsAllowed == false && runningSnapshots.stream().anyMatch(entry -> entry.state() != State.INIT)) {
                throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, " a snapshot is already running");
            }
            ensureNoCleanupInProgress(currentState, repositoryName, snapshotName);
            ensureBelowConcurrencyLimit(repositoryName, snapshotName, snapshots, deletionsInProgress);
            // Store newSnapshot here to be processed in clusterStateProcessed
            List<String> indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request));
            final List<String> dataStreams = indexNameExpressionResolver.dataStreamNames(currentState, request.indicesOptions(), request.indices());
            logger.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices);
            final List<IndexId> indexIds = repositoryData.resolveNewIndices(indices, getInFlightIndexIds(runningSnapshots, repositoryName));
            final Version version = minCompatibleVersion(currentState.nodes().getMinNodeVersion(), repositoryData, null);
            ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards = shards(snapshots, deletionsInProgress, currentState.metadata(), currentState.routingTable(), indexIds, useShardGenerations(version), repositoryData, repositoryName);
            if (request.partial() == false) {
                Set<String> missing = new HashSet<>();
                for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> entry : shards) {
                    if (entry.value.state() == ShardState.MISSING) {
                        missing.add(entry.key.getIndex().getName());
                    }
                }
                if (missing.isEmpty() == false) {
                    throw new SnapshotException(new Snapshot(repositoryName, snapshotId), "Indices don't have primary shards " + missing);
                }
            }
            newEntry = SnapshotsInProgress.startedEntry(new Snapshot(repositoryName, snapshotId), request.includeGlobalState(), request.partial(), indexIds, dataStreams, threadPool.absoluteTimeInMillis(), repositoryData.getGenId(), shards, userMeta, version);
            final List<SnapshotsInProgress.Entry> newEntries = new ArrayList<>(runningSnapshots);
            newEntries.add(newEntry);
            return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(new ArrayList<>(newEntries))).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            logger.warn(() -> new ParameterizedMessage("[{}][{}] failed to create snapshot", repositoryName, snapshotName), e);
            listener.onFailure(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
            try {
                logger.info("snapshot [{}] started", snapshot);
                listener.onResponse(snapshot);
            } finally {
                if (newEntry.state().completed()) {
                    endSnapshot(newEntry, newState.metadata(), repositoryData);
                }
            }
        }

        @Override
        public TimeValue timeout() {
            return request.masterNodeTimeout();
        }
    }, "create_snapshot [" + snapshotName + ']', listener::onFailure);
}

Example 62 with Version

use of org.opensearch.Version in project OpenSearch by opensearch-project.

the class SnapshotsService method endCompletedSnapshots.

/**
 * Cleanup all snapshots found in the given cluster state that have no more work left:
 * 1. Completed snapshots
 * 2. Snapshots in state INIT that a previous master of an older version failed to start
 * 3. Snapshots in any other state that have all their shard tasks completed
 */
private void endCompletedSnapshots(ClusterState state) {
    SnapshotsInProgress snapshotsInProgress = state.custom(SnapshotsInProgress.TYPE);
    assert snapshotsInProgress != null;
    snapshotsInProgress.entries().stream().filter(entry -> entry.state().completed() || entry.state() == State.INIT || completed(entry.shards().values())).forEach(entry -> endSnapshot(entry, state.metadata(), null));
}

Example 63 with Version

use of org.opensearch.Version in project OpenSearch by opensearch-project.

the class SnapshotsService method minCompatibleVersion.

/**
 * Determines the minimum {@link Version} that the snapshot repository must be compatible with from the current nodes in the cluster
 * and the contents of the repository. The minimum version is determined as the lowest version found across all snapshots in the
 * repository and all nodes in the cluster.
 *
 * @param minNodeVersion minimum node version in the cluster
 * @param repositoryData current {@link RepositoryData} of that repository
 * @param excluded       snapshot id to ignore when computing the minimum version
 *                       (used to use newer metadata version after a snapshot delete)
 * @return minimum node version that must still be able to read the repository metadata
 */
public Version minCompatibleVersion(Version minNodeVersion, RepositoryData repositoryData, @Nullable Collection<SnapshotId> excluded) {
    Version minCompatVersion = minNodeVersion;
    final Collection<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
    for (SnapshotId snapshotId : snapshotIds.stream().filter(excluded == null ? sn -> true : sn -> excluded.contains(sn) == false).collect(Collectors.toList())) {
        final Version known = repositoryData.getVersion(snapshotId);
        // If we don't have the version cached in the repository data yet we load it from the snapshot info blobs
        if (known == null) {
            assert repositoryData.shardGenerations().totalShards() == 0 : "Saw shard generations [" + repositoryData.shardGenerations() + "] but did not have versions tracked for snapshot [" + snapshotId + "]";
            return OLD_SNAPSHOT_FORMAT;
        } else {
            minCompatVersion = minCompatVersion.before(known) ? minCompatVersion : known;
        }
    }
    return minCompatVersion;
}

Example 64 with Version

use of org.opensearch.Version in project OpenSearch by opensearch-project.

the class TransportSearchHelperTests method testParseScrollId.

public void testParseScrollId() {
    final Version version = VersionUtils.randomVersion(random());
    boolean includeUUID = version.onOrAfter(LegacyESVersion.V_7_7_0);
    final AtomicArray<SearchPhaseResult> queryResults = generateQueryResults();
    String scrollId = TransportSearchHelper.buildScrollId(queryResults, version);
    ParsedScrollId parseScrollId = TransportSearchHelper.parseScrollId(scrollId);
    assertEquals(3, parseScrollId.getContext().length);
    assertEquals("node_1", parseScrollId.getContext()[0].getNode());
    assertEquals("cluster_x", parseScrollId.getContext()[0].getClusterAlias());
    assertEquals(1, parseScrollId.getContext()[0].getSearchContextId().getId());
    if (includeUUID) {
        assertThat(parseScrollId.getContext()[0].getSearchContextId().getSessionId(), equalTo("a"));
    } else {
        assertThat(parseScrollId.getContext()[0].getSearchContextId().getSessionId(), equalTo(""));
    }
    assertEquals("node_2", parseScrollId.getContext()[1].getNode());
    assertEquals("cluster_y", parseScrollId.getContext()[1].getClusterAlias());
    assertEquals(12, parseScrollId.getContext()[1].getSearchContextId().getId());
    if (includeUUID) {
        assertThat(parseScrollId.getContext()[1].getSearchContextId().getSessionId(), equalTo("b"));
    } else {
        assertThat(parseScrollId.getContext()[1].getSearchContextId().getSessionId(), equalTo(""));
    }
    assertEquals("node_3", parseScrollId.getContext()[2].getNode());
    assertNull(parseScrollId.getContext()[2].getClusterAlias());
    assertEquals(42, parseScrollId.getContext()[2].getSearchContextId().getId());
    if (includeUUID) {
        assertThat(parseScrollId.getContext()[2].getSearchContextId().getSessionId(), equalTo("c"));
    } else {
        assertThat(parseScrollId.getContext()[2].getSearchContextId().getSessionId(), equalTo(""));
    }
}

Example 65 with Version

use of org.opensearch.Version in project OpenSearch by opensearch-project.

the class ShardStateActionTests method testFailedShardEntrySerialization.

public void testFailedShardEntrySerialization() throws Exception {
    final ShardId shardId = new ShardId(randomRealisticUnicodeOfLengthBetween(10, 100), UUID.randomUUID().toString(), between(0, 1000));
    final String allocationId = randomRealisticUnicodeOfCodepointLengthBetween(10, 100);
    final long primaryTerm = randomIntBetween(0, 100);
    final String message = randomRealisticUnicodeOfCodepointLengthBetween(10, 100);
    final Exception failure = randomBoolean() ? null : getSimulatedFailure();
    final boolean markAsStale = randomBoolean();
    final Version version = randomFrom(randomCompatibleVersion(random(), Version.CURRENT));
    final FailedShardEntry failedShardEntry = new FailedShardEntry(shardId, allocationId, primaryTerm, message, failure, markAsStale);
    try (StreamInput in = serialize(failedShardEntry, version).streamInput()) {
        in.setVersion(version);
        final FailedShardEntry deserialized = new FailedShardEntry(in);
        assertThat(deserialized.shardId, equalTo(shardId));
        assertThat(deserialized.allocationId, equalTo(allocationId));
        assertThat(deserialized.primaryTerm, equalTo(primaryTerm));
        assertThat(deserialized.message, equalTo(message));
        if (failure != null) {
            assertThat(deserialized.failure, notNullValue());
            assertThat(deserialized.failure.getClass(), equalTo(failure.getClass()));
            assertThat(deserialized.failure.getMessage(), equalTo(failure.getMessage()));
        } else {
            assertThat(deserialized.failure, nullValue());
        }
        assertThat(deserialized.markAsStale, equalTo(markAsStale));
        assertEquals(failedShardEntry, deserialized);
    }
}