Examples with SnapshotState org.opensearch.snapshots.SnapshotState used on opensource projects

Example 1 with SnapshotState

use of org.opensearch.snapshots.SnapshotState in project OpenSearch by opensearch-project.

the class CorruptedFileIT method testCorruptFileThenSnapshotAndRestore.

/**
 * Tests that restoring of a corrupted shard fails and we get a partial snapshot.
 * TODO once checksum verification on snapshotting is implemented this test needs to be fixed or split into several
 * parts... We should also corrupt files on the actual snapshot and check that we don't restore the corrupted shard.
 */
public void testCorruptFileThenSnapshotAndRestore() throws ExecutionException, InterruptedException, IOException {
    int numDocs = scaledRandomIntBetween(100, 1000);
    internalCluster().ensureAtLeastNumDataNodes(2);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, // no replicas for this test
    "0").put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false).put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB))));
    ensureGreen();
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test").setSource("field", "value");
    }
    indexRandom(true, builders);
    ensureGreen();
    assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
    // we have to flush at least once here since we don't corrupt the translog
    SearchResponse countResponse = client().prepareSearch().setSize(0).get();
    assertHitCount(countResponse, numDocs);
    ShardRouting shardRouting = corruptRandomPrimaryFile(false);
    logger.info("--> shard {} has a corrupted file", shardRouting);
    // we don't corrupt segments.gen since S/R doesn't snapshot this file
    // the other problem here why we can't corrupt segments.X files is that the snapshot flushes again before
    // it snapshots and that will write a new segments.X+1 file
    logger.info("-->  creating repository");
    assertAcked(client().admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath().toAbsolutePath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    logger.info("--> snapshot");
    final CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test").get();
    final SnapshotState snapshotState = createSnapshotResponse.getSnapshotInfo().state();
    logger.info("--> snapshot terminated with state " + snapshotState);
    final List<Path> files = listShardFiles(shardRouting);
    Path corruptedFile = null;
    for (Path file : files) {
        if (file.getFileName().toString().startsWith("corrupted_")) {
            corruptedFile = file;
            break;
        }
    }
    assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.PARTIAL));
    assertThat(corruptedFile, notNullValue());
}

Example 2 with SnapshotState

use of org.opensearch.snapshots.SnapshotState in project OpenSearch by opensearch-project.

the class RepositoryData method parseSnapshots.

/**
 * Parses the "snapshots" field and fills maps for the various per snapshot properties. This method must run before
 * {@link #parseIndices} which will rely on the maps of snapshot properties to have been populated already.
 *
 * @param parser           x-content parse
 * @param snapshots        map of snapshot uuid to {@link SnapshotId}
 * @param snapshotStates   map of snapshot uuid to {@link SnapshotState}
 * @param snapshotVersions map of snapshot uuid to {@link Version} that the snapshot was taken in
 * @param indexMetaLookup  map of {@link SnapshotId} to map of index id (as returned by {@link IndexId#getId}) that defines the index
 *                         metadata generations for the snapshot
 */
private static void parseSnapshots(XContentParser parser, Map<String, SnapshotId> snapshots, Map<String, SnapshotState> snapshotStates, Map<String, Version> snapshotVersions, Map<SnapshotId, Map<String, String>> indexMetaLookup) throws IOException {
    XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser);
    final Map<String, String> stringDeduplicator = new HashMap<>();
    while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
        String name = null;
        String uuid = null;
        SnapshotState state = null;
        Map<String, String> metaGenerations = null;
        Version version = null;
        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
            String currentFieldName = parser.currentName();
            parser.nextToken();
            switch(currentFieldName) {
                case NAME:
                    name = parser.text();
                    break;
                case UUID:
                    uuid = parser.text();
                    break;
                case STATE:
                    state = SnapshotState.fromValue((byte) parser.intValue());
                    break;
                case INDEX_METADATA_LOOKUP:
                    metaGenerations = parser.map(HashMap::new, p -> stringDeduplicator.computeIfAbsent(p.text(), Function.identity()));
                    break;
                case VERSION:
                    version = Version.fromString(parser.text());
                    break;
            }
        }
        final SnapshotId snapshotId = new SnapshotId(name, uuid);
        if (state != null) {
            snapshotStates.put(uuid, state);
        }
        if (version != null) {
            snapshotVersions.put(uuid, version);
        }
        snapshots.put(uuid, snapshotId);
        if (metaGenerations != null && metaGenerations.isEmpty() == false) {
            indexMetaLookup.put(snapshotId, metaGenerations);
        }
    }
}

Example 3 with SnapshotState

use of org.opensearch.snapshots.SnapshotState in project OpenSearch by opensearch-project.

the class RepositoryData method snapshotsFromXContent.

/**
 * Reads an instance of {@link RepositoryData} from x-content, loading the snapshots and indices metadata.
 *
 * @param fixBrokenShardGens set to {@code true} to filter out broken shard generations read from the {@code parser} via
 *                           {@link ShardGenerations#fixShardGeneration}. Used to disable fixing broken generations when reading
 *                           from cached bytes that we trust to not contain broken generations.
 */
public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException {
    XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
    final Map<String, SnapshotId> snapshots = new HashMap<>();
    final Map<String, SnapshotState> snapshotStates = new HashMap<>();
    final Map<String, Version> snapshotVersions = new HashMap<>();
    final Map<IndexId, List<SnapshotId>> indexSnapshots = new HashMap<>();
    final Map<String, IndexId> indexLookup = new HashMap<>();
    final ShardGenerations.Builder shardGenerations = ShardGenerations.builder();
    final Map<SnapshotId, Map<String, String>> indexMetaLookup = new HashMap<>();
    Map<String, String> indexMetaIdentifiers = null;
    while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
        final String field = parser.currentName();
        switch(field) {
            case SNAPSHOTS:
                parseSnapshots(parser, snapshots, snapshotStates, snapshotVersions, indexMetaLookup);
                break;
            case INDICES:
                parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations);
                break;
            case INDEX_METADATA_IDENTIFIERS:
                XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
                indexMetaIdentifiers = parser.mapStrings();
                break;
            case MIN_VERSION:
                XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.nextToken(), parser);
                final Version version = Version.fromString(parser.text());
                assert SnapshotsService.useShardGenerations(version);
                break;
            default:
                XContentParserUtils.throwUnknownField(field, parser.getTokenLocation());
        }
    }
    return new RepositoryData(genId, snapshots, snapshotStates, snapshotVersions, indexSnapshots, shardGenerations.build(), buildIndexMetaGenerations(indexMetaLookup, indexLookup, indexMetaIdentifiers));
}

Example 4 with SnapshotState

use of org.opensearch.snapshots.SnapshotState in project OpenSearch by opensearch-project.

the class RepositoryData method snapshotsToXContent.

/**
 * Writes the snapshots metadata and the related indices metadata to x-content.
 */
public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final Version repoMetaVersion) throws IOException {
    builder.startObject();
    // write the snapshots list
    builder.startArray(SNAPSHOTS);
    final boolean shouldWriteIndexGens = SnapshotsService.useIndexGenerations(repoMetaVersion);
    final boolean shouldWriteShardGens = SnapshotsService.useShardGenerations(repoMetaVersion);
    for (final SnapshotId snapshot : getSnapshotIds()) {
        builder.startObject();
        builder.field(NAME, snapshot.getName());
        final String snapshotUUID = snapshot.getUUID();
        builder.field(UUID, snapshotUUID);
        final SnapshotState state = snapshotStates.get(snapshotUUID);
        if (state != null) {
            builder.field(STATE, state.value());
        }
        if (shouldWriteIndexGens) {
            builder.startObject(INDEX_METADATA_LOOKUP);
            for (Map.Entry<IndexId, String> entry : indexMetaDataGenerations.lookup.getOrDefault(snapshot, Collections.emptyMap()).entrySet()) {
                builder.field(entry.getKey().getId(), entry.getValue());
            }
            builder.endObject();
        }
        final Version version = snapshotVersions.get(snapshotUUID);
        if (version != null) {
            builder.field(VERSION, version.toString());
        }
        builder.endObject();
    }
    builder.endArray();
    // write the indices map
    builder.startObject(INDICES);
    for (final IndexId indexId : getIndices().values()) {
        builder.startObject(indexId.getName());
        builder.field(INDEX_ID, indexId.getId());
        builder.startArray(SNAPSHOTS);
        List<SnapshotId> snapshotIds = indexSnapshots.get(indexId);
        assert snapshotIds != null;
        for (final SnapshotId snapshotId : snapshotIds) {
            builder.value(snapshotId.getUUID());
        }
        builder.endArray();
        if (shouldWriteShardGens) {
            builder.startArray(SHARD_GENERATIONS);
            for (String gen : shardGenerations.getGens(indexId)) {
                builder.value(gen);
            }
            builder.endArray();
        }
        builder.endObject();
    }
    builder.endObject();
    if (shouldWriteIndexGens) {
        builder.field(MIN_VERSION, SnapshotsService.INDEX_GEN_IN_REPO_DATA_VERSION.toString());
        builder.field(INDEX_METADATA_IDENTIFIERS, indexMetaDataGenerations.identifiers);
    } else if (shouldWriteShardGens) {
        // Add min version field to make it impossible for older OpenSearch versions to deserialize this object
        builder.field(MIN_VERSION, SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.toString());
    }
    builder.endObject();
    return builder;
}

Example 5 with SnapshotState

use of org.opensearch.snapshots.SnapshotState in project OpenSearch by opensearch-project.

the class RepositoryDataTests method testInitIndices.

public void testInitIndices() {
    final int numSnapshots = randomIntBetween(1, 30);
    final Map<String, SnapshotId> snapshotIds = new HashMap<>(numSnapshots);
    final Map<String, SnapshotState> snapshotStates = new HashMap<>(numSnapshots);
    final Map<String, Version> snapshotVersions = new HashMap<>(numSnapshots);
    for (int i = 0; i < numSnapshots; i++) {
        final SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
        snapshotIds.put(snapshotId.getUUID(), snapshotId);
        snapshotStates.put(snapshotId.getUUID(), randomFrom(SnapshotState.values()));
        snapshotVersions.put(snapshotId.getUUID(), randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()));
    }
    RepositoryData repositoryData = new RepositoryData(EMPTY_REPO_GEN, snapshotIds, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY);
    // test that initializing indices works
    Map<IndexId, List<SnapshotId>> indices = randomIndices(snapshotIds);
    RepositoryData newRepoData = new RepositoryData(repositoryData.getGenId(), snapshotIds, snapshotStates, snapshotVersions, indices, ShardGenerations.EMPTY, IndexMetaDataGenerations.EMPTY);
    List<SnapshotId> expected = new ArrayList<>(repositoryData.getSnapshotIds());
    Collections.sort(expected);
    List<SnapshotId> actual = new ArrayList<>(newRepoData.getSnapshotIds());
    Collections.sort(actual);
    assertEquals(expected, actual);
    for (IndexId indexId : indices.keySet()) {
        assertEquals(indices.get(indexId), newRepoData.getSnapshots(indexId));
    }
}