Examples with Index org.elasticsearch.index.Index used on opensource projects

Example 36 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class MetaDataDeleteIndexService method deleteIndices.

/**
     * Delete some indices from the cluster state.
     */
public ClusterState deleteIndices(ClusterState currentState, Set<Index> indices) {
    final MetaData meta = currentState.metaData();
    final Set<IndexMetaData> metaDatas = indices.stream().map(i -> meta.getIndexSafe(i)).collect(toSet());
    // Check if index deletion conflicts with any running snapshots
    SnapshotsService.checkIndexDeletion(currentState, metaDatas);
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
    MetaData.Builder metaDataBuilder = MetaData.builder(meta);
    ClusterBlocks.Builder clusterBlocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
    final IndexGraveyard.Builder graveyardBuilder = IndexGraveyard.builder(metaDataBuilder.indexGraveyard());
    final int previousGraveyardSize = graveyardBuilder.tombstones().size();
    for (final Index index : indices) {
        String indexName = index.getName();
        logger.info("{} deleting index", index);
        routingTableBuilder.remove(indexName);
        clusterBlocksBuilder.removeIndexBlocks(indexName);
        metaDataBuilder.remove(indexName);
    }
    // add tombstones to the cluster state for each deleted index
    final IndexGraveyard currentGraveyard = graveyardBuilder.addTombstones(indices).build(settings);
    // the new graveyard set on the metadata
    metaDataBuilder.indexGraveyard(currentGraveyard);
    logger.trace("{} tombstones purged from the cluster state. Previous tombstone size: {}. Current tombstone size: {}.", graveyardBuilder.getNumPurged(), previousGraveyardSize, currentGraveyard.getTombstones().size());
    MetaData newMetaData = metaDataBuilder.build();
    ClusterBlocks blocks = clusterBlocksBuilder.build();
    // update snapshot restore entries
    ImmutableOpenMap<String, ClusterState.Custom> customs = currentState.getCustoms();
    final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
    if (restoreInProgress != null) {
        RestoreInProgress updatedRestoreInProgress = RestoreService.updateRestoreStateWithDeletedIndices(restoreInProgress, indices);
        if (updatedRestoreInProgress != restoreInProgress) {
            ImmutableOpenMap.Builder<String, ClusterState.Custom> builder = ImmutableOpenMap.builder(customs);
            builder.put(RestoreInProgress.TYPE, updatedRestoreInProgress);
            customs = builder.build();
        }
    }
    return allocationService.reroute(ClusterState.builder(currentState).routingTable(routingTableBuilder.build()).metaData(newMetaData).blocks(blocks).customs(customs).build(), "deleted indices [" + indices + "]");
}

Example 37 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class MetaDataIndexTemplateService method validateAndAddTemplate.

private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService, NamedXContentRegistry xContentRegistry) throws Exception {
    Index createdIndex = null;
    final String temporaryIndexName = UUIDs.randomBase64UUID();
    try {
        // use the provided values, otherwise just pick valid dummy values
        int dummyPartitionSize = IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(request.settings);
        int dummyShards = request.settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1);
        //create index service for parsing and validating "mappings"
        Settings dummySettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(request.settings).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, dummyShards).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()).build();
        final IndexMetaData tmpIndexMetadata = IndexMetaData.builder(temporaryIndexName).settings(dummySettings).build();
        IndexService dummyIndexService = indicesService.createIndex(tmpIndexMetadata, Collections.emptyList(), shardId -> {
        });
        createdIndex = dummyIndexService.index();
        templateBuilder.order(request.order);
        templateBuilder.version(request.version);
        templateBuilder.patterns(request.indexPatterns);
        templateBuilder.settings(request.settings);
        Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
        for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
            try {
                templateBuilder.putMapping(entry.getKey(), entry.getValue());
            } catch (Exception e) {
                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
            }
            mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue()));
        }
        dummyIndexService.mapperService().merge(mappingsForValidation, MergeReason.MAPPING_UPDATE, false);
    } finally {
        if (createdIndex != null) {
            indicesService.removeIndex(createdIndex, NO_LONGER_ASSIGNED, " created for parsing template mapping");
        }
    }
}

Example 38 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class MetaDataMappingService method executeRefresh.

/**
     * Batch method to apply all the queued refresh operations. The idea is to try and batch as much
     * as possible so we won't create the same index all the time for example for the updates on the same mapping
     * and generate a single cluster change event out of all of those.
     */
ClusterState executeRefresh(final ClusterState currentState, final List<RefreshTask> allTasks) throws Exception {
    // break down to tasks per index, so we can optimize the on demand index service creation
    // to only happen for the duration of a single index processing of its respective events
    Map<String, List<RefreshTask>> tasksPerIndex = new HashMap<>();
    for (RefreshTask task : allTasks) {
        if (task.index == null) {
            logger.debug("ignoring a mapping task of type [{}] with a null index.", task);
        }
        tasksPerIndex.computeIfAbsent(task.index, k -> new ArrayList<>()).add(task);
    }
    boolean dirty = false;
    MetaData.Builder mdBuilder = MetaData.builder(currentState.metaData());
    for (Map.Entry<String, List<RefreshTask>> entry : tasksPerIndex.entrySet()) {
        IndexMetaData indexMetaData = mdBuilder.get(entry.getKey());
        if (indexMetaData == null) {
            // index got deleted on us, ignore...
            logger.debug("[{}] ignoring tasks - index meta data doesn't exist", entry.getKey());
            continue;
        }
        final Index index = indexMetaData.getIndex();
        // the tasks lists to iterate over, filled with the list of mapping tasks, trying to keep
        // the latest (based on order) update mapping one per node
        List<RefreshTask> allIndexTasks = entry.getValue();
        boolean hasTaskWithRightUUID = false;
        for (RefreshTask task : allIndexTasks) {
            if (indexMetaData.isSameUUID(task.indexUUID)) {
                hasTaskWithRightUUID = true;
            } else {
                logger.debug("{} ignoring task [{}] - index meta data doesn't match task uuid", index, task);
            }
        }
        if (hasTaskWithRightUUID == false) {
            continue;
        }
        // construct the actual index if needed, and make sure the relevant mappings are there
        boolean removeIndex = false;
        IndexService indexService = indicesService.indexService(indexMetaData.getIndex());
        if (indexService == null) {
            // we need to create the index here, and add the current mapping to it, so we can merge
            indexService = indicesService.createIndex(indexMetaData, Collections.emptyList(), shardId -> {
            });
            removeIndex = true;
            indexService.mapperService().merge(indexMetaData, MergeReason.MAPPING_RECOVERY, true);
        }
        IndexMetaData.Builder builder = IndexMetaData.builder(indexMetaData);
        try {
            boolean indexDirty = refreshIndexMapping(indexService, builder);
            if (indexDirty) {
                mdBuilder.put(builder);
                dirty = true;
            }
        } finally {
            if (removeIndex) {
                indicesService.removeIndex(index, NO_LONGER_ASSIGNED, "created for mapping processing");
            }
        }
    }
    if (!dirty) {
        return currentState;
    }
    return ClusterState.builder(currentState).metaData(mdBuilder).build();
}

Example 39 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class IndexMetaDataUpdater method removeStaleIdsWithoutRoutings.

/**
     * Removes allocation ids from the in-sync set for shard copies for which there is no routing entries in the routing table.
     * This method is called in AllocationService before any changes to the routing table are made.
     */
public static ClusterState removeStaleIdsWithoutRoutings(ClusterState clusterState, List<StaleShard> staleShards) {
    MetaData oldMetaData = clusterState.metaData();
    RoutingTable oldRoutingTable = clusterState.routingTable();
    MetaData.Builder metaDataBuilder = null;
    // group staleShards entries by index
    for (Map.Entry<Index, List<StaleShard>> indexEntry : staleShards.stream().collect(Collectors.groupingBy(fs -> fs.getShardId().getIndex())).entrySet()) {
        final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(indexEntry.getKey());
        IndexMetaData.Builder indexMetaDataBuilder = null;
        // group staleShards entries by shard id
        for (Map.Entry<ShardId, List<StaleShard>> shardEntry : indexEntry.getValue().stream().collect(Collectors.groupingBy(staleShard -> staleShard.getShardId())).entrySet()) {
            int shardNumber = shardEntry.getKey().getId();
            Set<String> oldInSyncAllocations = oldIndexMetaData.inSyncAllocationIds(shardNumber);
            Set<String> idsToRemove = shardEntry.getValue().stream().map(e -> e.getAllocationId()).collect(Collectors.toSet());
            assert idsToRemove.stream().allMatch(id -> oldRoutingTable.getByAllocationId(shardEntry.getKey(), id) == null) : "removing stale ids: " + idsToRemove + ", some of which have still a routing entry: " + oldRoutingTable;
            Set<String> remainingInSyncAllocations = Sets.difference(oldInSyncAllocations, idsToRemove);
            assert remainingInSyncAllocations.isEmpty() == false : "Set of in-sync ids cannot become empty for shard " + shardEntry.getKey() + " (before: " + oldInSyncAllocations + ", ids to remove: " + idsToRemove + ")";
            // (see ShardRouting#allocatedPostIndexCreate)
            if (remainingInSyncAllocations.isEmpty() == false) {
                if (indexMetaDataBuilder == null) {
                    indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
                }
                indexMetaDataBuilder.putInSyncAllocationIds(shardNumber, remainingInSyncAllocations);
            }
        }
        if (indexMetaDataBuilder != null) {
            if (metaDataBuilder == null) {
                metaDataBuilder = MetaData.builder(oldMetaData);
            }
            metaDataBuilder.put(indexMetaDataBuilder);
        }
    }
    if (metaDataBuilder != null) {
        return ClusterState.builder(clusterState).metaData(metaDataBuilder).build();
    } else {
        return clusterState;
    }
}

Example 40 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class IndexMetaDataUpdater method applyChanges.

/**
     * Updates the current {@link MetaData} based on the changes of this RoutingChangesObserver. Specifically
     * we update {@link IndexMetaData#getInSyncAllocationIds()} and {@link IndexMetaData#primaryTerm(int)} based on
     * the changes made during this allocation.
     *
     * @param oldMetaData {@link MetaData} object from before the routing nodes was changed.
     * @param newRoutingTable {@link RoutingTable} object after routing changes were applied.
     * @return adapted {@link MetaData}, potentially the original one if no change was needed.
     */
public MetaData applyChanges(MetaData oldMetaData, RoutingTable newRoutingTable) {
    Map<Index, List<Map.Entry<ShardId, Updates>>> changesGroupedByIndex = shardChanges.entrySet().stream().collect(Collectors.groupingBy(e -> e.getKey().getIndex()));
    MetaData.Builder metaDataBuilder = null;
    for (Map.Entry<Index, List<Map.Entry<ShardId, Updates>>> indexChanges : changesGroupedByIndex.entrySet()) {
        Index index = indexChanges.getKey();
        final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(index);
        IndexMetaData.Builder indexMetaDataBuilder = null;
        for (Map.Entry<ShardId, Updates> shardEntry : indexChanges.getValue()) {
            ShardId shardId = shardEntry.getKey();
            Updates updates = shardEntry.getValue();
            indexMetaDataBuilder = updateInSyncAllocations(newRoutingTable, oldIndexMetaData, indexMetaDataBuilder, shardId, updates);
            indexMetaDataBuilder = updatePrimaryTerm(oldIndexMetaData, indexMetaDataBuilder, shardId, updates);
        }
        if (indexMetaDataBuilder != null) {
            if (metaDataBuilder == null) {
                metaDataBuilder = MetaData.builder(oldMetaData);
            }
            metaDataBuilder.put(indexMetaDataBuilder);
        }
    }
    if (metaDataBuilder != null) {
        return metaDataBuilder.build();
    } else {
        return oldMetaData;
    }
}