Examples with ShardStats org.opensearch.action.admin.indices.stats.ShardStats used on opensource projects

Example 6 with ShardStats

use of org.opensearch.action.admin.indices.stats.ShardStats in project OpenSearch by opensearch-project.

the class ShardIndexingPressureSettingsIT method getPrimaryReplicaNodeNames.

private Tuple<String, String> getPrimaryReplicaNodeNames(String indexName) {
    IndicesStatsResponse response = client().admin().indices().prepareStats(indexName).get();
    String primaryId = Stream.of(response.getShards()).map(ShardStats::getShardRouting).filter(ShardRouting::primary).findAny().get().currentNodeId();
    String replicaId = Stream.of(response.getShards()).map(ShardStats::getShardRouting).filter(sr -> sr.primary() == false).findAny().get().currentNodeId();
    DiscoveryNodes nodes = client().admin().cluster().prepareState().get().getState().nodes();
    String primaryName = nodes.get(primaryId).getName();
    String replicaName = nodes.get(replicaId).getName();
    return new Tuple<>(primaryName, replicaName);
}

Example 7 with ShardStats

use of org.opensearch.action.admin.indices.stats.ShardStats in project OpenSearch by opensearch-project.

the class MasterDisruptionIT method testMappingTimeout.

public void testMappingTimeout() throws Exception {
    startCluster(3);
    createIndex("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1).put("index.routing.allocation.exclude._name", internalCluster().getMasterName()).build());
    // create one field
    index("test", "doc", "1", "{ \"f\": 1 }");
    ensureGreen();
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("indices.mapping.dynamic_timeout", "1ms")));
    ServiceDisruptionScheme disruption = new BlockMasterServiceOnMaster(random());
    setDisruptionScheme(disruption);
    disruption.startDisrupting();
    BulkRequestBuilder bulk = client().prepareBulk();
    bulk.add(client().prepareIndex("test").setId("2").setSource("{ \"f\": 1 }", XContentType.JSON));
    bulk.add(client().prepareIndex("test").setId("3").setSource("{ \"g\": 1 }", XContentType.JSON));
    bulk.add(client().prepareIndex("test").setId("4").setSource("{ \"f\": 1 }", XContentType.JSON));
    BulkResponse bulkResponse = bulk.get();
    assertTrue(bulkResponse.hasFailures());
    disruption.stopDisrupting();
    assertBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats("test").clear().get();
        for (ShardStats shardStats : stats.getShards()) {
            assertThat(shardStats.getShardRouting().toString(), shardStats.getSeqNoStats().getGlobalCheckpoint(), equalTo(shardStats.getSeqNoStats().getLocalCheckpoint()));
        }
    });
}

Example 8 with ShardStats

use of org.opensearch.action.admin.indices.stats.ShardStats in project OpenSearch by opensearch-project.

the class ReplicaShardAllocatorIT method testRecentPrimaryInformation.

/**
 * Ensure that we fetch the latest shard store from the primary when a new node joins so we won't cancel the current recovery
 * for the copy on the newly joined node unless we can perform a noop recovery with that node.
 */
public void testRecentPrimaryInformation() throws Exception {
    String indexName = "test";
    String nodeWithPrimary = internalCluster().startNode();
    assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 0.1f).put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms").put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms").put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "1ms")));
    String nodeWithReplica = internalCluster().startDataOnlyNode();
    DiscoveryNode discoNodeWithReplica = internalCluster().getInstance(ClusterService.class, nodeWithReplica).localNode();
    Settings nodeWithReplicaSettings = internalCluster().dataPathSettings(nodeWithReplica);
    ensureGreen(indexName);
    indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, between(10, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithReplica));
    if (randomBoolean()) {
        indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, between(10, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
    }
    CountDownLatch blockRecovery = new CountDownLatch(1);
    CountDownLatch recoveryStarted = new CountDownLatch(1);
    MockTransportService transportServiceOnPrimary = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeWithPrimary);
    transportServiceOnPrimary.addSendBehavior((connection, requestId, action, request, options) -> {
        if (PeerRecoveryTargetService.Actions.FILES_INFO.equals(action)) {
            recoveryStarted.countDown();
            try {
                blockRecovery.await();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    String newNode = internalCluster().startDataOnlyNode();
    recoveryStarted.await();
    // Index more documents and flush to destroy sync_id and remove the retention lease (as file_based_recovery_threshold reached).
    indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, between(50, 200)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
    client().admin().indices().prepareFlush(indexName).get();
    assertBusy(() -> {
        for (ShardStats shardStats : client().admin().indices().prepareStats(indexName).get().getShards()) {
            for (RetentionLease lease : shardStats.getRetentionLeaseStats().retentionLeases().leases()) {
                assertThat(lease.id(), not(equalTo(ReplicationTracker.getPeerRecoveryRetentionLeaseId(discoNodeWithReplica.getId()))));
            }
        }
    });
    // AllocationService only calls GatewayAllocator if there are unassigned shards
    assertAcked(client().admin().indices().prepareCreate("dummy-index").setWaitForActiveShards(0).setSettings(Settings.builder().put("index.routing.allocation.require.attr", "not-found")));
    internalCluster().startDataOnlyNode(nodeWithReplicaSettings);
    // need to wait for events to ensure the reroute has happened since we perform it async when a new node joins.
    client().admin().cluster().prepareHealth(indexName).setWaitForYellowStatus().setWaitForEvents(Priority.LANGUID).get();
    blockRecovery.countDown();
    ensureGreen(indexName);
    assertThat(internalCluster().nodesInclude(indexName), hasItem(newNode));
    for (RecoveryState recovery : client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) {
        if (recovery.getPrimary() == false) {
            assertThat(recovery.getIndex().fileDetails(), not(empty()));
        }
    }
    transportServiceOnPrimary.clearAllRules();
}

Example 9 with ShardStats

use of org.opensearch.action.admin.indices.stats.ShardStats in project OpenSearch by opensearch-project.

the class TransportClusterStatsAction method nodeOperation.

@Override
protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
    NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false);
    NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, true, true, true, false, true, false, false, false, false, false, true, false, false, false, false);
    List<ShardStats> shardsStats = new ArrayList<>();
    for (IndexService indexService : indicesService) {
        for (IndexShard indexShard : indexService) {
            if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
                // only report on fully started shards
                CommitStats commitStats;
                SeqNoStats seqNoStats;
                RetentionLeaseStats retentionLeaseStats;
                try {
                    commitStats = indexShard.commitStats();
                    seqNoStats = indexShard.seqNoStats();
                    retentionLeaseStats = indexShard.getRetentionLeaseStats();
                } catch (final AlreadyClosedException e) {
                    // shard is closed - no stats is fine
                    commitStats = null;
                    seqNoStats = null;
                    retentionLeaseStats = null;
                }
                shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), commitStats, seqNoStats, retentionLeaseStats));
            }
        }
    }
    ClusterHealthStatus clusterStatus = null;
    if (clusterService.state().nodes().isLocalNodeElectedMaster()) {
        clusterStatus = new ClusterStateHealth(clusterService.state()).getStatus();
    }
    return new ClusterStatsNodeResponse(nodeInfo.getNode(), clusterStatus, nodeInfo, nodeStats, shardsStats.toArray(new ShardStats[shardsStats.size()]));
}

Example 10 with ShardStats

use of org.opensearch.action.admin.indices.stats.ShardStats in project OpenSearch by opensearch-project.

the class InternalClusterInfoService method buildShardLevelInfo.

static void buildShardLevelInfo(Logger logger, ShardStats[] stats, ImmutableOpenMap.Builder<String, Long> shardSizes, ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath, Map<ClusterInfo.NodeAndPath, ClusterInfo.ReservedSpace.Builder> reservedSpaceByShard) {
    for (ShardStats s : stats) {
        final ShardRouting shardRouting = s.getShardRouting();
        newShardRoutingToDataPath.put(shardRouting, s.getDataPath());
        final StoreStats storeStats = s.getStats().getStore();
        if (storeStats == null) {
            continue;
        }
        final long size = storeStats.sizeInBytes();
        final long reserved = storeStats.getReservedSize().getBytes();
        final String shardIdentifier = ClusterInfo.shardIdentifierFromRouting(shardRouting);
        logger.trace("shard: {} size: {} reserved: {}", shardIdentifier, size, reserved);
        shardSizes.put(shardIdentifier, size);
        if (reserved != StoreStats.UNKNOWN_RESERVED_BYTES) {
            final ClusterInfo.ReservedSpace.Builder reservedSpaceBuilder = reservedSpaceByShard.computeIfAbsent(new ClusterInfo.NodeAndPath(shardRouting.currentNodeId(), s.getDataPath()), t -> new ClusterInfo.ReservedSpace.Builder());
            reservedSpaceBuilder.add(shardRouting.shardId(), reserved);
        }
    }
}