Examples with CommonStatsFlags org.elasticsearch.action.admin.indices.stats.CommonStatsFlags used on opensource projects

Example 6 with CommonStatsFlags

use of org.elasticsearch.action.admin.indices.stats.CommonStatsFlags in project elasticsearch by elastic.

the class IndexRecoveryIT method testRerouteRecovery.

public void testRerouteRecovery() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    ByteSizeValue shardSize = createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT).getShards()[0].getStats().getStore().size();
    logger.info("--> start node B");
    final String nodeB = internalCluster().startNode();
    ensureGreen();
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeB)).execute().actionGet().getState();
    logger.info("--> waiting for recovery to start both on source and target");
    final Index index = resolveIndex(INDEX_NAME);
    assertBusy(new Runnable() {

        @Override
        public void run() {
            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
            indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
        }
    });
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    List<RecoveryState> nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    logger.info("--> request node recovery stats");
    NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    long nodeAThrottling = Long.MAX_VALUE;
    long nodeBThrottling = Long.MAX_VALUE;
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A should have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(1));
            assertThat("node A should not have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(0));
            nodeAThrottling = recoveryStats.throttleTime().millis();
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B should not have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(0));
            assertThat("node B should have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(1));
            nodeBThrottling = recoveryStats.throttleTime().millis();
        }
    }
    logger.info("--> checking throttling increases");
    final long finalNodeAThrottling = nodeAThrottling;
    final long finalNodeBThrottling = nodeBThrottling;
    assertBusy(new Runnable() {

        @Override
        public void run() {
            NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
            assertThat(statsResponse.getNodes(), hasSize(2));
            for (NodeStats nodeStats : statsResponse.getNodes()) {
                final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
                if (nodeStats.getNode().getName().equals(nodeA)) {
                    assertThat("node A throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeAThrottling));
                }
                if (nodeStats.getNode().getName().equals(nodeB)) {
                    assertThat("node B throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeBThrottling));
                }
            }
        }
    });
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    // wait for it to be finished
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(1));
    assertRecoveryState(recoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(recoveryStates.get(0).getIndex());
    statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B throttling should be >0 ", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
    }
    logger.info("--> bump replica count");
    client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet();
    ensureGreen();
    statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B throttling should be >0 ", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
    }
    logger.info("--> start node C");
    String nodeC = internalCluster().startNode();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").get().isTimedOut());
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move replica shard from: {} to: {}", nodeA, nodeC);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeC)).execute().actionGet().getState();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    if (randomBoolean()) {
        // shutdown node with relocation source of replica shard and check if recovery continues
        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
        ensureStableCluster(2);
        response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
        recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
        nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
        assertThat(nodeARecoveryStates.size(), equalTo(0));
        nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
        assertThat(nodeBRecoveryStates.size(), equalTo(1));
        nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
        assertThat(nodeCRecoveryStates.size(), equalTo(1));
        assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
        validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
        assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
        validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    }
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(0));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
}

Example 7 with CommonStatsFlags

use of org.elasticsearch.action.admin.indices.stats.CommonStatsFlags in project elasticsearch by elastic.

the class IndexStatsIT method testFieldDataStats.

public void testFieldDataStats() {
    assertAcked(client().admin().indices().prepareCreate("test").setSettings(settingsBuilder().put("index.number_of_shards", 2)).addMapping("type", "field", "type=text,fielddata=true", "field2", "type=text,fielddata=true").get());
    ensureGreen();
    client().prepareIndex("test", "type", "1").setSource("field", "value1", "field2", "value1").execute().actionGet();
    client().prepareIndex("test", "type", "2").setSource("field", "value2", "field2", "value2").execute().actionGet();
    client().admin().indices().prepareRefresh().execute().actionGet();
    NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
    // sort to load it to field data...
    client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
    client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
    nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
    indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
    // sort to load it to field data...
    client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
    client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
    // now check the per field stats
    nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.FieldData, true).fieldDataFields("*")).execute().actionGet();
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), greaterThan(0L));
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), lessThan(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes()));
    indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).setFieldDataFields("*").execute().actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), greaterThan(0L));
    assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes()));
    client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
    nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
    indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
}

Example 8 with CommonStatsFlags

use of org.elasticsearch.action.admin.indices.stats.CommonStatsFlags in project elasticsearch by elastic.

the class IndexStatsIT method testEncodeDecodeCommonStats.

public void testEncodeDecodeCommonStats() throws IOException {
    CommonStatsFlags flags = new CommonStatsFlags();
    Flag[] values = CommonStatsFlags.Flag.values();
    assertThat(flags.anySet(), equalTo(true));
    for (Flag flag : values) {
        flags.set(flag, false);
    }
    assertThat(flags.anySet(), equalTo(false));
    for (Flag flag : values) {
        flags.set(flag, true);
    }
    assertThat(flags.anySet(), equalTo(true));
    Random random = random();
    flags.set(values[random.nextInt(values.length)], false);
    assertThat(flags.anySet(), equalTo(true));
    {
        BytesStreamOutput out = new BytesStreamOutput();
        flags.writeTo(out);
        out.close();
        BytesReference bytes = out.bytes();
        CommonStatsFlags readStats = new CommonStatsFlags(bytes.streamInput());
        for (Flag flag : values) {
            assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag)));
        }
    }
    {
        for (Flag flag : values) {
            flags.set(flag, random.nextBoolean());
        }
        BytesStreamOutput out = new BytesStreamOutput();
        flags.writeTo(out);
        out.close();
        BytesReference bytes = out.bytes();
        CommonStatsFlags readStats = new CommonStatsFlags(bytes.streamInput());
        for (Flag flag : values) {
            assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag)));
        }
    }
}