Examples with ByteSizeValue org.elasticsearch.common.unit.ByteSizeValue used on opensource projects

Example 66 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project crate by crate.

the class PrimaryReplicaSyncerTests method testSyncerSendsOffCorrectDocuments.

@Test
public void testSyncerSendsOffCorrectDocuments() throws Exception {
    IndexShard shard = newStartedShard(true);
    AtomicBoolean syncActionCalled = new AtomicBoolean();
    List<ResyncReplicationRequest> resyncRequests = new ArrayList<>();
    PrimaryReplicaSyncer.SyncAction syncAction = (request, allocationId, primaryTerm, listener) -> {
        logger.info("Sending off {} operations", request.getOperations().length);
        syncActionCalled.set(true);
        resyncRequests.add(request);
        listener.onResponse(new ReplicationResponse());
    };
    PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(syncAction);
    syncer.setChunkSize(new ByteSizeValue(randomIntBetween(1, 10)));
    int numDocs = randomInt(10);
    for (int i = 0; i < numDocs; i++) {
        // Index doc but not advance local checkpoint.
        shard.applyIndexOperationOnPrimary(Versions.MATCH_ANY, VersionType.INTERNAL, new SourceToParse(shard.shardId().getIndexName(), Integer.toString(i), new BytesArray("{}"), XContentType.JSON), SequenceNumbers.UNASSIGNED_SEQ_NO, 0, -1L, true);
    }
    long globalCheckPoint = numDocs > 0 ? randomIntBetween(0, numDocs - 1) : 0;
    boolean syncNeeded = numDocs > 0;
    String allocationId = shard.routingEntry().allocationId().getId();
    shard.updateShardState(shard.routingEntry(), shard.getPendingPrimaryTerm(), null, 1000L, Collections.singleton(allocationId), new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build());
    shard.updateLocalCheckpointForShard(allocationId, globalCheckPoint);
    assertEquals(globalCheckPoint, shard.getLastKnownGlobalCheckpoint());
    logger.info("Total ops: {}, global checkpoint: {}", numDocs, globalCheckPoint);
    PlainActionFuture<PrimaryReplicaSyncer.ResyncTask> fut = new PlainActionFuture<>();
    syncer.resync(shard, fut);
    PrimaryReplicaSyncer.ResyncTask resyncTask = fut.get();
    if (syncNeeded) {
        assertTrue("Sync action was not called", syncActionCalled.get());
        ResyncReplicationRequest resyncRequest = resyncRequests.remove(0);
        assertThat(resyncRequest.getTrimAboveSeqNo(), equalTo(numDocs - 1L));
        assertThat("trimAboveSeqNo has to be specified in request #0 only", resyncRequests.stream().mapToLong(ResyncReplicationRequest::getTrimAboveSeqNo).filter(seqNo -> seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO).findFirst().isPresent(), is(false));
        assertThat(resyncRequest.getMaxSeenAutoIdTimestampOnPrimary(), equalTo(shard.getMaxSeenAutoIdTimestamp()));
    }
    if (syncNeeded && globalCheckPoint < numDocs - 1) {
        assertThat(resyncTask.getSkippedOperations(), equalTo(0));
        assertThat(resyncTask.getResyncedOperations(), equalTo(Math.toIntExact(numDocs - 1 - globalCheckPoint)));
        if (shard.indexSettings.isSoftDeleteEnabled()) {
            assertThat(resyncTask.getTotalOperations(), equalTo(Math.toIntExact(numDocs - 1 - globalCheckPoint)));
        } else {
            assertThat(resyncTask.getTotalOperations(), equalTo(numDocs));
        }
    } else {
        assertThat(resyncTask.getSkippedOperations(), equalTo(0));
        assertThat(resyncTask.getResyncedOperations(), equalTo(0));
        assertThat(resyncTask.getTotalOperations(), equalTo(0));
    }
    closeShards(shard);
}

Example 67 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project crate by crate.

the class NodeEnvironment method maybeLogHeapDetails.

private void maybeLogHeapDetails() {
    JvmInfo jvmInfo = JvmInfo.jvmInfo();
    ByteSizeValue maxHeapSize = jvmInfo.getMem().getHeapMax();
    String useCompressedOops = jvmInfo.useCompressedOops();
    logger.info("heap size [{}], compressed ordinary object pointers [{}]", maxHeapSize, useCompressedOops);
}

Example 68 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project crate by crate.

the class JvmGcMonitorService method logSlowGc.

static void logSlowGc(final Logger logger, final JvmMonitor.Threshold threshold, final long seq, final JvmMonitor.SlowGcEvent slowGcEvent, BiFunction<JvmStats, JvmStats, String> pools) {
    final String name = slowGcEvent.currentGc.getName();
    final long elapsed = slowGcEvent.elapsed;
    final long totalGcCollectionCount = slowGcEvent.currentGc.getCollectionCount();
    final long currentGcCollectionCount = slowGcEvent.collectionCount;
    final TimeValue totalGcCollectionTime = slowGcEvent.currentGc.getCollectionTime();
    final TimeValue currentGcCollectionTime = slowGcEvent.collectionTime;
    final JvmStats lastJvmStats = slowGcEvent.lastJvmStats;
    final JvmStats currentJvmStats = slowGcEvent.currentJvmStats;
    final ByteSizeValue maxHeapUsed = slowGcEvent.maxHeapUsed;
    switch(threshold) {
        case WARN:
            if (logger.isWarnEnabled()) {
                logger.warn(SLOW_GC_LOG_MESSAGE, name, seq, totalGcCollectionCount, currentGcCollectionTime, currentGcCollectionCount, TimeValue.timeValueMillis(elapsed), currentGcCollectionTime, totalGcCollectionTime, lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), maxHeapUsed, pools.apply(lastJvmStats, currentJvmStats));
            }
            break;
        case INFO:
            if (logger.isInfoEnabled()) {
                logger.info(SLOW_GC_LOG_MESSAGE, name, seq, totalGcCollectionCount, currentGcCollectionTime, currentGcCollectionCount, TimeValue.timeValueMillis(elapsed), currentGcCollectionTime, totalGcCollectionTime, lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), maxHeapUsed, pools.apply(lastJvmStats, currentJvmStats));
            }
            break;
        case DEBUG:
            if (logger.isDebugEnabled()) {
                logger.debug(SLOW_GC_LOG_MESSAGE, name, seq, totalGcCollectionCount, currentGcCollectionTime, currentGcCollectionCount, TimeValue.timeValueMillis(elapsed), currentGcCollectionTime, totalGcCollectionTime, lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), maxHeapUsed, pools.apply(lastJvmStats, currentJvmStats));
            }
            break;
        default:
            break;
    }
}

Example 69 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project crate by crate.

the class MemoryCircuitBreaker method addEstimateBytesAndMaybeBreak.

/**
 * Add a number of bytes, tripping the circuit breaker if the aggregated
 * estimates are above the limit. Automatically trips the breaker if the
 * memory limit is set to 0. Will never trip the breaker if the limit is
 * set < 0, but can still be used to aggregate estimations.
 * @param bytes number of bytes to add to the breaker
 * @return number of "used" bytes so far
 */
@Override
public double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException {
    // short-circuit on no data allowed, immediately throwing an exception
    if (memoryBytesLimit == 0) {
        circuitBreak(label, bytes);
    }
    long newUsed;
    // limit), which makes the RamAccountingTermsEnum case faster.
    if (this.memoryBytesLimit == -1) {
        newUsed = this.used.addAndGet(bytes);
        if (logger.isTraceEnabled()) {
            logger.trace("Adding [{}][{}] to used bytes [new used: [{}], limit: [-1b]]", new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed));
        }
        return newUsed;
    }
    // Otherwise, check the addition and commit the addition, looping if
    // there are conflicts. May result in additional logging, but it's
    // trace logging and shouldn't be counted on for additions.
    long currentUsed;
    do {
        currentUsed = this.used.get();
        newUsed = currentUsed + bytes;
        long newUsedWithOverhead = (long) (newUsed * overheadConstant);
        if (logger.isTraceEnabled()) {
            logger.trace("Adding [{}][{}] to used bytes [new used: [{}], limit: {} [{}], estimate: {} [{}]]", new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed), memoryBytesLimit, new ByteSizeValue(memoryBytesLimit), newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead));
        }
        if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) {
            logger.warn("New used memory {} [{}] from field [{}] would be larger than configured breaker: {} [{}], breaking", newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label, memoryBytesLimit, new ByteSizeValue(memoryBytesLimit));
            circuitBreak(label, newUsedWithOverhead);
        }
    // Attempt to set the new used value, but make sure it hasn't changed
    // underneath us, if it has, keep trying until we are able to set it
    } while (!this.used.compareAndSet(currentUsed, newUsed));
    return newUsed;
}

Example 70 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project crate by crate.

the class MemorySizeSettingsTests method testCircuitBreakerSettings.

@Test
public void testCircuitBreakerSettings() {
    double defaultTotalPercentage = 0.95d;
    assertMemorySizeSetting(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.total.limit", new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * defaultTotalPercentage)));
    assertMemorySizeSetting(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING, "indices.breaker.request.limit", new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.6)));
    assertMemorySizeSetting(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING, "network.breaker.inflight_requests.limit", new ByteSizeValue((JvmInfo.jvmInfo().getMem().getHeapMax().getBytes())));
}