Search in sources :

Example 16 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project j2objc by google.

the class LongSummaryStatisticsTest method test_empty.

public void test_empty() {
    LongSummaryStatistics lss = new LongSummaryStatistics();
    assertEquals(0, lss.getCount());
    assertEquals(0, lss.getSum());
    assertEquals(0.0d, lss.getAverage());
    assertEquals(Long.MAX_VALUE, lss.getMin());
    assertEquals(Long.MIN_VALUE, lss.getMax());
}
Also used : LongSummaryStatistics(java.util.LongSummaryStatistics)

Example 17 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project j2objc by google.

the class LongSummaryStatisticsTest method test_accept.

public void test_accept() {
    LongSummaryStatistics lss = new LongSummaryStatistics();
    // For long values
    lss.accept(100L);
    assertEquals(1, lss.getCount());
    assertEquals(100L, lss.getSum());
    lss.accept(250L);
    assertEquals(2, lss.getCount());
    assertEquals(350L, lss.getSum());
    // for int values
    lss.accept(50);
    assertEquals(3, lss.getCount());
    assertEquals(400L, lss.getSum());
    lss.accept(200);
    assertEquals(4, lss.getCount());
    assertEquals(600L, lss.getSum());
}
Also used : LongSummaryStatistics(java.util.LongSummaryStatistics)

Example 18 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project j2objc by google.

the class LongSummaryStatisticsTest method test_getSum.

public void test_getSum() {
    LongSummaryStatistics lss1 = getLongSummaryStatisticsData1();
    assertEquals(104L, lss1.getSum());
}
Also used : LongSummaryStatistics(java.util.LongSummaryStatistics)

Example 19 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project j2objc by google.

the class LongSummaryStatisticsTest method test_getAverage.

public void test_getAverage() {
    LongSummaryStatistics lss1 = getLongSummaryStatisticsData1();
    assertEquals(14.857142, lss1.getAverage(), 1E-6);
}
Also used : LongSummaryStatistics(java.util.LongSummaryStatistics)

Example 20 with LongSummaryStatistics

use of java.util.LongSummaryStatistics in project hive by apache.

the class Initiator method foundCurrentOrFailedCompactions.

private boolean foundCurrentOrFailedCompactions(ShowCompactResponse compactions, CompactionInfo ci) throws MetaException {
    if (compactions.getCompacts() == null) {
        return false;
    }
    List<ShowCompactResponseElement> filteredElements = compactions.getCompacts().stream().filter(e -> e.getDbname().equals(ci.dbname) && e.getTablename().equals(ci.tableName) && (e.getPartitionname() == null && ci.partName == null || e.getPartitionname().equals(ci.partName))).collect(Collectors.toList());
    // Figure out if there are any currently running compactions on the same table or partition.
    if (filteredElements.stream().anyMatch(e -> TxnStore.WORKING_RESPONSE.equals(e.getState()) || TxnStore.INITIATED_RESPONSE.equals(e.getState()))) {
        LOG.info("Found currently initiated or working compaction for " + ci.getFullPartitionName() + " so we will not initiate another compaction");
        return true;
    }
    // Check if there is already sufficient number of consecutive failures for this table/partition
    // so that no new automatic compactions needs to be scheduled.
    int failedThreshold = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
    LongSummaryStatistics failedStats = filteredElements.stream().filter(e -> TxnStore.SUCCEEDED_RESPONSE.equals(e.getState()) || TxnStore.FAILED_RESPONSE.equals(e.getState())).sorted(Comparator.comparingLong(ShowCompactResponseElement::getId).reversed()).limit(failedThreshold).filter(e -> TxnStore.FAILED_RESPONSE.equals(e.getState())).collect(Collectors.summarizingLong(ShowCompactResponseElement::getEnqueueTime));
    // If the last attempt was too long ago, ignore the failed threshold and try compaction again
    long retryTime = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_RETRY_TIME, TimeUnit.MILLISECONDS);
    boolean needsRetry = (retryTime > 0) && (failedStats.getMax() + retryTime < System.currentTimeMillis());
    if (failedStats.getCount() == failedThreshold && !needsRetry) {
        LOG.warn("Will not initiate compaction for " + ci.getFullPartitionName() + " since last " + MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " attempts to compact it failed.");
        ci.errorMessage = "Compaction is not initiated since last " + MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " consecutive compaction attempts failed)";
        txnHandler.markFailed(ci);
        return true;
    }
    return false;
}
Also used : CompactionRequest(org.apache.hadoop.hive.metastore.api.CompactionRequest) ValidWriteIdList(org.apache.hadoop.hive.common.ValidWriteIdList) MetaException(org.apache.hadoop.hive.metastore.api.MetaException) FileSystem(org.apache.hadoop.fs.FileSystem) AcidMetricService(org.apache.hadoop.hive.metastore.metrics.AcidMetricService) LoggerFactory(org.slf4j.LoggerFactory) COMPACTOR_INTIATOR_THREAD_NAME_FORMAT(org.apache.hadoop.hive.conf.Constants.COMPACTOR_INTIATOR_THREAD_NAME_FORMAT) FileStatus(org.apache.hadoop.fs.FileStatus) CompactionType(org.apache.hadoop.hive.metastore.api.CompactionType) NoSuchTxnException(org.apache.hadoop.hive.metastore.api.NoSuchTxnException) ShowCompactRequest(org.apache.hadoop.hive.metastore.api.ShowCompactRequest) AcidDirectory(org.apache.hadoop.hive.ql.io.AcidDirectory) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) ShowCompactResponse(org.apache.hadoop.hive.metastore.api.ShowCompactResponse) StorageDescriptor(org.apache.hadoop.hive.metastore.api.StorageDescriptor) TxnStore(org.apache.hadoop.hive.metastore.txn.TxnStore) CompactionInfo(org.apache.hadoop.hive.metastore.txn.CompactionInfo) ParsedDirectory(org.apache.hadoop.hive.ql.io.AcidUtils.ParsedDirectory) Set(java.util.Set) GetValidWriteIdsRequest(org.apache.hadoop.hive.metastore.api.GetValidWriteIdsRequest) PrivilegedExceptionAction(java.security.PrivilegedExceptionAction) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) CompactionResponse(org.apache.hadoop.hive.metastore.api.CompactionResponse) ValidReadTxnList(org.apache.hadoop.hive.common.ValidReadTxnList) TxnUtils(org.apache.hadoop.hive.metastore.txn.TxnUtils) List(java.util.List) ServerUtils(org.apache.hadoop.hive.common.ServerUtils) MetastoreConf(org.apache.hadoop.hive.metastore.conf.MetastoreConf) Optional(java.util.Optional) CacheBuilder(com.google.common.cache.CacheBuilder) AcidUtils(org.apache.hadoop.hive.ql.io.AcidUtils) HdfsFileStatusWithId(org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Partition(org.apache.hadoop.hive.metastore.api.Partition) TxnCommonUtils(org.apache.hadoop.hive.metastore.txn.TxnCommonUtils) ArrayList(java.util.ArrayList) StringUtils(org.apache.hadoop.util.StringUtils) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) MetricsConstants(org.apache.hadoop.hive.metastore.metrics.MetricsConstants) LongSummaryStatistics(java.util.LongSummaryStatistics) ExecutorService(java.util.concurrent.ExecutorService) Ref(org.apache.hive.common.util.Ref) Logger(org.slf4j.Logger) HiveConf(org.apache.hadoop.hive.conf.HiveConf) IOException(java.io.IOException) Table(org.apache.hadoop.hive.metastore.api.Table) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Metrics(org.apache.hadoop.hive.metastore.metrics.Metrics) ValidTxnList(org.apache.hadoop.hive.common.ValidTxnList) MetaStoreUtils.isNoAutoCompactSet(org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.isNoAutoCompactSet) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Cache(com.google.common.cache.Cache) Comparator(java.util.Comparator) org.apache.hadoop.hive.metastore.api.hive_metastoreConstants(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants) Collections(java.util.Collections) PerfLogger(org.apache.hadoop.hive.metastore.metrics.PerfLogger) LongSummaryStatistics(java.util.LongSummaryStatistics) ShowCompactResponseElement(org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement)

Aggregations

LongSummaryStatistics (java.util.LongSummaryStatistics)47 Test (org.junit.Test)18 List (java.util.List)15 Map (java.util.Map)12 CountDownLatch (java.util.concurrent.CountDownLatch)11 ArrayList (java.util.ArrayList)10 Collectors (java.util.stream.Collectors)10 Set (java.util.Set)9 TimeUnit (java.util.concurrent.TimeUnit)9 LoggerFactory (org.slf4j.LoggerFactory)9 HashMap (java.util.HashMap)8 Logger (org.slf4j.Logger)8 BaseTest (io.scalecube.testlib.BaseTest)7 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)6 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)6 Arrays (java.util.Arrays)5 Stream (java.util.stream.Stream)5 Collections (java.util.Collections)4 Comparator (java.util.Comparator)4 Entry (java.util.Map.Entry)4