Examples with Pair com.github.ambry.utils.Pair used on opensource projects

Example 31 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class BlobStoreCompactor method setupState.

// copy() helpers
/**
 * Sets up the state required for copying data by populating all the data structures, creating a log and index that
 * wraps the swap spaces and pausing hard delete in the application log.
 * @throws IOException if logs and indexes could not be set up.
 * @throws StoreException if there were exceptions reading to writing to store components.
 */
private void setupState() throws IOException, StoreException {
    CompactionDetails details = compactionLog.getCompactionDetails();
    long highestGeneration = 0;
    for (LogSegmentName segmentName : details.getLogSegmentsUnderCompaction()) {
        highestGeneration = Math.max(highestGeneration, segmentName.getGeneration());
    }
    logger.debug("Generation of target segments will be {} for {} in {}", highestGeneration + 1, details, storeId);
    List<Pair<LogSegmentName, String>> targetSegmentNamesAndFilenames = new ArrayList<>();
    List<LogSegment> existingTargetLogSegments = new ArrayList<>();
    for (LogSegmentName segmentName : details.getLogSegmentsUnderCompaction()) {
        long pos = segmentName.getPosition();
        LogSegmentName targetSegmentName = LogSegmentName.fromPositionAndGeneration(pos, highestGeneration + 1);
        String targetSegmentFileName = targetSegmentName.toFilename() + TEMP_LOG_SEGMENT_NAME_SUFFIX;
        File targetSegmentFile = new File(dataDir, targetSegmentFileName);
        if (targetSegmentFile.exists()) {
            existingTargetLogSegments.add(new LogSegment(targetSegmentName, targetSegmentFile, config, tgtMetrics));
        } else {
            targetSegmentNamesAndFilenames.add(new Pair<>(targetSegmentName, targetSegmentFileName));
        }
    }
    // TODO: available swap space count should be obtained from DiskManager. For now, assumed to be 1.
    long targetLogTotalCapacity = srcLog.getSegmentCapacity();
    logger.debug("Target log capacity is {} for {}. Existing log segments are {}. Future names and files are {}", targetLogTotalCapacity, storeId, existingTargetLogSegments, targetSegmentNamesAndFilenames);
    tgtLog = new Log(dataDir.getAbsolutePath(), targetLogTotalCapacity, diskSpaceAllocator, config, tgtMetrics, true, existingTargetLogSegments, targetSegmentNamesAndFilenames.iterator(), diskMetrics);
    Journal journal = new Journal(dataDir.getAbsolutePath(), 2 * config.storeIndexMaxNumberOfInmemElements, config.storeMaxNumberOfEntriesToReturnFromJournal);
    tgtIndex = new PersistentIndex(dataDir.getAbsolutePath(), storeId, null, tgtLog, config, storeKeyFactory, null, null, diskIOScheduler, tgtMetrics, journal, time, sessionId, incarnationId, TARGET_INDEX_CLEAN_SHUTDOWN_FILE_NAME);
    if (srcIndex.hardDeleter != null && !srcIndex.hardDeleter.isPaused()) {
        logger.debug("Pausing hard delete for {}", storeId);
        srcIndex.hardDeleter.pause();
    }
}

Example 32 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class HelixClusterAggregator method doWorkOnStatsWrapperMap.

Pair<StatsSnapshot, StatsSnapshot> doWorkOnStatsWrapperMap(Map<String, StatsWrapper> statsWrappers, StatsReportType type, boolean removeExceptionOnType) throws IOException {
    StatsSnapshot partitionSnapshot = new StatsSnapshot(0L, new HashMap<>());
    Map<String, Long> partitionTimestampMap = new HashMap<>();
    StatsSnapshot rawPartitionSnapshot = new StatsSnapshot(0L, new HashMap<>());
    if (removeExceptionOnType) {
        exceptionOccurredInstances.remove(type);
    }
    for (Map.Entry<String, StatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry != null && statsWrapperEntry.getValue() != null) {
            try {
                StatsWrapper snapshotWrapper = statsWrapperEntry.getValue();
                StatsWrapper snapshotWrapperCopy = new StatsWrapper(new StatsHeader(snapshotWrapper.getHeader()), new StatsSnapshot(snapshotWrapper.getSnapshot()));
                combineRawStats(rawPartitionSnapshot, snapshotWrapper);
                switch(type) {
                    case ACCOUNT_REPORT:
                        combineValidStatsByAccount(partitionSnapshot, snapshotWrapperCopy, statsWrapperEntry.getKey(), partitionTimestampMap);
                        break;
                    case PARTITION_CLASS_REPORT:
                        combineValidStatsByPartitionClass(partitionSnapshot, snapshotWrapperCopy, statsWrapperEntry.getKey(), partitionTimestampMap);
                        break;
                    default:
                        throw new IllegalArgumentException("Unrecognized stats report type: " + type);
                }
            } catch (Exception e) {
                logger.error("Exception occurred while processing stats from {}", statsWrapperEntry.getKey(), e);
                exceptionOccurredInstances.computeIfAbsent(type, key -> new ArrayList<>()).add(statsWrapperEntry.getKey());
            }
        }
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw snapshot {}", mapper.writeValueAsString(rawPartitionSnapshot));
        logger.trace("Combined valid snapshot {}", mapper.writeValueAsString(partitionSnapshot));
    }
    StatsSnapshot reducedRawSnapshot;
    StatsSnapshot reducedSnapshot;
    switch(type) {
        case ACCOUNT_REPORT:
            reducedRawSnapshot = reduceByAccount(rawPartitionSnapshot);
            reducedSnapshot = reduceByAccount(partitionSnapshot);
            break;
        case PARTITION_CLASS_REPORT:
            reducedRawSnapshot = reduceByPartitionClass(rawPartitionSnapshot);
            reducedSnapshot = reduceByPartitionClass(partitionSnapshot);
            break;
        default:
            throw new IllegalArgumentException("Unrecognized stats report type: " + type);
    }
    reducedRawSnapshot.removeZeroValueSnapshots();
    reducedSnapshot.removeZeroValueSnapshots();
    if (logger.isTraceEnabled()) {
        logger.trace("Reduced raw snapshot {}", mapper.writeValueAsString(reducedRawSnapshot));
        logger.trace("Reduced valid snapshot {}", mapper.writeValueAsString(reducedSnapshot));
    }
    return new Pair<>(reducedRawSnapshot, reducedSnapshot);
}

Example 33 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostAccountStorageStatsWrappers.

/**
 * Aggregate all {@link HostAccountStorageStatsWrapper} to generate two {@link AggregatedAccountStorageStats}s. First
 * {@link AggregatedAccountStorageStats} is the sum of all {@link HostAccountStorageStatsWrapper}s. The second {@link AggregatedAccountStorageStats}
 * is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostAccountStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedAccountStorageStats}.
 * @throws IOException
 */
Pair<AggregatedAccountStorageStats, AggregatedAccountStorageStats> aggregateHostAccountStorageStatsWrappers(Map<String, HostAccountStorageStatsWrapper> statsWrappers) throws IOException {
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> combinedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Map<Short, Map<Short, ContainerStorageStats>>> selectedHostAccountStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostAccountStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostAccountStorageStatsWrapper hostAccountStorageStatsWrapper = statsWrapperEntry.getValue();
        HostAccountStorageStats hostAccountStorageStats = hostAccountStorageStatsWrapper.getStats();
        HostAccountStorageStats hostAccountStorageStatsCopy1 = new HostAccountStorageStats(hostAccountStorageStats);
        HostAccountStorageStats hostAccountStorageStatsCopy2 = new HostAccountStorageStats(hostAccountStorageStats);
        combineRawHostAccountStorageStatsMap(combinedHostAccountStorageStatsMap, hostAccountStorageStatsCopy1.getStorageStats());
        selectRawHostAccountStorageStatsMap(selectedHostAccountStorageStatsMap, hostAccountStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostAccountStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostAccountStorageStats {}", mapper.writeValueAsString(combinedHostAccountStorageStatsMap));
        logger.trace("Selected raw HostAccountStorageStats {}", mapper.writeValueAsString(selectedHostAccountStorageStatsMap));
    }
    AggregatedAccountStorageStats combinedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(combinedHostAccountStorageStatsMap));
    AggregatedAccountStorageStats selectedAggregated = new AggregatedAccountStorageStats(aggregateHostAccountStorageStats(selectedHostAccountStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}

Example 34 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class MySqlClusterAggregator method aggregateHostPartitionClassStorageStatsWrappers.

/**
 * Aggregate all {@link HostPartitionClassStorageStatsWrapper} to generate two {@link AggregatedPartitionClassStorageStats}s. First
 * {@link AggregatedPartitionClassStorageStats} is the sum of all {@link HostPartitionClassStorageStatsWrapper}s. The second
 * {@link AggregatedPartitionClassStorageStats} is the valid aggregated storage stats for all replicas of each partition.
 * @param statsWrappers A map from instance name to {@link HostPartitionClassStorageStatsWrapper}.
 * @return A {@link Pair} of {@link AggregatedPartitionClassStorageStats}.
 * @throws IOException
 */
Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregateHostPartitionClassStorageStatsWrappers(Map<String, HostPartitionClassStorageStatsWrapper> statsWrappers) throws IOException {
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> combinedHostPartitionClassStorageStatsMap = new HashMap<>();
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> selectedHostPartitionClassStorageStatsMap = new HashMap<>();
    Map<Long, Long> partitionTimestampMap = new HashMap<>();
    Map<Long, Long> partitionPhysicalStorageMap = new HashMap<>();
    for (Map.Entry<String, HostPartitionClassStorageStatsWrapper> statsWrapperEntry : statsWrappers.entrySet()) {
        if (statsWrapperEntry.getValue() == null) {
            continue;
        }
        String instanceName = statsWrapperEntry.getKey();
        HostPartitionClassStorageStatsWrapper hostPartitionClassStorageStatsWrapper = statsWrapperEntry.getValue();
        HostPartitionClassStorageStats hostPartitionClassStorageStats = hostPartitionClassStorageStatsWrapper.getStats();
        HostPartitionClassStorageStats hostPartitionClassStorageStatsCopy1 = new HostPartitionClassStorageStats(hostPartitionClassStorageStats);
        HostPartitionClassStorageStats hostPartitionClassStorageStatsCopy2 = new HostPartitionClassStorageStats(hostPartitionClassStorageStats);
        combineRawHostPartitionClassStorageStatsMap(combinedHostPartitionClassStorageStatsMap, hostPartitionClassStorageStatsCopy1.getStorageStats());
        selectRawHostPartitionClassStorageStatsMap(selectedHostPartitionClassStorageStatsMap, hostPartitionClassStorageStatsCopy2.getStorageStats(), partitionTimestampMap, partitionPhysicalStorageMap, hostPartitionClassStorageStatsWrapper.getHeader().getTimestamp(), instanceName);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("Combined raw HostPartitionClassStorageStats {}", mapper.writeValueAsString(combinedHostPartitionClassStorageStatsMap));
        logger.trace("Selected raw HostPartitionClassStorageStats {}", mapper.writeValueAsString(selectedHostPartitionClassStorageStatsMap));
    }
    AggregatedPartitionClassStorageStats combinedAggregated = new AggregatedPartitionClassStorageStats(aggregateHostPartitionClassStorageStats(combinedHostPartitionClassStorageStatsMap));
    AggregatedPartitionClassStorageStats selectedAggregated = new AggregatedPartitionClassStorageStats(aggregateHostPartitionClassStorageStats(selectedHostPartitionClassStorageStatsMap));
    if (logger.isTraceEnabled()) {
        logger.trace("Aggregated combined {}", mapper.writeValueAsString(combinedAggregated));
        logger.trace("Aggregated selected {}", mapper.writeValueAsString(selectedAggregated));
    }
    return new Pair<>(combinedAggregated, selectedAggregated);
}

Example 35 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class MySqlReportAggregatorTask method removeInvalidAggregatedPartitionClassStats.

private void removeInvalidAggregatedPartitionClassStats(AggregatedPartitionClassStorageStats currentStats) throws Exception {
    List<Pair<String, Pair<Short, Short>>> toBeDeletedPartitionClassNameAndAccountContainer = new ArrayList<>();
    AggregatedPartitionClassStorageStats existingStats = accountStatsStore.queryAggregatedPartitionClassStorageStats();
    for (String partitionClassName : existingStats.getStorageStats().keySet()) {
        Map<Short, Map<Short, ContainerStorageStats>> currentAccountMap = currentStats.getStorageStats().get(partitionClassName);
        for (Map.Entry<Short, Map<Short, ContainerStorageStats>> accountEntry : existingStats.getStorageStats().get(partitionClassName).entrySet()) {
            short accountId = accountEntry.getKey();
            for (short containerId : accountEntry.getValue().keySet()) {
                if (currentAccountMap == null || !currentAccountMap.containsKey(accountId) || !currentAccountMap.get(accountId).containsKey(containerId)) {
                    toBeDeletedPartitionClassNameAndAccountContainer.add(new Pair<>(partitionClassName, new Pair<>(accountId, containerId)));
                }
            }
        }
    }
    for (Pair<String, Pair<Short, Short>> pair : toBeDeletedPartitionClassNameAndAccountContainer) {
        Pair<Short, Short> accountContainerId = pair.getSecond();
        accountStatsStore.deleteAggregatedPartitionClassStatsForAccountContainer(pair.getFirst(), accountContainerId.getFirst(), accountContainerId.getSecond());
    }
}