Examples with Pair com.github.ambry.utils.Pair used on opensource projects

Example 11 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class MockBlobStoreStats method initializeBlobStore.

// helper methods
/**
 * Initializes {@link BlobStore}
 * @param minLogSizeToTriggerCompactionInPercentage Property value to be set for
 * {@link StoreConfig#storeMinUsedCapacityToTriggerCompactionInPercentage}
 * @param messageRetentionInHours Property value to be set for {@link StoreConfig#storeDeletedMessageRetentionHours
 * @throws InterruptedException
 */
static Pair<MockBlobStore, StoreConfig> initializeBlobStore(Properties properties, Time time, int minLogSizeToTriggerCompactionInPercentage, int messageRetentionInHours, long maxBlobSize) throws InterruptedException {
    if (minLogSizeToTriggerCompactionInPercentage != -1) {
        properties.setProperty("store.min.log.size.to.trigger.compaction.in.percent", String.valueOf(minLogSizeToTriggerCompactionInPercentage));
    }
    if (messageRetentionInHours != -1) {
        properties.setProperty("store.deleted.message.retention.hours", String.valueOf(messageRetentionInHours));
    }
    StoreConfig config = new StoreConfig(new VerifiableProperties(properties));
    time.sleep(10 * TimeUnit.HOURS.toMillis(config.storeDeletedMessageRetentionHours));
    MetricRegistry metricRegistry = new MetricRegistry();
    StoreMetrics metrics = new StoreMetrics(metricRegistry);
    MockBlobStoreStats mockBlobStoreStats = new MockBlobStoreStats(maxBlobSize);
    MockBlobStore blobStore = new MockBlobStore(config, metrics, time, CAPACITY_IN_BYTES, SEGMENT_CAPACITY_IN_BYTES, SEGMENT_HEADER_SIZE, DEFAULT_USED_CAPACITY_IN_BYTES, mockBlobStoreStats);
    return new Pair<>(blobStore, config);
}

Example 12 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class LogTest method logSegmentCustomNamesTest.

/**
 * Checks that the constructor that receives segments and segment names iterator,
 * {@link Log#Log(String, long, DiskSpaceAllocator, StoreConfig, StoreMetrics, boolean, List, Iterator, DiskMetrics)}, loads
 * the segments correctly and uses the iterator to name new segments and uses the default algorithm once the names
 * run out.
 * @throws IOException
 */
@Test
public void logSegmentCustomNamesTest() throws IOException, StoreException {
    int numSegments = (int) (LOG_CAPACITY / SEGMENT_CAPACITY);
    LogSegment segment = getLogSegment(LogSegmentName.fromPositionAndGeneration(0, 0), SEGMENT_CAPACITY, true);
    long startPos = 2 * numSegments;
    List<Pair<LogSegmentName, String>> expectedSegmentAndFileNames = new ArrayList<>(numSegments);
    expectedSegmentAndFileNames.add(new Pair<>(segment.getName(), segment.getView().getFirst().getName()));
    List<Pair<LogSegmentName, String>> segmentNameAndFileNamesDesired = new ArrayList<>();
    LogSegmentName lastName = null;
    for (int i = 0; i < 2; i++) {
        lastName = LogSegmentName.fromPositionAndGeneration(startPos + i, 0);
        String fileName = lastName.toFilename() + "_modified";
        segmentNameAndFileNamesDesired.add(new Pair<>(lastName, fileName));
        expectedSegmentAndFileNames.add(new Pair<>(lastName, fileName));
    }
    for (int i = expectedSegmentAndFileNames.size(); i < numSegments; i++) {
        lastName = lastName.getNextPositionName();
        String fileName = lastName.toFilename();
        expectedSegmentAndFileNames.add(new Pair<>(lastName, fileName));
    }
    Log log = new Log(tempDir.getAbsolutePath(), LOG_CAPACITY, StoreTestUtils.DEFAULT_DISK_SPACE_ALLOCATOR, createStoreConfig(SEGMENT_CAPACITY, setFilePermissionEnabled), metrics, true, Collections.singletonList(segment), segmentNameAndFileNamesDesired.iterator(), null);
    // write enough so that all segments are allocated
    ByteBuffer buffer = ByteBuffer.allocate((int) (segment.getCapacityInBytes() - segment.getStartOffset()));
    for (int i = 0; i < numSegments; i++) {
        buffer.rewind();
        CHANNEL_APPENDER.append(log, buffer);
    }
    segment = log.getFirstSegment();
    for (Pair<LogSegmentName, String> nameAndFilename : expectedSegmentAndFileNames) {
        assertEquals("Segment name does not match", nameAndFilename.getFirst(), segment.getName());
        assertEquals("Segment file does not match", nameAndFilename.getSecond(), segment.getView().getFirst().getName());
        segment = log.getNextSegment(segment);
    }
    assertNull("There should be no more segments", segment);
}

Example 13 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class MySqlClusterAggregatorTest method testAggregateHostPartitionClassStorageStats.

/**
 * Test basic functionality of {@link MySqlClusterAggregator#aggregateHostPartitionClassStorageStats}.
 * @throws Exception
 */
@Test
public void testAggregateHostPartitionClassStorageStats() throws Exception {
    int nodeCount = 3;
    int numberOfPartitions = 4;
    Map<String, Map<Long, Map<Short, Map<Short, ContainerStorageStats>>>> storageStatsMap = new HashMap<>();
    String[] partitionClassNames = { "default", "newClass" };
    for (int i = 0; i < numberOfPartitions; i++) {
        String partitionClassName = partitionClassNames[i % partitionClassNames.length];
        storageStatsMap.computeIfAbsent(partitionClassName, k -> new HashMap<>()).put((long) i, StorageStatsUtilTest.generateRandomAggregatedAccountStorageStats((short) 0, i + 3, 3, 10000L, 2, 10));
    }
    StatsHeader header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, numberOfPartitions, numberOfPartitions, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper nodeStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats(storageStatsMap));
    header = new StatsHeader(StatsHeader.StatsDescription.STORED_DATA_SIZE, DEFAULT_TIMESTAMP, 0, 0, Collections.emptyList());
    HostPartitionClassStorageStatsWrapper emptyStats = new HostPartitionClassStorageStatsWrapper(header, new HostPartitionClassStorageStats());
    Map<String, HostPartitionClassStorageStatsWrapper> instanceToStatsMap = new HashMap<>();
    for (int i = 0; i < nodeCount; i++) {
        instanceToStatsMap.put("Instance_" + i, new HostPartitionClassStorageStatsWrapper(new StatsHeader(nodeStats.getHeader()), new HostPartitionClassStorageStats(nodeStats.getStats())));
    }
    instanceToStatsMap.put("Instance_" + nodeCount, emptyStats);
    Pair<AggregatedPartitionClassStorageStats, AggregatedPartitionClassStorageStats> aggregatedRawAndValidStats = clusterAggregator.aggregateHostPartitionClassStorageStatsWrappers(instanceToStatsMap);
    Map<String, Map<Short, Map<Short, ContainerStorageStats>>> expectedAggregatedStorageStatsMap = clusterAggregator.aggregateHostPartitionClassStorageStats(storageStatsMap);
    Assert.assertEquals(expectedAggregatedStorageStatsMap, aggregatedRawAndValidStats.getSecond().getStorageStats());
    assertAggregatedRawStatsForPartitionClassStorageStats(aggregatedRawAndValidStats.getFirst().getStorageStats(), expectedAggregatedStorageStatsMap, nodeCount);
}

Example 14 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class BlobIdTest method testComparisons.

/**
 * Tests blobIds comparisons. Among other things, ensures the following requirements are met:
 * <br>
 * V1s are always less than V2s and V3s.
 * V2s are always less than V3s.
 */
@Test
public void testComparisons() {
    // the version check is to do this inter-version test just once (since this is a parametrized test).
    assumeTrue(version == BLOB_ID_V1);
    for (int i = 0; i < 100; i++) {
        Map<Short, Pair<BlobId, BlobId>> blobIds = Arrays.stream(BlobId.getAllValidVersions()).collect(Collectors.toMap(Function.identity(), v -> new Pair<>(getRandomBlobId(v), getRandomBlobId(v))));
        for (short version : BlobId.getAllValidVersions()) {
            BlobId blobId = blobIds.get(version).getFirst();
            BlobId altBlobId = blobIds.get(version).getSecond();
            assertEquals("blobIdV" + version + " should be equal to itself", 0, blobId.compareTo(blobId));
            assertEquals("blobIdV" + version + " should be equal to itself", blobId, blobId);
            assertThat("Two randomly generated blobIdV" + version + "s should be unequal", blobId.compareTo(altBlobId), not(0));
            assertThat("Two randomly generated blobIdV" + version + "s should be unequal", blobId, not(altBlobId));
            for (short otherVersion = 1; otherVersion < version; otherVersion++) {
                BlobId otherBlobId = blobIds.get(otherVersion).getFirst();
                assertThat("blobIdV" + otherVersion + " should not equal blobIdV" + version, otherBlobId, not(blobId));
                assertThat("blobIdV" + version + " should not equal blobIdV" + otherVersion, blobId, not(otherBlobId));
                boolean differentVersionGroup = version < BLOB_ID_V3 || (version < BLOB_ID_V6 ? otherVersion < BLOB_ID_V3 : otherVersion < BLOB_ID_V6);
                if (differentVersionGroup) {
                    assertTrue("blobIdV" + otherVersion + " should be less than blobIdV" + version, otherBlobId.compareTo(blobId) < 0);
                    assertTrue("blobIdV" + version + " should be greater than blobIdV" + otherVersion, blobId.compareTo(otherBlobId) > 0);
                } else {
                    assertEquals("Comparison between blobIdV" + version + " and blobIDV" + otherVersion + " are based on uuid only", blobId.getUuid().compareTo(otherBlobId.getUuid()), blobId.compareTo(otherBlobId));
                    assertEquals("Comparison between blobIdV" + otherVersion + " and blobIDV" + version + " are based on uuid only", otherBlobId.getUuid().compareTo(blobId.getUuid()), otherBlobId.compareTo(blobId));
                }
            }
        }
    }
}

Example 15 with Pair

use of com.github.ambry.utils.Pair in project ambry by linkedin.

the class AccountStatsMySqlStoreIntegrationTest method testHostPartitionClassStats.

/**
 * Test methods to store and fetch partition class, partition name partition id and partition class stats.
 * @throws Exception
 */
@Test
public void testHostPartitionClassStats() throws Exception {
    // First write some stats to account reports
    testMultiStoreStats();
    StatsWrapper accountStats1 = mySqlStore.queryAccountStatsByHost(hostname1, port1);
    StatsWrapper accountStats2 = mySqlStore.queryAccountStatsByHost(hostname2, port2);
    AccountStatsMySqlStore mySqlStore3 = createAccountStatsMySqlStore(clusterName2, hostname3, port3);
    StatsWrapper accountStats3 = mySqlStore3.queryAccountStatsByHost(hostname3, port3);
    // From this account stats, create partition class stats;
    Set<String> allPartitionKeys = new HashSet<String>() {

        {
            addAll(accountStats1.getSnapshot().getSubMap().keySet());
            addAll(accountStats2.getSnapshot().getSubMap().keySet());
            addAll(accountStats3.getSnapshot().getSubMap().keySet());
        }
    };
    List<String> partitionClassNames = Arrays.asList("default", "new");
    Map<String, String> partitionKeyToClassName = new HashMap<>();
    int ind = 0;
    for (String partitionKey : allPartitionKeys) {
        partitionKeyToClassName.put(partitionKey, partitionClassNames.get(ind % partitionClassNames.size()));
        ind++;
    }
    StatsWrapper partitionClassStats1 = convertAccountStatsToPartitionClassStats(accountStats1, partitionKeyToClassName);
    StatsWrapper partitionClassStats2 = convertAccountStatsToPartitionClassStats(accountStats2, partitionKeyToClassName);
    StatsWrapper partitionClassStats3 = convertAccountStatsToPartitionClassStats(accountStats3, partitionKeyToClassName);
    mySqlStore.storePartitionClassStats(partitionClassStats1);
    mySqlStore.storePartitionClassStats(partitionClassStats2);
    mySqlStore3.storePartitionClassStats(partitionClassStats3);
    Map<String, Set<Integer>> partitionNameAndIds = mySqlStore.queryPartitionNameAndIds();
    assertEquals(new HashSet<>(partitionClassNames), partitionNameAndIds.keySet());
    Map<String, String> dbPartitionKeyToClassName = partitionNameAndIds.entrySet().stream().flatMap(ent -> ent.getValue().stream().map(pid -> new Pair<String, String>(ent.getKey(), "Partition[" + pid + "]"))).collect(Collectors.toMap(Pair::getSecond, Pair::getFirst));
    assertEquals(partitionKeyToClassName, dbPartitionKeyToClassName);
    StatsWrapper obtainedStats1 = mySqlStore.queryPartitionClassStatsByHost(hostname1, port1, partitionNameAndIds);
    assertEquals(partitionClassStats1.getSnapshot(), obtainedStats1.getSnapshot());
    StatsWrapper obtainedStats2 = mySqlStore.queryPartitionClassStatsByHost(hostname2, port2, partitionNameAndIds);
    assertEquals(partitionClassStats2.getSnapshot(), obtainedStats2.getSnapshot());
    StatsWrapper obtainedStats3 = mySqlStore3.queryPartitionClassStatsByHost(hostname3, port3, partitionNameAndIds);
    assertEquals(partitionClassStats3.getSnapshot(), obtainedStats3.getSnapshot());
    mySqlStore3.shutdown();
}