Examples with ReadOnlyStorageEngine voldemort.store.readonly.ReadOnlyStorageEngine used on opensource projects

Example 21 with ReadOnlyStorageEngine

use of voldemort.store.readonly.ReadOnlyStorageEngine in project voldemort by voldemort.

the class AdminServiceBasicTest method generateAndFetchFiles.

@SuppressWarnings("unchecked")
private void generateAndFetchFiles(int numChunks, long versionId, long indexSize, long dataSize) throws IOException {
    Map<Integer, Set<Pair<Integer, Integer>>> buckets = ROTestUtils.getNodeIdToAllPartitions(cluster, StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, "test-readonly-fetchfiles"), true);
    for (Node node : cluster.getNodes()) {
        ReadOnlyStorageEngine store = (ReadOnlyStorageEngine) getStore(node.getId(), "test-readonly-fetchfiles");
        // Create list of buckets ( replica to partition )
        Set<Pair<Integer, Integer>> nodeBucketsSet = buckets.get(node.getId());
        HashMap<Integer, List<Integer>> nodeBuckets = ROTestUtils.flattenPartitionTuples(nodeBucketsSet);
        // Split the buckets into primary and replica buckets
        HashMap<Integer, List<Integer>> primaryNodeBuckets = Maps.newHashMap();
        primaryNodeBuckets.put(0, nodeBuckets.get(0));
        int primaryPartitions = nodeBuckets.get(0).size();
        HashMap<Integer, List<Integer>> replicaNodeBuckets = Maps.newHashMap(nodeBuckets);
        replicaNodeBuckets.remove(0);
        int replicaPartitions = 0;
        for (List<Integer> partitions : replicaNodeBuckets.values()) {
            replicaPartitions += partitions.size();
        }
        // Generate data...
        File newVersionDir = new File(store.getStoreDirPath(), "version-" + Long.toString(versionId));
        Utils.mkdirs(newVersionDir);
        generateROFiles(numChunks, indexSize, dataSize, nodeBuckets, newVersionDir);
        // Swap it...
        store.swapFiles(newVersionDir.getAbsolutePath());
        // Check if everything got mmap-ed correctly...
        HashMap<Object, Integer> chunkIdToNumChunks = store.getChunkedFileSet().getChunkIdToNumChunks();
        for (Object bucket : chunkIdToNumChunks.keySet()) {
            Pair<Integer, Integer> partitionToReplicaBucket = (Pair<Integer, Integer>) bucket;
            Pair<Integer, Integer> replicaToPartitionBucket = Pair.create(partitionToReplicaBucket.getSecond(), partitionToReplicaBucket.getFirst());
            assertTrue(nodeBucketsSet.contains(replicaToPartitionBucket));
        }
        // Test 0) Try to fetch a partition which doesn't exist
        File tempDir = TestUtils.createTempDir();
        HashMap<Integer, List<Integer>> dumbMap = Maps.newHashMap();
        dumbMap.put(0, Lists.newArrayList(100));
        try {
            getAdminClient().readonlyOps.fetchPartitionFiles(node.getId(), "test-readonly-fetchfiles", Lists.newArrayList(100), tempDir.getAbsolutePath(), null, running);
            fail("Should throw exception since partition map passed is bad");
        } catch (VoldemortException e) {
        }
        // Test 1) Fetch all the primary partitions...
        tempDir = TestUtils.createTempDir();
        getAdminClient().readonlyOps.fetchPartitionFiles(node.getId(), "test-readonly-fetchfiles", primaryNodeBuckets.get(0), tempDir.getAbsolutePath(), null, running);
        // Check it...
        assertEquals(tempDir.list().length, 2 * primaryPartitions * numChunks + 1);
        for (Entry<Integer, List<Integer>> entry : primaryNodeBuckets.entrySet()) {
            int replicaType = entry.getKey();
            for (int partitionId : entry.getValue()) {
                for (int chunkId = 0; chunkId < numChunks; chunkId++) {
                    File indexFile = new File(tempDir, Integer.toString(partitionId) + "_" + Integer.toString(replicaType) + "_" + Integer.toString(chunkId) + ".index");
                    File dataFile = new File(tempDir, Integer.toString(partitionId) + "_" + Integer.toString(replicaType) + "_" + Integer.toString(chunkId) + ".data");
                    assertTrue(indexFile.exists());
                    assertTrue(dataFile.exists());
                    assertEquals(indexFile.length(), indexSize);
                    assertEquals(dataFile.length(), dataSize);
                }
            }
        }
        // Check if metadata file exists
        File metadataFile = new File(tempDir, ".metadata");
        assertEquals(metadataFile.exists(), true);
        // Test 2) Fetch all the replica partitions...
        tempDir = TestUtils.createTempDir();
        for (Entry<Integer, List<Integer>> entry : replicaNodeBuckets.entrySet()) {
            getAdminClient().readonlyOps.fetchPartitionFiles(node.getId(), "test-readonly-fetchfiles", entry.getValue(), tempDir.getAbsolutePath(), null, running);
        }
        // Check it...
        assertEquals(tempDir.list().length, 2 * replicaPartitions * numChunks + 1);
        for (Entry<Integer, List<Integer>> entry : replicaNodeBuckets.entrySet()) {
            int replicaType = entry.getKey();
            for (int partitionId : entry.getValue()) {
                for (int chunkId = 0; chunkId < numChunks; chunkId++) {
                    File indexFile = new File(tempDir, Integer.toString(partitionId) + "_" + Integer.toString(replicaType) + "_" + Integer.toString(chunkId) + ".index");
                    File dataFile = new File(tempDir, Integer.toString(partitionId) + "_" + Integer.toString(replicaType) + "_" + Integer.toString(chunkId) + ".data");
                    assertTrue(indexFile.exists());
                    assertTrue(dataFile.exists());
                    assertEquals(indexFile.length(), indexSize);
                    assertEquals(dataFile.length(), dataSize);
                }
            }
        }
        // Check if metadata file exists
        metadataFile = new File(tempDir, ".metadata");
        assertEquals(metadataFile.exists(), true);
        // Test 3) Fetch all the partitions...
        tempDir = TestUtils.createTempDir();
        for (Entry<Integer, List<Integer>> entry : nodeBuckets.entrySet()) {
            getAdminClient().readonlyOps.fetchPartitionFiles(node.getId(), "test-readonly-fetchfiles", entry.getValue(), tempDir.getAbsolutePath(), null, running);
        }
        // Check it...
        assertEquals(tempDir.list().length, 2 * (primaryPartitions + replicaPartitions) * numChunks + 1);
        for (Entry<Integer, List<Integer>> entry : nodeBuckets.entrySet()) {
            int replicaType = entry.getKey();
            for (int partitionId : entry.getValue()) {
                for (int chunkId = 0; chunkId < numChunks; chunkId++) {
                    File indexFile = new File(tempDir, Integer.toString(partitionId) + "_" + Integer.toString(replicaType) + "_" + Integer.toString(chunkId) + ".index");
                    File dataFile = new File(tempDir, Integer.toString(partitionId) + "_" + Integer.toString(replicaType) + "_" + Integer.toString(chunkId) + ".data");
                    assertTrue(indexFile.exists());
                    assertTrue(dataFile.exists());
                    assertEquals(indexFile.length(), indexSize);
                    assertEquals(dataFile.length(), dataSize);
                }
            }
        }
        // Check if metadata file exists
        metadataFile = new File(tempDir, ".metadata");
        assertEquals(metadataFile.exists(), true);
        // testGetROStorageFileList
        List<String> fileList = getAdminClient().readonlyOps.getROStorageFileList(node.getId(), "test-readonly-fetchfiles");
        int fileCount = 0;
        for (Entry<Integer, List<Integer>> entry : nodeBuckets.entrySet()) {
            int replicaType = entry.getKey();
            for (int partitionId : entry.getValue()) {
                for (int chunkId = 0; chunkId < numChunks; chunkId++) {
                    String fileName = Integer.toString(partitionId) + "_" + Integer.toString(replicaType) + "_" + Integer.toString(chunkId);
                    assertTrue("Assuming file exists:" + fileName, fileList.contains(fileName));
                    fileCount++;
                }
            }
        }
        assertEquals(fileCount, fileList.size());
    }
}