Examples with ReadOnlyStorageMetadata voldemort.store.readonly.ReadOnlyStorageMetadata used on opensource projects

Example 1 with ReadOnlyStorageMetadata

use of voldemort.store.readonly.ReadOnlyStorageMetadata in project voldemort by voldemort.

the class HdfsFetcher method fetchFromSource.

private File fetchFromSource(String sourceFileUrl, String destinationFile, AsyncOperationStatus status, String storeName, long pushVersion, Long diskQuotaSizeInKB, MetadataStore metadataStore) throws Exception {
    ObjectName jmxName = null;
    HdfsCopyStats stats = null;
    FileSystem fs = null;
    sourceFileUrl = VoldemortUtils.modifyURL(sourceFileUrl, voldemortConfig);
    // Flag to indicate whether the fetch is complete or not
    boolean isCompleteFetch = false;
    try {
        // Record as one store fetch
        HdfsCopyStats.storeFetch();
        fs = HadoopUtils.getHadoopFileSystem(voldemortConfig, sourceFileUrl);
        final Path rootPath = new Path(sourceFileUrl);
        File destination = new File(destinationFile);
        if (destination.exists()) {
            throw new VoldemortException("Version directory " + destination.getAbsolutePath() + " already exists");
        }
        boolean isFile = isFile(fs, rootPath);
        stats = new HdfsCopyStats(sourceFileUrl, destination, // stats file initially disabled, to fetch just the first metadata file
        false, maxVersionsStatsFile, isFile, null);
        jmxName = JmxUtils.registerMbean("hdfs-copy-" + copyCount.getAndIncrement(), stats);
        logger.info("Starting fetch for : " + sourceFileUrl);
        FetchStrategy fetchStrategy = new BasicFetchStrategy(this, fs, stats, status, bufferSize);
        if (!isFile) {
            // We are asked to fetch a directory
            Utils.mkdirs(destination);
            HdfsDirectory rootDirectory = new HdfsDirectory(fs, rootPath, this.voldemortConfig);
            List<HdfsDirectory> directoriesToFetch = Lists.newArrayList();
            HdfsFile metadataFile = rootDirectory.getMetadataFile();
            Long expectedDiskSize = -1L;
            if (metadataFile != null) {
                File copyLocation = new File(destination, metadataFile.getPath().getName());
                fetchStrategy.fetch(metadataFile, copyLocation, null);
                rootDirectory.initializeMetadata(copyLocation);
                if (metadataFile.getDiskFileName().equals(ReadOnlyUtils.FULL_STORE_METADATA_FILE)) {
                    // Then we are in build.primary.replica.only mode, and we need to determine which
                    // partition sub-directories to download
                    Set<Integer> partitions = getPartitionsForCurrentNode(metadataStore, storeName);
                    for (int partitionId : partitions) {
                        String partitionKey = ReadOnlyUtils.PARTITION_DIRECTORY_PREFIX + partitionId;
                        ReadOnlyStorageMetadata partitionMetadata = rootDirectory.getMetadata().getNestedMetadata(partitionKey);
                        String diskSizeInBytes = (String) partitionMetadata.get(ReadOnlyStorageMetadata.DISK_SIZE_IN_BYTES);
                        if (diskSizeInBytes != null && diskSizeInBytes != "") {
                            logger.debug("Partition " + partitionId + " is served by this node and is not empty, so it will be downloaded.");
                            if (expectedDiskSize == -1) {
                                expectedDiskSize = Long.parseLong(diskSizeInBytes);
                            } else {
                                expectedDiskSize += Long.parseLong(diskSizeInBytes);
                            }
                            HdfsDirectory partitionDirectory = new HdfsDirectory(fs, new Path(rootPath, partitionKey), this.voldemortConfig);
                            partitionDirectory.initializeMetadata(partitionMetadata);
                            directoriesToFetch.add(partitionDirectory);
                        } else {
                            logger.debug("Partition " + partitionId + " is served by this node but it is empty, so it will be skipped.");
                        }
                    }
                } else {
                    // Then we are not in build.primary.replica.only mode (old behavior), and we
                    // need to download the entire node directory we're currently in.
                    String diskSizeInBytes = (String) rootDirectory.getMetadata().get(ReadOnlyStorageMetadata.DISK_SIZE_IN_BYTES);
                    if (diskSizeInBytes != null && diskSizeInBytes != "") {
                        expectedDiskSize = Long.parseLong(diskSizeInBytes);
                    }
                    directoriesToFetch.add(rootDirectory);
                }
            }
            checkIfQuotaExceeded(diskQuotaSizeInKB, storeName, destination, expectedDiskSize);
            stats = new HdfsCopyStats(sourceFileUrl, destination, enableStatsFile, maxVersionsStatsFile, isFile, new HdfsPathInfo(directoriesToFetch));
            fetchStrategy = new BasicFetchStrategy(this, fs, stats, status, bufferSize);
            logger.debug("directoriesToFetch for store '" + storeName + "': " + Arrays.toString(directoriesToFetch.toArray()));
            for (HdfsDirectory directoryToFetch : directoriesToFetch) {
                Map<HdfsFile, byte[]> fileCheckSumMap = fetchStrategy.fetch(directoryToFetch, destination);
                if (directoryToFetch.validateCheckSum(fileCheckSumMap)) {
                    logger.info("Completed fetch: " + sourceFileUrl);
                } else {
                    stats.checkSumFailed();
                    logger.error("Checksum did not match for " + directoryToFetch.toString() + " !");
                    return null;
                }
            }
            isCompleteFetch = true;
            return destination;
        } else if (allowFetchingOfSingleFile) {
            /**
             * This code path is only used by {@link #main(String[])}
             */
            Utils.mkdirs(destination);
            HdfsFile file = new HdfsFile(fs.getFileStatus(rootPath));
            String fileName = file.getDiskFileName();
            File copyLocation = new File(destination, fileName);
            fetchStrategy.fetch(file, copyLocation, CheckSumType.NONE);
            logger.info("Completed fetch : " + sourceFileUrl);
            isCompleteFetch = true;
            return destination;
        } else {
            logger.error("Source " + rootPath.toString() + " should be a directory");
            return null;
        }
    } catch (Exception e) {
        if (stats != null) {
            stats.reportError("File fetcher failed for destination " + destinationFile, e);
        }
        // Since AuthenticationException may happen before stats object initialization (HadoopUtils.getHadoopFileSystem),
        // we use the static method to capture all the exceptions here.
        HdfsCopyStats.reportExceptionForStats(e);
        if (e instanceof VoldemortException) {
            throw e;
        } else {
            throw new VoldemortException("Error thrown while trying to get data from Hadoop filesystem: " + e.getMessage(), e);
        }
    } finally {
        if (jmxName != null)
            JmxUtils.unregisterMbean(jmxName);
        if (stats != null) {
            stats.complete();
        }
        if (!isCompleteFetch) {
            HdfsCopyStats.incompleteFetch();
        }
        if (fs != null) {
            try {
                fs.close();
            } catch (Exception e) {
                String errorMessage = "Caught " + e.getClass().getSimpleName() + " while trying to close the filesystem instance (harmless).";
                if (stats != null) {
                    stats.reportError(errorMessage, e);
                }
                logger.debug(errorMessage, e);
            }
        }
    }
}

Example 2 with ReadOnlyStorageMetadata

use of voldemort.store.readonly.ReadOnlyStorageMetadata in project voldemort by voldemort.

the class HdfsFetcherAdvancedTest method testCheckSumMetadata.

/*
     * Tests that HdfsFetcher can correctly fetch a file in happy path
     *
     * Checks for both checksum and correctness in case of decompression.
     */
@Test
public void testCheckSumMetadata() throws Exception {
    cleanUp();
    testSourceDir = createTempDir();
    File testUncompressedSourceDir = null;
    // generate index , data and , metadata files
    File indexFile = new File(testSourceDir, "0_0.index");
    FileUtils.writeByteArrayToFile(indexFile, TestUtils.randomBytes(100));
    File dataFile = new File(testSourceDir, "0_0.data");
    FileUtils.writeByteArrayToFile(dataFile, TestUtils.randomBytes(400));
    HdfsFetcher fetcher = new HdfsFetcher();
    File metadataFile = new File(testSourceDir, ".metadata");
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
    metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.MD5));
    // Correct metadata checksum - MD5
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDir.listFiles(), CheckSumType.MD5))));
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    /*
         * if isCompressed == true replace .index and .data files with their
         * compressed files before invoking fetch. Move the original
         * uncompressed .index and .data files to a temporary location so they
         * can be used later to check for data equality.
         */
    if (isCompressed) {
        String destIndexPath = indexFile.getAbsolutePath() + ".gz";
        gzipFile(indexFile.getAbsolutePath(), destIndexPath);
        String destDataPath = dataFile.getAbsolutePath() + ".gz";
        gzipFile(dataFile.getAbsolutePath(), destDataPath);
        testUncompressedSourceDir = new File(testSourceDir.getAbsolutePath() + "-uncompressed");
        testUncompressedSourceDir.delete();
        testUncompressedSourceDir.mkdir();
        if (!indexFile.renameTo(new File(testUncompressedSourceDir, indexFile.getName())) || !dataFile.renameTo(new File(testUncompressedSourceDir, dataFile.getName()))) {
            throw new Exception("cannot move irrelevant files");
        }
    }
    testDestDir = new File(testSourceDir.getAbsolutePath() + "1");
    if (testDestDir.exists()) {
        deleteDir(testDestDir);
    }
    File fetchedFile = fetcher.fetch(testSourceDir.getAbsolutePath(), testDestDir.getAbsolutePath());
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestDir.getAbsolutePath());
    if (isCompressed) {
        for (File file : testUncompressedSourceDir.listFiles()) {
            if (file.isFile()) {
                Assert.assertTrue(ReadOnlyTestUtils.areTwoBinaryFilesEqual(file, new File(testDestDir, file.getName())));
            }
        }
    }
    if (testDestDir.exists()) {
        deleteDir(testDestDir);
    }
}

Example 3 with ReadOnlyStorageMetadata

use of voldemort.store.readonly.ReadOnlyStorageMetadata in project voldemort by voldemort.

the class HdfsFetcherTest method testAggStatsWithUnauthorizedStoreException.

public void testAggStatsWithUnauthorizedStoreException() throws Exception {
    HdfsFetcherAggStats stats = HdfsFetcherAggStats.getStats();
    long totalBytesFetchedBefore = stats.getTotalBytesFetched();
    long totalUnauthorizedStoreFailuresBefore = stats.getTotalUnauthorizedStoreFailures();
    long totalFetchesBefore = stats.getTotalFetches();
    long totalIncompleteFetchesBefore = stats.getTotalIncompleteFetches();
    // Generate 0_0.[index | data] and their corresponding metadata
    File testSourceDirectory = TestUtils.createTempDir();
    File testDestinationDirectory = TestUtils.createTempDir();
    // Missing metadata file
    File indexFile = new File(testSourceDirectory, "0_0.index");
    FileUtils.writeByteArrayToFile(indexFile, TestUtils.randomBytes(100));
    File dataFile = new File(testSourceDirectory, "0_0.data");
    FileUtils.writeByteArrayToFile(dataFile, TestUtils.randomBytes(400));
    File metadataFile = new File(testSourceDirectory, ".metadata");
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
    metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(), CheckSumType.MD5))));
    metadata.add(ReadOnlyStorageMetadata.DISK_SIZE_IN_BYTES, "5000");
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    HdfsFetcher fetcher = new HdfsFetcher();
    File fetchedFile = null;
    try {
        fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "1", 0);
    } catch (Exception e) {
    }
    assertNull(fetchedFile);
    // The total bytes fetched includes meta data file as well.
    assertEquals(totalBytesFetchedBefore + metadata.toJsonString().length(), stats.getTotalBytesFetched());
    assertEquals(totalUnauthorizedStoreFailuresBefore + 1, stats.getTotalUnauthorizedStoreFailures());
    assertEquals(totalFetchesBefore + 1, stats.getTotalFetches());
    assertEquals(totalIncompleteFetchesBefore + 1, stats.getTotalIncompleteFetches());
}

Example 4 with ReadOnlyStorageMetadata

use of voldemort.store.readonly.ReadOnlyStorageMetadata in project voldemort by voldemort.

the class HdfsFetcherTest method testCheckSumMetadata.

public void testCheckSumMetadata() throws Exception {
    // Generate 0_0.[index | data] and their corresponding metadata
    File testSourceDirectory = TestUtils.createTempDir();
    File testDestinationDirectory = TestUtils.createTempDir();
    // Missing metadata file
    File indexFile = new File(testSourceDirectory, "0_0.index");
    FileUtils.writeByteArrayToFile(indexFile, TestUtils.randomBytes(100));
    File dataFile = new File(testSourceDirectory, "0_0.data");
    FileUtils.writeByteArrayToFile(dataFile, TestUtils.randomBytes(400));
    HdfsFetcher fetcher = new HdfsFetcher();
    File fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "1");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "1");
    // Write bad metadata file
    File metadataFile = new File(testSourceDirectory, ".metadata");
    FileUtils.writeByteArrayToFile(metadataFile, TestUtils.randomBytes(100));
    try {
        fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "2");
        fail("Should have thrown an exception since metadata file is corrupt");
    } catch (VoldemortException e) {
    }
    metadataFile.delete();
    // Missing metadata checksum type
    metadataFile = new File(testSourceDirectory, ".metadata");
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
    metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "3");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "3");
    metadataFile.delete();
    // Incorrect checksum type + missing checksum
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, "blah");
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "4");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "4");
    metadataFile.delete();
    // Incorrect metadata checksum
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.MD5));
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, "1234");
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "5");
    assertNull(fetchedFile);
    metadataFile.delete();
    // Correct metadata checksum - MD5
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(), CheckSumType.MD5))));
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "6");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "6");
    // Correct metadata checksum - ADLER32
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.ADLER32));
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(), CheckSumType.ADLER32))));
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "7");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "7");
    // Correct metadata checksum - CRC32
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.CRC32));
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, new String(Hex.encodeHex(CheckSumTests.calculateCheckSum(testSourceDirectory.listFiles(), CheckSumType.CRC32))));
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "8");
    assertNotNull(fetchedFile);
    assertEquals(fetchedFile.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "8");
}

Example 5 with ReadOnlyStorageMetadata

use of voldemort.store.readonly.ReadOnlyStorageMetadata in project voldemort by voldemort.

the class HdfsFetcherTest method testAggStatsWithCheckSumFailure.

public void testAggStatsWithCheckSumFailure() throws Exception {
    HdfsFetcherAggStats stats = HdfsFetcherAggStats.getStats();
    long totalBytesFetchedBefore = stats.getTotalBytesFetched();
    long totalCheckSumFailuresBefore = stats.getTotalCheckSumFailures();
    long totalFetchesBefore = stats.getTotalFetches();
    long totalIncompleteFetchesBefore = stats.getTotalIncompleteFetches();
    // Generate 0_0.[index | data] and their corresponding metadata
    File testSourceDirectory = TestUtils.createTempDir();
    File testDestinationDirectory = TestUtils.createTempDir();
    // Missing metadata file
    File indexFile = new File(testSourceDirectory, "0_0.index");
    FileUtils.writeByteArrayToFile(indexFile, TestUtils.randomBytes(100));
    File dataFile = new File(testSourceDirectory, "0_0.data");
    FileUtils.writeByteArrayToFile(dataFile, TestUtils.randomBytes(400));
    File metadataFile = new File(testSourceDirectory, ".metadata");
    ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
    metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM_TYPE, CheckSum.toString(CheckSumType.MD5));
    metadata.add(ReadOnlyStorageMetadata.CHECKSUM, "1234");
    FileUtils.writeStringToFile(metadataFile, metadata.toJsonString());
    HdfsFetcher fetcher = new HdfsFetcher();
    File fetchedFile = null;
    try {
        fetchedFile = fetcher.fetch(testSourceDirectory.getAbsolutePath(), testDestinationDirectory.getAbsolutePath() + "1");
    } catch (Exception e) {
    }
    assertNull(fetchedFile);
    // The total bytes fetched includes meta data file as well.
    assertEquals(totalBytesFetchedBefore + 500 + metadata.toJsonString().length(), stats.getTotalBytesFetched());
    assertEquals(totalCheckSumFailuresBefore + 1, stats.getTotalCheckSumFailures());
    assertEquals(totalFetchesBefore + 1, stats.getTotalFetches());
    assertEquals(totalIncompleteFetchesBefore + 1, stats.getTotalIncompleteFetches());
}