Examples with CheckSum voldemort.store.readonly.checksum.CheckSum used on opensource projects

Example 1 with CheckSum

use of voldemort.store.readonly.checksum.CheckSum in project voldemort by voldemort.

the class HdfsDirectory method validateCheckSum.

public boolean validateCheckSum(Map<HdfsFile, byte[]> fileCheckSumMap) {
    if (checkSumType == CheckSumType.NONE) {
        logger.info("No check-sum verification required");
        return true;
    }
    CheckSum checkSumGenerator = CheckSum.getInstance(checkSumType);
    for (HdfsFile file : allFiles) {
        byte[] fileCheckSum = fileCheckSumMap.get(file);
        if (fileCheckSum != null) {
            checkSumGenerator.update(fileCheckSum);
        }
    }
    byte[] computedCheckSum = checkSumGenerator.getCheckSum();
    boolean checkSumComparison = (ByteUtils.compare(expectedCheckSum, computedCheckSum) == 0);
    logger.info("Checksum generated from streaming - " + new String(Hex.encodeHex(computedCheckSum)));
    logger.info("Checksum on file - " + new String(Hex.encodeHex(expectedCheckSum)));
    logger.info("Check-sum verification - " + checkSumComparison);
    return checkSumComparison;
}

Example 2 with CheckSum

use of voldemort.store.readonly.checksum.CheckSum in project voldemort by voldemort.

the class HadoopStoreBuilder method build.

/**
 * Run the job
 */
public void build() {
    try {
        JobConf conf = prepareJobConf(baseJobConf);
        FileSystem fs = outputDir.getFileSystem(conf);
        if (fs.exists(outputDir)) {
            info("Deleting previous output in " + outputDir + " for building store " + this.storeDef.getName());
            fs.delete(outputDir, true);
        }
        conf.setInt("io.file.buffer.size", DEFAULT_BUFFER_SIZE);
        conf.set("cluster.xml", new ClusterMapper().writeCluster(cluster));
        conf.set("stores.xml", new StoreDefinitionsMapper().writeStoreList(Collections.singletonList(storeDef)));
        conf.setBoolean(VoldemortBuildAndPushJob.SAVE_KEYS, saveKeys);
        conf.setBoolean(VoldemortBuildAndPushJob.REDUCER_PER_BUCKET, reducerPerBucket);
        conf.setBoolean(VoldemortBuildAndPushJob.BUILD_PRIMARY_REPLICAS_ONLY, buildPrimaryReplicasOnly);
        if (!isAvro) {
            conf.setPartitionerClass(HadoopStoreBuilderPartitioner.class);
            conf.setMapperClass(mapperClass);
            conf.setMapOutputKeyClass(BytesWritable.class);
            conf.setMapOutputValueClass(BytesWritable.class);
            conf.setReducerClass(HadoopStoreBuilderReducer.class);
        }
        conf.setInputFormat(inputFormatClass);
        conf.setOutputFormat(SequenceFileOutputFormat.class);
        conf.setOutputKeyClass(BytesWritable.class);
        conf.setOutputValueClass(BytesWritable.class);
        conf.setJarByClass(getClass());
        conf.setReduceSpeculativeExecution(false);
        FileInputFormat.setInputPaths(conf, inputPath);
        conf.set("final.output.dir", outputDir.toString());
        conf.set(VoldemortBuildAndPushJob.CHECKSUM_TYPE, CheckSum.toString(checkSumType));
        conf.set("dfs.umaskmode", "002");
        FileOutputFormat.setOutputPath(conf, tempDir);
        FileSystem outputFs = outputDir.getFileSystem(conf);
        if (outputFs.exists(outputDir)) {
            throw new IOException("Final output directory already exists.");
        }
        // delete output dir if it already exists
        FileSystem tempFs = tempDir.getFileSystem(conf);
        tempFs.delete(tempDir, true);
        long size = sizeOfPath(tempFs, inputPath);
        logger.info("Data size = " + size + ", replication factor = " + storeDef.getReplicationFactor() + ", numNodes = " + cluster.getNumberOfNodes() + ", numPartitions = " + cluster.getNumberOfPartitions() + ", chunk size = " + chunkSizeBytes);
        // Base numbers of chunks and reducers, will get modified according to various settings
        int numChunks = (int) (size / cluster.getNumberOfPartitions() / chunkSizeBytes) + 1;
        /* +1 so we round up */
        int numReducers = cluster.getNumberOfPartitions();
        // question, but in order to avoid breaking anything we'll just maintain the original behavior.
        if (saveKeys) {
            if (buildPrimaryReplicasOnly) {
            // The buildPrimaryReplicasOnly mode is supported exclusively in combination with
            // saveKeys. If enabled, then we don't want to shuffle extra keys redundantly,
            // hence we don't change the number of reducers.
            } else {
                // Old behavior, where all keys are redundantly shuffled to redundant reducers.
                numReducers = numReducers * storeDef.getReplicationFactor();
            }
        } else {
            numChunks = numChunks * storeDef.getReplicationFactor();
        }
        // Ensure at least one chunk
        numChunks = Math.max(numChunks, 1);
        if (reducerPerBucket) {
        // Then all chunks for a given partition/replica combination are shuffled to the same
        // reducer, hence, the number of reducers remains the same as previously defined.
        } else {
            // Otherwise, we want one reducer per chunk, hence we multiply the number of reducers.
            numReducers = numReducers * numChunks;
        }
        conf.setInt(AbstractStoreBuilderConfigurable.NUM_CHUNKS, numChunks);
        conf.setNumReduceTasks(numReducers);
        logger.info("Number of chunks: " + numChunks + ", number of reducers: " + numReducers + ", save keys: " + saveKeys + ", reducerPerBucket: " + reducerPerBucket + ", buildPrimaryReplicasOnly: " + buildPrimaryReplicasOnly);
        if (isAvro) {
            conf.setPartitionerClass(AvroStoreBuilderPartitioner.class);
            // conf.setMapperClass(mapperClass);
            conf.setMapOutputKeyClass(ByteBuffer.class);
            conf.setMapOutputValueClass(ByteBuffer.class);
            conf.setInputFormat(inputFormatClass);
            conf.setOutputFormat((Class<? extends OutputFormat>) AvroOutputFormat.class);
            conf.setOutputKeyClass(ByteBuffer.class);
            conf.setOutputValueClass(ByteBuffer.class);
            // AvroJob confs for the avro mapper
            AvroJob.setInputSchema(conf, Schema.parse(baseJobConf.get(AVRO_REC_SCHEMA)));
            AvroJob.setOutputSchema(conf, Pair.getPairSchema(Schema.create(Schema.Type.BYTES), Schema.create(Schema.Type.BYTES)));
            AvroJob.setMapperClass(conf, mapperClass);
            conf.setReducerClass(AvroStoreBuilderReducer.class);
        }
        logger.info("Building store...");
        // The snipped below copied and adapted from: JobClient.runJob(conf);
        // We have more control in the error handling this way.
        JobClient jc = new JobClient(conf);
        RunningJob runningJob = jc.submitJob(conf);
        Counters counters;
        try {
            if (!jc.monitorAndPrintJob(conf, runningJob)) {
                counters = runningJob.getCounters();
                // For some datasets, the number of chunks that we calculated is inadequate.
                // Here, we try to identify if this is the case.
                long mapOutputBytes = counters.getCounter(Task.Counter.MAP_OUTPUT_BYTES);
                long averageNumberOfBytesPerChunk = mapOutputBytes / numChunks / cluster.getNumberOfPartitions();
                if (averageNumberOfBytesPerChunk > (HadoopStoreWriter.DEFAULT_CHUNK_SIZE)) {
                    float chunkSizeBloat = averageNumberOfBytesPerChunk / (float) HadoopStoreWriter.DEFAULT_CHUNK_SIZE;
                    long suggestedTargetChunkSize = (long) (HadoopStoreWriter.DEFAULT_CHUNK_SIZE / chunkSizeBloat);
                    logger.error("The number of bytes per chunk may be too high." + " averageNumberOfBytesPerChunk = " + averageNumberOfBytesPerChunk + ". Consider setting " + VoldemortBuildAndPushJob.BUILD_CHUNK_SIZE + "=" + suggestedTargetChunkSize);
                } else {
                    logger.error("Job Failed: " + runningJob.getFailureInfo());
                }
                throw new VoldemortException("BnP's MapReduce job failed.");
            }
        } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
        }
        counters = runningJob.getCounters();
        long numberOfRecords = counters.getCounter(Task.Counter.REDUCE_INPUT_GROUPS);
        if (numberOfRecords < minNumberOfRecords) {
            throw new VoldemortException("The number of records in the data set (" + numberOfRecords + ") is lower than the minimum required (" + minNumberOfRecords + "). Aborting.");
        }
        if (saveKeys) {
            logger.info("Number of collisions in the job - " + counters.getCounter(KeyValueWriter.CollisionCounter.NUM_COLLISIONS));
            logger.info("Maximum number of collisions for one entry - " + counters.getCounter(KeyValueWriter.CollisionCounter.MAX_COLLISIONS));
        }
        // Do a CheckSumOfCheckSum - Similar to HDFS
        CheckSum checkSumGenerator = CheckSum.getInstance(this.checkSumType);
        if (!this.checkSumType.equals(CheckSumType.NONE) && checkSumGenerator == null) {
            throw new VoldemortException("Could not generate checksum digest for type " + this.checkSumType);
        }
        List<Integer> directorySuffixes = Lists.newArrayList();
        if (buildPrimaryReplicasOnly) {
            // Files are grouped by partitions
            for (int partitionId = 0; partitionId < cluster.getNumberOfPartitions(); partitionId++) {
                directorySuffixes.add(partitionId);
            }
        } else {
            // Files are grouped by node
            for (Node node : cluster.getNodes()) {
                directorySuffixes.add(node.getId());
            }
        }
        ReadOnlyStorageMetadata fullStoreMetadata = new ReadOnlyStorageMetadata();
        List<Integer> emptyDirectories = Lists.newArrayList();
        final String directoryPrefix = buildPrimaryReplicasOnly ? ReadOnlyUtils.PARTITION_DIRECTORY_PREFIX : ReadOnlyUtils.NODE_DIRECTORY_PREFIX;
        // Generate a log message every 30 seconds or after processing every 100 directories.
        final long LOG_INTERVAL_TIME = TimeUnit.MILLISECONDS.convert(30, TimeUnit.SECONDS);
        final int LOG_INTERVAL_COUNT = buildPrimaryReplicasOnly ? 100 : 5;
        int lastLogCount = 0;
        long lastLogTime = 0;
        long startTimeMS = System.currentTimeMillis();
        // Check if all folder exists and with format file
        for (int index = 0; index < directorySuffixes.size(); index++) {
            int directorySuffix = directorySuffixes.get(index);
            long elapsedTime = System.currentTimeMillis() - lastLogTime;
            long elapsedCount = index - lastLogCount;
            if (elapsedTime >= LOG_INTERVAL_TIME || elapsedCount >= LOG_INTERVAL_COUNT) {
                lastLogTime = System.currentTimeMillis();
                lastLogCount = index;
                logger.info("Processed " + directorySuffix + " out of " + directorySuffixes.size() + " directories.");
            }
            String directoryName = directoryPrefix + directorySuffix;
            ReadOnlyStorageMetadata metadata = new ReadOnlyStorageMetadata();
            if (saveKeys) {
                metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V2.getCode());
            } else {
                metadata.add(ReadOnlyStorageMetadata.FORMAT, ReadOnlyStorageFormat.READONLY_V1.getCode());
            }
            Path directoryPath = new Path(outputDir.toString(), directoryName);
            if (!outputFs.exists(directoryPath)) {
                logger.debug("No data generated for " + directoryName + ". Generating empty folder");
                emptyDirectories.add(directorySuffix);
                // Create empty folder
                outputFs.mkdirs(directoryPath);
                outputFs.setPermission(directoryPath, new FsPermission(HADOOP_FILE_PERMISSION));
                logger.debug("Setting permission to 755 for " + directoryPath);
            }
            processCheckSumMetadataFile(directoryName, outputFs, checkSumGenerator, directoryPath, metadata);
            if (buildPrimaryReplicasOnly) {
            // In buildPrimaryReplicasOnly mode, writing a metadata file for each partitions
            // takes too long, so we skip it. We will rely on the full-store.metadata file instead.
            } else {
                // Maintaining the old behavior: we write the node-specific metadata file
                writeMetadataFile(directoryPath, outputFs, ReadOnlyUtils.METADATA_FILE_EXTENSION, metadata);
            }
            fullStoreMetadata.addNestedMetadata(directoryName, metadata);
        }
        // Write the aggregate metadata file
        writeMetadataFile(outputDir, outputFs, ReadOnlyUtils.FULL_STORE_METADATA_FILE, fullStoreMetadata);
        long elapsedTimeMs = System.currentTimeMillis() - startTimeMS;
        long elapsedTimeSeconds = TimeUnit.SECONDS.convert(elapsedTimeMs, TimeUnit.MILLISECONDS);
        logger.info("Total Processed directories: " + directorySuffixes.size() + ". Elapsed Time (Seconds):" + elapsedTimeSeconds);
        if (emptyDirectories.size() > 0) {
            logger.info("Empty directories: " + Arrays.toString(emptyDirectories.toArray()));
        }
    } catch (Exception e) {
        logger.error("Error in Store builder", e);
        throw new VoldemortException(e);
    }
}

Example 3 with CheckSum

use of voldemort.store.readonly.checksum.CheckSum in project voldemort by voldemort.

the class BasicFetchStrategy method copyFileWithCheckSum.

/**
 * Function to copy a file from the given filesystem with a checksum of type
 * 'checkSumType' computed and returned. In case an error occurs during such
 * a copy, we do a retry for a maximum of NUM_RETRIES
 *
 * @param source
 *            Source path of the file to copy
 * @param dest
 *            Destination path of the file on the local machine
 * @param checkSumType
 *            Type of the Checksum to be computed for this file
 * @return A Checksum (generator) of type checkSumType which contains the
 *         computed checksum of the copied file
 * @throws IOException
 */
private byte[] copyFileWithCheckSum(HdfsFile source, File dest, CheckSumType checkSumType) throws IOException {
    byte[] checkSum = null;
    CheckSum bufferCheckSumGenerator = null;
    logger.debug("Starting copy of " + source + " to " + dest);
    // Check if its Gzip compressed
    boolean isCompressed = source.isCompressed();
    FilterInputStream input = null;
    OutputStream output = null;
    long startTimeMS = System.currentTimeMillis();
    int previousAttempt = 0;
    for (int attempt = 1; attempt <= fetcher.getMaxAttempts(); attempt++) {
        boolean success = false;
        long totalBytesRead = 0;
        boolean fsOpened = false;
        bufferCheckSumGenerator = null;
        stats.singleFileFetchStart(attempt != 1);
        try {
            // Create a per file checksum generator
            if (checkSumType != null) {
                bufferCheckSumGenerator = CheckSum.getInstance(checkSumType);
            }
            logger.info("Starting attempt #" + attempt + "/" + fetcher.getMaxAttempts() + " to fetch remote file: " + source + " to local destination: " + dest);
            input = new ThrottledInputStream(fs.open(source.getPath()), fetcher.getThrottler(), stats);
            if (isCompressed) {
                // We are already bounded by the "hdfs.fetcher.buffer.size"
                // specified in the Voldemort config, the default value of
                // which is 64K. Using the same as the buffer size for
                // GZIPInputStream as well.
                input = new GZIPInputStream(input, this.bufferSize);
            }
            fsOpened = true;
            output = new BufferedOutputStream(new FileOutputStream(dest));
            int read;
            while (true) {
                if (status != null && status.hasException()) {
                    Exception ex = status.getException();
                    if (ex instanceof AsyncOperationStoppedException) {
                        // Then stop() has been called, so let's bubble up the exception
                        throw (AsyncOperationStoppedException) ex;
                    }
                }
                read = input.read(buffer);
                if (read < 0) {
                    break;
                } else {
                    output.write(buffer, 0, read);
                }
                // Update the per file checksum
                if (bufferCheckSumGenerator != null) {
                    bufferCheckSumGenerator.update(buffer, 0, read);
                }
                stats.recordBytesWritten(read);
                totalBytesRead += read;
                boolean reportIntervalPassed = stats.getBytesTransferredSinceLastReport() > fetcher.getReportingIntervalBytes();
                if (attempt != previousAttempt || reportIntervalPassed) {
                    previousAttempt = attempt;
                    NumberFormat format = NumberFormat.getNumberInstance();
                    format.setMaximumFractionDigits(2);
                    String message = stats.getTotalBytesTransferred() / (1024 * 1024) + " MB copied at " + format.format(stats.getBytesTransferredPerSecond() / (1024 * 1024)) + " MB/sec" + ", " + format.format(stats.getPercentCopied()) + " % complete" + ", attempt: #" + attempt + "/" + fetcher.getMaxAttempts() + ", current file: " + dest.getName();
                    if (this.status == null) {
                        // This is to accommodate tests and the old ReadOnlyStoreManagementServlet code path
                        // FIXME: Delete this when we get rid of the old code which does not use status
                        logger.info(message);
                    } else {
                        this.status.setStatus(message);
                        // status.toString() is more detailed than just the message. We print the whole
                        // thing so that server-side logs are very similar to client (BnP) -side logs.
                        logger.info(this.status.toString());
                    }
                    if (reportIntervalPassed) {
                        stats.reset();
                    }
                }
            }
            if (bufferCheckSumGenerator != null) {
                checkSum = bufferCheckSumGenerator.getCheckSum();
            }
            stats.reportFileDownloaded(dest, startTimeMS, source.getSize(), System.currentTimeMillis() - startTimeMS, attempt, totalBytesRead, checkSum);
            logger.info("Completed copy of " + source + " to " + dest);
            success = true;
        } catch (IOException e) {
            if (!fsOpened) {
                logger.error("Error while opening the file stream to " + source, e);
            } else {
                logger.error("Error while copying file " + source + " after " + totalBytesRead + " bytes.", e);
            }
            if (e.getCause() != null) {
                logger.error("Cause of error ", e.getCause());
            }
            if (attempt < fetcher.getMaxAttempts()) {
                logger.info("Will retry copying after " + fetcher.getRetryDelayMs() + " ms");
                sleepForRetryDelayMs();
            } else {
                stats.reportFileError(dest, fetcher.getMaxAttempts(), startTimeMS, e);
                logger.info("Fetcher giving up copy after " + fetcher.getMaxAttempts() + " attempts");
                throw e;
            }
        } finally {
            stats.singleFileFetchEnd();
            IOUtils.closeQuietly(output);
            IOUtils.closeQuietly(input);
            if (success) {
                break;
            }
        }
    }
    // second time checksum validation. Check if the local file is consistent with the buffer
    if (bufferCheckSumGenerator != null) {
        CheckSum fileCheckSumGenerator = CheckSum.getInstance(checkSumType);
        BufferedInputStream in = new BufferedInputStream(new FileInputStream(dest));
        int read;
        try {
            while ((read = in.read(buffer)) >= 0) {
                fileCheckSumGenerator.update(buffer, 0, read);
            }
            if (ByteUtils.compare(fileCheckSumGenerator.getCheckSum(), checkSum) != 0)
                throw new VoldemortException("Local file: " + dest.getAbsolutePath() + " checksum (" + ByteUtils.toHexString(fileCheckSumGenerator.getCheckSum()) + ") does not match with the checksum in the buffer (" + ByteUtils.toHexString(fileCheckSumGenerator.getCheckSum()) + ")");
        } finally {
            IOUtils.closeQuietly(in);
        }
    }
    return checkSum;
}

Example 4 with CheckSum

use of voldemort.store.readonly.checksum.CheckSum in project voldemort by voldemort.

the class HdfsFetcherAdvancedTest method calculateCheckSumForFile.

/*
     * Helper method to calculate checksum for a single file
     */
private byte[] calculateCheckSumForFile(Path source) throws Exception {
    CheckSum fileCheckSumGenerator = CheckSum.getInstance(CheckSumType.MD5);
    byte[] buffer = new byte[VoldemortConfig.DEFAULT_FETCHER_BUFFER_SIZE];
    FSDataInputStream input = null;
    Configuration config = new Configuration();
    FileSystem fs = source.getFileSystem(config);
    input = fs.open(source);
    while (true) {
        int read = input.read(buffer);
        if (read < 0) {
            break;
        }
        // Update the per file checksum
        if (fileCheckSumGenerator != null) {
            fileCheckSumGenerator.update(buffer, 0, read);
        }
    }
    return fileCheckSumGenerator.getCheckSum();
}