Examples with CheckSumType voldemort.store.readonly.checksum.CheckSum.CheckSumType used on opensource projects

Example 1 with CheckSumType

use of voldemort.store.readonly.checksum.CheckSum.CheckSumType in project voldemort by voldemort.

the class BasicFetchStrategy method fetch.

@Override
public Map<HdfsFile, byte[]> fetch(HdfsDirectory directory, File dest) throws IOException {
    Map<HdfsFile, byte[]> fileCheckSumMap = new HashMap<HdfsFile, byte[]>(directory.getFiles().size());
    CheckSumType checkSumType = directory.getCheckSumType();
    for (HdfsFile file : directory.getFiles()) {
        String fileName = file.getDiskFileName();
        File copyLocation = new File(dest, fileName);
        byte[] fileCheckSum = copyFileWithCheckSum(file, copyLocation, checkSumType);
        if (fileCheckSum != null) {
            fileCheckSumMap.put(file, fileCheckSum);
        }
    }
    return fileCheckSumMap;
}

Example 2 with CheckSumType

use of voldemort.store.readonly.checksum.CheckSum.CheckSumType in project voldemort by voldemort.

the class VoldemortBuildAndPushJob method runBuildStore.

public String runBuildStore(Props props, String url) throws Exception {
    Path tempDir = new Path(props.getString(BUILD_TEMP_DIR, "/tmp/vold-build-and-push-" + new Random().nextLong()));
    Path outputDir = new Path(props.getString(BUILD_OUTPUT_DIR), new URI(url).getHost());
    CheckSumType checkSumType = CheckSum.fromString(props.getString(CHECKSUM_TYPE, CheckSum.toString(CheckSumType.MD5)));
    JobConf configuration = new JobConf();
    Class mapperClass;
    Class<? extends InputFormat> inputFormatClass;
    // for the key value schema of the avro record
    if (this.isAvroJob) {
        configuration.set(HadoopStoreBuilder.AVRO_REC_SCHEMA, getRecordSchema());
        configuration.set(AvroStoreBuilderMapper.AVRO_KEY_SCHEMA, getKeySchema());
        configuration.set(AvroStoreBuilderMapper.AVRO_VALUE_SCHEMA, getValueSchema());
        configuration.set(VoldemortBuildAndPushJob.AVRO_KEY_FIELD, this.keyFieldName);
        configuration.set(VoldemortBuildAndPushJob.AVRO_VALUE_FIELD, this.valueFieldName);
        mapperClass = AvroStoreBuilderMapper.class;
        inputFormatClass = AvroInputFormat.class;
    } else {
        mapperClass = JsonStoreBuilderMapper.class;
        inputFormatClass = JsonSequenceFileInputFormat.class;
    }
    if (props.containsKey(AbstractStoreBuilderConfigurable.NUM_CHUNKS)) {
        log.warn("N.B.: The '" + AbstractStoreBuilderConfigurable.NUM_CHUNKS + "' config parameter is now " + "deprecated and ignored. The BnP job will automatically determine a proper value for this setting.");
    }
    HadoopStoreBuilder builder = new HadoopStoreBuilder(getId() + "-build-store", props, configuration, mapperClass, inputFormatClass, this.adminClientPerCluster.get(url).getAdminClientCluster(), this.storeDef, tempDir, outputDir, getInputPath(), checkSumType, props.getBoolean(SAVE_KEYS, true), props.getBoolean(REDUCER_PER_BUCKET, true), props.getInt(BUILD_CHUNK_SIZE, (int) HadoopStoreWriter.DEFAULT_CHUNK_SIZE), this.isAvroJob, this.minNumberOfRecords, this.buildPrimaryReplicasOnly);
    builder.build();
    return outputDir.toString();
}