Examples with HadoopStoreBuilder voldemort.store.readonly.mr.HadoopStoreBuilder used on opensource projects

Example 1 with HadoopStoreBuilder

use of voldemort.store.readonly.mr.HadoopStoreBuilder in project voldemort by voldemort.

the class VoldemortBuildAndPushJob method runBuildStore.

public String runBuildStore(Props props, String url) throws Exception {
    Path tempDir = new Path(props.getString(BUILD_TEMP_DIR, "/tmp/vold-build-and-push-" + new Random().nextLong()));
    Path outputDir = new Path(props.getString(BUILD_OUTPUT_DIR), new URI(url).getHost());
    CheckSumType checkSumType = CheckSum.fromString(props.getString(CHECKSUM_TYPE, CheckSum.toString(CheckSumType.MD5)));
    JobConf configuration = new JobConf();
    Class mapperClass;
    Class<? extends InputFormat> inputFormatClass;
    // for the key value schema of the avro record
    if (this.isAvroJob) {
        configuration.set(HadoopStoreBuilder.AVRO_REC_SCHEMA, getRecordSchema());
        configuration.set(AvroStoreBuilderMapper.AVRO_KEY_SCHEMA, getKeySchema());
        configuration.set(AvroStoreBuilderMapper.AVRO_VALUE_SCHEMA, getValueSchema());
        configuration.set(VoldemortBuildAndPushJob.AVRO_KEY_FIELD, this.keyFieldName);
        configuration.set(VoldemortBuildAndPushJob.AVRO_VALUE_FIELD, this.valueFieldName);
        mapperClass = AvroStoreBuilderMapper.class;
        inputFormatClass = AvroInputFormat.class;
    } else {
        mapperClass = JsonStoreBuilderMapper.class;
        inputFormatClass = JsonSequenceFileInputFormat.class;
    }
    if (props.containsKey(AbstractStoreBuilderConfigurable.NUM_CHUNKS)) {
        log.warn("N.B.: The '" + AbstractStoreBuilderConfigurable.NUM_CHUNKS + "' config parameter is now " + "deprecated and ignored. The BnP job will automatically determine a proper value for this setting.");
    }
    HadoopStoreBuilder builder = new HadoopStoreBuilder(getId() + "-build-store", props, configuration, mapperClass, inputFormatClass, this.adminClientPerCluster.get(url).getAdminClientCluster(), this.storeDef, tempDir, outputDir, getInputPath(), checkSumType, props.getBoolean(SAVE_KEYS, true), props.getBoolean(REDUCER_PER_BUCKET, true), props.getInt(BUILD_CHUNK_SIZE, (int) HadoopStoreWriter.DEFAULT_CHUNK_SIZE), this.isAvroJob, this.minNumberOfRecords, this.buildPrimaryReplicasOnly);
    builder.build();
    return outputDir.toString();
}

Example 2 with HadoopStoreBuilder

use of voldemort.store.readonly.mr.HadoopStoreBuilder in project voldemort by voldemort.

the class BuildTestStore method run.

public int run(String[] args) throws Exception {
    if (args.length != 5)
        Utils.croak("Expected arguments store_name config_dir temp_dir input_path output_path");
    String storeName = args[0];
    String configDir = args[1];
    String tempDir = args[2];
    String inputDir = args[3];
    String outputDir = args[4];
    List<StoreDefinition> storeDefs = new StoreDefinitionsMapper().readStoreList(new File(configDir, "stores.xml"));
    StoreDefinition def = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        def = d;
    Cluster cluster = new ClusterMapper().readCluster(new File(configDir, "cluster.xml"));
    JobConf config = new JobConf();
    String jobName = "test-store-builder";
    config.set("mapred.job.name", jobName);
    HadoopStoreBuilder builder = new HadoopStoreBuilder(jobName, new Props(), config, BuildTestStoreMapper.class, SequenceFileInputFormat.class, cluster, def, new Path(tempDir), new Path(outputDir), new Path(inputDir), CheckSum.CheckSumType.NONE, false, false, (long) (1.5 * 1024 * 1024 * 1024), false, null, false);
    builder.build();
    return 0;
}