Examples with CompactionPathParser org.apache.gobblin.compaction.parser.CompactionPathParser used on opensource projects

Example 6 with CompactionPathParser

use of org.apache.gobblin.compaction.parser.CompactionPathParser in project incubator-gobblin by apache.

the class CompactionAvroJobConfigurator method configureInputAndOutputPaths.

/**
 * Refer to {@link MRCompactorAvroKeyDedupJobRunner#configureInputAndOutputPaths(Job)}
 */
protected void configureInputAndOutputPaths(Job job, FileSystemDataset dataset) throws IOException {
    this.mapReduceInputPaths = getGranularInputPaths(dataset.datasetRoot());
    for (Path path : mapReduceInputPaths) {
        FileInputFormat.addInputPath(job, path);
    }
    String mrOutputBase = this.state.getProp(MRCompactor.COMPACTION_JOB_DIR);
    CompactionPathParser parser = new CompactionPathParser(this.state);
    CompactionPathParser.CompactionParserResult rst = parser.parse(dataset);
    this.mrOutputPath = concatPaths(mrOutputBase, rst.getDatasetName(), rst.getDstSubDir(), rst.getTimeString());
    log.info("Cleaning temporary MR output directory: " + mrOutputPath);
    this.fs.delete(mrOutputPath, true);
    FileOutputFormat.setOutputPath(job, mrOutputPath);
}