Examples with CarbonOutputCommitter org.apache.carbondata.hadoop.api.CarbonOutputCommitter used on opensource projects

Example 1 with CarbonOutputCommitter

use of org.apache.carbondata.hadoop.api.CarbonOutputCommitter in project carbondata by apache.

the class MapredCarbonOutputCommitter method setupJob.

@Override
public void setupJob(JobContext jobContext) throws IOException {
    Random random = new Random();
    JobID jobId = new JobID(UUID.randomUUID().toString(), 0);
    TaskID task = new TaskID(jobId, TaskType.MAP, random.nextInt());
    TaskAttemptID attemptID = new TaskAttemptID(task, random.nextInt());
    org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl context = new TaskAttemptContextImpl(jobContext.getJobConf(), attemptID);
    CarbonLoadModel carbonLoadModel = null;
    String encodedString = jobContext.getJobConf().get(CarbonTableOutputFormat.LOAD_MODEL);
    // org.apache.carbondata.presto.impl.CarbonTableConfig.CARBON_PRESTO_LOAD_MODEL
    if (encodedString != null) {
        carbonLoadModel = (CarbonLoadModel) ObjectSerializationUtil.convertStringToObject(encodedString);
    }
    boolean setLoadModelToEnv = false;
    if (null == carbonLoadModel) {
        ThreadLocalSessionInfo.setConfigurationToCurrentThread(jobContext.getConfiguration());
        carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(jobContext.getConfiguration());
        CarbonTableOutputFormat.setLoadModel(jobContext.getConfiguration(), carbonLoadModel);
        setLoadModelToEnv = true;
    }
    carbonOutputCommitter = new CarbonOutputCommitter(new Path(carbonLoadModel.getTablePath()), context);
    carbonOutputCommitter.setupJob(jobContext);
    if (setLoadModelToEnv) {
        String loadModelStr = jobContext.getConfiguration().get(CarbonTableOutputFormat.LOAD_MODEL);
        String mapReduceMapTaskEnv = jobContext.getJobConf().get(JobConf.MAPRED_MAP_TASK_ENV);
        // Set the loadModel string to mapreduce.map.env so that it will be published to all
        // containers later during job execution.
        jobContext.getJobConf().set(JobConf.MAPRED_MAP_TASK_ENV, mapReduceMapTaskEnv + ",carbon=" + loadModelStr);
        jobContext.getJobConf().set(JobConf.MAPRED_REDUCE_TASK_ENV, mapReduceMapTaskEnv + ",carbon=" + loadModelStr);
    }
}