Examples with JoinPhaseJob com.linkedin.thirdeye.hadoop.join.JoinPhaseJob used on opensource projects

Example 1 with JoinPhaseJob

use of com.linkedin.thirdeye.hadoop.join.JoinPhaseJob in project pinot by linkedin.

the class ThirdEyeJob method run.

@SuppressWarnings("unchecked")
public void run() throws Exception {
    LOGGER.info("Input config:{}", inputConfig);
    PhaseSpec phaseSpec;
    try {
        phaseSpec = PhaseSpec.valueOf(phaseName.toUpperCase());
    } catch (Exception e) {
        usage();
        throw e;
    }
    if (PhaseSpec.TRANSFORM.equals(phaseSpec)) {
        TransformPhaseJob job = new TransformPhaseJob("Transform Job", inputConfig);
        job.run();
        return;
    } else if (PhaseSpec.JOIN.equals(phaseSpec)) {
        JoinPhaseJob job = new JoinPhaseJob("Join Job", inputConfig);
        job.run();
        return;
    } else if (PhaseSpec.WAIT.equals(phaseSpec)) {
        WaitPhaseJob job = new WaitPhaseJob("Wait for inputs", inputConfig);
        job.run();
        return;
    }
    // Get root, collection, input paths
    String root = getAndCheck(ThirdEyeJobProperties.THIRDEYE_ROOT.getName(), inputConfig);
    String collection = getAndCheck(ThirdEyeJobProperties.THIRDEYE_COLLECTION.getName(), inputConfig);
    String inputPaths = getAndCheck(ThirdEyeJobProperties.INPUT_PATHS.getName(), inputConfig);
    // Get min / max time
    DateTime minTime;
    DateTime maxTime;
    String minTimeProp = inputConfig.getProperty(ThirdEyeJobProperties.THIRDEYE_TIME_MIN.getName());
    String maxTimeProp = inputConfig.getProperty(ThirdEyeJobProperties.THIRDEYE_TIME_MAX.getName());
    minTime = ISODateTimeFormat.dateTimeParser().parseDateTime(minTimeProp);
    maxTime = ISODateTimeFormat.dateTimeParser().parseDateTime(maxTimeProp);
    Properties jobProperties = phaseSpec.getJobProperties(inputConfig, root, collection, minTime, maxTime, inputPaths);
    for (Object key : inputConfig.keySet()) {
        jobProperties.setProperty(key.toString(), inputConfig.getProperty(key.toString()));
    }
    // Instantiate the job
    Constructor<Configured> constructor = (Constructor<Configured>) phaseSpec.getKlazz().getConstructor(String.class, Properties.class);
    Configured instance = constructor.newInstance(phaseSpec.getName(), jobProperties);
    setMapreduceConfig(instance.getConf());
    // Run the job
    Method runMethod = instance.getClass().getMethod("run");
    Job job = (Job) runMethod.invoke(instance);
    if (job != null) {
        JobStatus status = job.getStatus();
        if (status.getState() != JobStatus.State.SUCCEEDED) {
            throw new RuntimeException("Job " + job.getJobName() + " failed to execute: Ran with config:" + jobProperties);
        }
    }
}