Examples with Job com.twitter.ambrose.model.Job used on opensource projects

Example 6 with Job

use of com.twitter.ambrose.model.Job in project ambrose by twitter.

the class AmbrosePigProgressNotificationListener method initialPlanNotification.

/**
   * Called after the job DAG has been created, but before any jobs are fired.
   * @param plan the MROperPlan that represents the DAG of operations. Each operation will become
   * a MapReduce job when it's launched.
   */
@Override
public void initialPlanNotification(String scriptId, OperatorPlan<?> plan) {
    log.info("initialPlanNotification - scriptId " + scriptId + " plan " + plan);
    // For ambrose to work above 3 must be non-null
    Preconditions.checkNotNull(pigConfig.getJobClient());
    Preconditions.checkNotNull(pigConfig.getJobGraph());
    Preconditions.checkNotNull(pigConfig.getPigProperties());
    try {
        statsWriteService.initWriteService(pigConfig.getPigProperties());
    } catch (IOException ioe) {
        throw new RuntimeException("Exception while initializing statsWriteService", ioe);
    }
    this.workflowVersion = pigConfig.getPigProperties().getProperty("pig.logical.plan.signature");
    OperatorPlan<MapReduceOper> mrPlan;
    try {
        mrPlan = (OperatorPlan<MapReduceOper>) plan;
    } catch (Exception e) {
        log.error(String.format("Failed to cast OperatorPlan: %s", plan), e);
        return;
    }
    Map<OperatorKey, MapReduceOper> planKeys = mrPlan.getKeys();
    Configuration flowConfig = new Configuration(false);
    boolean initialized = false;
    // first pass builds all nodes
    for (Map.Entry<OperatorKey, MapReduceOper> entry : planKeys.entrySet()) {
        String nodeName = entry.getKey().toString();
        MapReduceOper op = entry.getValue();
        MRScriptState scriptState = MRScriptState.get();
        String[] aliases = toArray(scriptState.getAlias(op).trim());
        String[] features = toArray(scriptState.getPigFeature(op).trim());
        if (!initialized) {
            scriptState.addSettingsToConf(op, flowConfig);
            pigConfig.getPigProperties().putAll(ConfigurationUtil.toProperties(flowConfig));
            initialized = true;
        }
        PigJob job = new PigJob();
        job.setAliases(aliases);
        job.setFeatures(features);
        job.setConfiguration(pigConfig.getPigProperties());
        DAGNode<PigJob> node = new DAGNode<PigJob>(nodeName, job);
        this.dagNodeNameMap.put(node.getName(), node);
        // this shows how we can get the basic info about all nameless jobs before any execute.
        // we can traverse the plan to build a DAG of this info
        log.info("initialPlanNotification: aliases: " + Arrays.toString(aliases) + ", name: " + node.getName() + ", features: " + Arrays.toString(features));
    }
    // second pass connects the edges
    for (Map.Entry<OperatorKey, MapReduceOper> entry : planKeys.entrySet()) {
        DAGNode node = this.dagNodeNameMap.get(entry.getKey().toString());
        List<DAGNode<? extends Job>> successorNodeList = Lists.newArrayList();
        List<MapReduceOper> successors = mrPlan.getSuccessors(entry.getValue());
        if (successors != null) {
            for (MapReduceOper successor : successors) {
                DAGNode<? extends Job> successorNode = this.dagNodeNameMap.get(successor.getOperatorKey().toString());
                successorNodeList.add(successorNode);
            }
        }
        node.setSuccessors(successorNodeList);
    }
    AmbroseUtils.sendDagNodeNameMap(statsWriteService, scriptId, dagNodeNameMap);
}

Example 7 with Job

use of com.twitter.ambrose.model.Job in project ambrose by twitter.

the class AmbroseHiveFailHook method run.

@Override
public void run(HookContext hookContext) throws Exception {
    HiveConf conf = hookContext.getConf();
    Properties allConfProps = conf.getAllProperties();
    String queryId = AmbroseHiveUtil.getHiveQueryId(conf);
    EmbeddedAmbroseHiveProgressReporter reporter = getEmbeddedProgressReporter();
    List<TaskRunner> completeTaskList = hookContext.getCompleteTaskList();
    Field _taskResultField = accessTaskResultField();
    for (TaskRunner taskRunner : completeTaskList) {
        TaskResult taskResult = (TaskResult) _taskResultField.get(taskRunner);
        // get non-running, failed jobs
        if (!taskResult.isRunning() && taskResult.getExitVal() != 0) {
            Task<? extends Serializable> task = taskRunner.getTask();
            String nodeId = AmbroseHiveUtil.getNodeIdFromNodeName(conf, task.getId());
            DAGNode<Job> dagNode = reporter.getDAGNodeFromNodeId(nodeId);
            HiveJob job = (HiveJob) dagNode.getJob();
            job.setConfiguration(allConfProps);
            MapReduceJobState mrJobState = getJobState(job);
            mrJobState.setSuccessful(false);
            reporter.addJob((Job) job);
            reporter.pushEvent(queryId, new Event.JobFailedEvent(dagNode));
        }
    }
    reporter.restoreEventStack();
    String sleepTime = System.getProperty(POST_SCRIPT_SLEEP_SECS_PARAM, "10");
    try {
        int sleepTimeSeconds = Integer.parseInt(sleepTime);
        LOG.info("Script failed but sleeping for " + sleepTimeSeconds + " seconds to keep the HiveStats REST server running. Hit ctrl-c to exit.");
        Thread.sleep(sleepTimeSeconds * 1000L);
        reporter.stopServer();
    } catch (NumberFormatException e) {
        LOG.warn(POST_SCRIPT_SLEEP_SECS_PARAM + " param is not a valid number, not sleeping: " + sleepTime);
    } catch (InterruptedException e) {
        LOG.warn("Sleep interrupted", e);
    }
}

Example 8 with Job

use of com.twitter.ambrose.model.Job in project ambrose by twitter.

the class AmbroseCascadingGraphConverter method getNodeSuccessors.

/**
   * Return a Collection of successor nodes of a certain vertex.
   *
   * @param vertex the step or node its successors nodes will be returned.
   * @return collection of successor DAGNodes for each node.
   */
protected Collection<DAGNode<? extends Job>> getNodeSuccessors(Object vertex) {
    Collection<DAGNode<? extends Job>> nodeSuccessors = Sets.newHashSet();
    List successorNodes = Graphs.successorListOf(jobsGraph, vertex);
    for (Object node : successorNodes) {
        BaseFlowStep step = (BaseFlowStep) node;
        String name = step.getName();
        nodeSuccessors.add(dagNamesMap.get(name));
    }
    return nodeSuccessors;
}

Example 9 with Job

use of com.twitter.ambrose.model.Job in project ambrose by twitter.

the class CascadingJobTest method doTestRoundTrip.

private void doTestRoundTrip(CascadingJob expected) throws IOException {
    String asJson = expected.toJson();
    Job asJobAgain = Job.fromJson(asJson);
    // assert that if we get a PigJob without having to ask for it explicitly
    assertTrue(asJobAgain instanceof CascadingJob);
    assertJobEquals(expected, (CascadingJob) asJobAgain);
}

Example 10 with Job

use of com.twitter.ambrose.model.Job in project ambrose by twitter.

the class AmbroseCascadingGraphConverter method getNodeSuccessors.

/**
   * Return a Collection of successor nodes of a certain vertex.
   *
   * @param vertex the step or node its successors nodes will be returned.
   * @return collection of successor DAGNodes for each node.
   */
protected Collection<DAGNode<? extends Job>> getNodeSuccessors(Object vertex) {
    Collection<DAGNode<? extends Job>> nodeSuccessors = Sets.newHashSet();
    List successorNodes = Graphs.successorListOf(jobsGraph, vertex);
    for (Object node : successorNodes) {
        BaseFlowStep step = (BaseFlowStep) node;
        String name = step.getName();
        nodeSuccessors.add(dagNamesMap.get(name));
    }
    return nodeSuccessors;
}