Examples with SystemClock org.apache.hadoop.yarn.util.SystemClock used on opensource projects

Example 1 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestJobImpl method createCommitterEventHandler.

private static CommitterEventHandler createCommitterEventHandler(Dispatcher dispatcher, OutputCommitter committer) {
    final SystemClock clock = SystemClock.getInstance();
    AppContext appContext = mock(AppContext.class);
    when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
    when(appContext.getClock()).thenReturn(clock);
    RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() {

        @Override
        public long getLastHeartbeatTime() {
            return clock.getTime();
        }

        @Override
        public void runOnNextHeartbeat(Runnable callback) {
            callback.run();
        }
    };
    ApplicationAttemptId id = ApplicationAttemptId.fromString("appattempt_1234567890000_0001_0");
    when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
    when(appContext.getApplicationAttemptId()).thenReturn(id);
    CommitterEventHandler handler = new CommitterEventHandler(appContext, committer, heartbeatHandler);
    dispatcher.register(CommitterEventType.class, handler);
    return handler;
}

Example 2 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestTaskAttemptListenerImpl method testCheckpointIDTracking.

@Test
public void testCheckpointIDTracking() throws IOException, InterruptedException {
    SystemClock clock = SystemClock.getInstance();
    org.apache.hadoop.mapreduce.v2.app.job.Task mockTask = mock(org.apache.hadoop.mapreduce.v2.app.job.Task.class);
    when(mockTask.canCommit(any(TaskAttemptId.class))).thenReturn(true);
    Job mockJob = mock(Job.class);
    when(mockJob.getTask(any(TaskId.class))).thenReturn(mockTask);
    Dispatcher dispatcher = mock(Dispatcher.class);
    @SuppressWarnings("unchecked") EventHandler<Event> ea = mock(EventHandler.class);
    when(dispatcher.getEventHandler()).thenReturn(ea);
    RMHeartbeatHandler rmHeartbeatHandler = mock(RMHeartbeatHandler.class);
    AppContext appCtx = mock(AppContext.class);
    when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob);
    when(appCtx.getClock()).thenReturn(clock);
    when(appCtx.getEventHandler()).thenReturn(ea);
    JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
    final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
    when(appCtx.getEventHandler()).thenReturn(ea);
    CheckpointAMPreemptionPolicy policy = new CheckpointAMPreemptionPolicy();
    policy.init(appCtx);
    TaskAttemptListenerImpl listener = new MockTaskAttemptListenerImpl(appCtx, secret, rmHeartbeatHandler, policy) {

        @Override
        protected void registerHeartbeatHandler(Configuration conf) {
            taskHeartbeatHandler = hbHandler;
        }
    };
    Configuration conf = new Configuration();
    conf.setBoolean(MRJobConfig.TASK_PREEMPTION, true);
    //conf.setBoolean("preemption.reduce", true);
    listener.init(conf);
    listener.start();
    TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 0);
    List<Path> partialOut = new ArrayList<Path>();
    partialOut.add(new Path("/prev1"));
    partialOut.add(new Path("/prev2"));
    Counters counters = mock(Counters.class);
    final long CBYTES = 64L * 1024 * 1024;
    final long CTIME = 4344L;
    final Path CLOC = new Path("/test/1");
    Counter cbytes = mock(Counter.class);
    when(cbytes.getValue()).thenReturn(CBYTES);
    Counter ctime = mock(Counter.class);
    when(ctime.getValue()).thenReturn(CTIME);
    when(counters.findCounter(eq(EnumCounter.CHECKPOINT_BYTES))).thenReturn(cbytes);
    when(counters.findCounter(eq(EnumCounter.CHECKPOINT_MS))).thenReturn(ctime);
    // propagating a taskstatus that contains a checkpoint id
    TaskCheckpointID incid = new TaskCheckpointID(new FSCheckpointID(CLOC), partialOut, counters);
    listener.setCheckpointID(org.apache.hadoop.mapred.TaskID.downgrade(tid.getTaskID()), incid);
    // and try to get it back
    CheckpointID outcid = listener.getCheckpointID(tid.getTaskID());
    TaskCheckpointID tcid = (TaskCheckpointID) outcid;
    assertEquals(CBYTES, tcid.getCheckpointBytes());
    assertEquals(CTIME, tcid.getCheckpointTime());
    assertTrue(partialOut.containsAll(tcid.getPartialCommittedOutput()));
    assertTrue(tcid.getPartialCommittedOutput().containsAll(partialOut));
    //assert it worked
    assert outcid == incid;
    listener.stop();
}

Example 3 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project apex-core by apache.

the class StreamingContainerManager method getInstance.

/**
   * Get the instance for the given application. If the application directory contains a checkpoint, the state will be restored.
   *
   * @param rh
   * @param dag
   * @param enableEventRecording
   * @return instance of {@link StreamingContainerManager}
   * @throws IOException
   */
public static StreamingContainerManager getInstance(RecoveryHandler rh, LogicalPlan dag, boolean enableEventRecording) throws IOException {
    try {
        CheckpointState checkpointedState = (CheckpointState) rh.restore();
        StreamingContainerManager scm;
        if (checkpointedState == null) {
            scm = new StreamingContainerManager(dag, enableEventRecording, new SystemClock());
        } else {
            // find better way to support final transient members
            PhysicalPlan plan = checkpointedState.physicalPlan;
            plan.getLogicalPlan().setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, dag.getAttributes().get(LogicalPlan.APPLICATION_ATTEMPT_ID));
            scm = new StreamingContainerManager(checkpointedState, enableEventRecording);
            for (Field f : plan.getClass().getDeclaredFields()) {
                if (f.getType() == PlanContext.class) {
                    f.setAccessible(true);
                    try {
                        f.set(plan, scm);
                    } catch (Exception e) {
                        throw new RuntimeException("Failed to set " + f, e);
                    }
                    f.setAccessible(false);
                }
            }
            DataInputStream logStream = rh.getLog();
            scm.journal.replay(logStream);
            logStream.close();
            // restore checkpoint info
            plan.syncCheckpoints(scm.vars.windowStartMillis, scm.clock.getTime());
            scm.committedWindowId = scm.updateCheckpoints(true);
            // populate container agents for existing containers
            for (PTContainer c : plan.getContainers()) {
                if (c.getExternalId() != null) {
                    LOG.debug("Restore container agent {} for {}", c.getExternalId(), c);
                    StreamingContainerAgent sca = new StreamingContainerAgent(c, scm.newStreamingContainerContext(c), scm);
                    scm.containers.put(c.getExternalId(), sca);
                } else {
                    LOG.debug("Requesting new resource for {}", c.toIdStateString());
                    scm.requestContainer(c);
                }
            }
            scm.startedFromCheckpoint = true;
        }
        scm.recoveryHandler = rh;
        scm.checkpoint();
        return scm;
    } catch (IOException e) {
        throw new IllegalStateException("Failed to read checkpointed state", e);
    }
}

Example 4 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestFairSchedulerQueueInfo method testEmptyChildQueues.

@Test
public void testEmptyChildQueues() throws Exception {
    FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
    FairScheduler scheduler = mock(FairScheduler.class);
    AllocationConfiguration allocConf = new AllocationConfiguration(conf);
    when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
    when(scheduler.getConf()).thenReturn(conf);
    when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1));
    SystemClock clock = SystemClock.getInstance();
    when(scheduler.getClock()).thenReturn(clock);
    QueueManager queueManager = new QueueManager(scheduler);
    queueManager.initialize(conf);
    FSQueue testQueue = queueManager.getLeafQueue("test", true);
    FairSchedulerQueueInfo queueInfo = new FairSchedulerQueueInfo(testQueue, scheduler);
    Collection<FairSchedulerQueueInfo> childQueues = queueInfo.getChildQueues();
    Assert.assertNotNull(childQueues);
    Assert.assertEquals("Child QueueInfo was not empty", 0, childQueues.size());
}

Example 5 with SystemClock

use of org.apache.hadoop.yarn.util.SystemClock in project hadoop by apache.

the class TestFSParentQueue method setUp.

@Before
public void setUp() throws Exception {
    conf = new FairSchedulerConfiguration();
    FairScheduler scheduler = mock(FairScheduler.class);
    AllocationConfiguration allocConf = new AllocationConfiguration(conf);
    when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
    when(scheduler.getConf()).thenReturn(conf);
    SystemClock clock = SystemClock.getInstance();
    when(scheduler.getClock()).thenReturn(clock);
    notEmptyQueues = new HashSet<FSQueue>();
    queueManager = new QueueManager(scheduler) {

        @Override
        public boolean isEmpty(FSQueue queue) {
            return !notEmptyQueues.contains(queue);
        }
    };
    FSQueueMetrics.forQueue("root", null, true, conf);
    queueManager.initialize(conf);
}