Examples with TaskAttemptID org.apache.hadoop.mapred.TaskAttemptID used on opensource projects

Example 11 with TaskAttemptID

use of org.apache.hadoop.mapred.TaskAttemptID in project hadoop by apache.

the class TestHistoryViewerPrinter method addTaskAttemptInfo.

private static void addTaskAttemptInfo(JobHistoryParser.TaskInfo task, int id) {
    JobHistoryParser.TaskAttemptInfo attempt = new JobHistoryParser.TaskAttemptInfo();
    attempt.attemptId = new TaskAttemptID(TaskID.downgrade(task.getTaskId()), id);
    attempt.startTime = task.getStartTime();
    attempt.finishTime = task.getFinishTime();
    attempt.shuffleFinishTime = task.getFinishTime();
    attempt.sortFinishTime = task.getFinishTime();
    attempt.mapFinishTime = task.getFinishTime();
    attempt.status = task.getTaskStatus();
    attempt.taskType = task.getTaskType();
    attempt.trackerName = "localhost";
    attempt.httpPort = 1234;
    attempt.hostname = "localhost";
    task.attemptsMap.put(attempt.getAttemptId(), attempt);
}

Example 12 with TaskAttemptID

use of org.apache.hadoop.mapred.TaskAttemptID in project hadoop by apache.

the class TestShuffleScheduler method TestAggregatedTransferRate.

@SuppressWarnings("rawtypes")
@Test
public <K, V> void TestAggregatedTransferRate() throws Exception {
    JobConf job = new JobConf();
    job.setNumMapTasks(10);
    //mock creation
    TaskUmbilicalProtocol mockUmbilical = mock(TaskUmbilicalProtocol.class);
    Reporter mockReporter = mock(Reporter.class);
    FileSystem mockFileSystem = mock(FileSystem.class);
    Class<? extends org.apache.hadoop.mapred.Reducer> combinerClass = job.getCombinerClass();
    // needed for mock with generic
    @SuppressWarnings("unchecked") CombineOutputCollector<K, V> mockCombineOutputCollector = (CombineOutputCollector<K, V>) mock(CombineOutputCollector.class);
    org.apache.hadoop.mapreduce.TaskAttemptID mockTaskAttemptID = mock(org.apache.hadoop.mapreduce.TaskAttemptID.class);
    LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class);
    CompressionCodec mockCompressionCodec = mock(CompressionCodec.class);
    Counter mockCounter = mock(Counter.class);
    TaskStatus mockTaskStatus = mock(TaskStatus.class);
    Progress mockProgress = mock(Progress.class);
    MapOutputFile mockMapOutputFile = mock(MapOutputFile.class);
    Task mockTask = mock(Task.class);
    @SuppressWarnings("unchecked") MapOutput<K, V> output = mock(MapOutput.class);
    ShuffleConsumerPlugin.Context<K, V> context = new ShuffleConsumerPlugin.Context<K, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
    TaskStatus status = new TaskStatus() {

        @Override
        public boolean getIsMap() {
            return false;
        }

        @Override
        public void addFetchFailedMap(TaskAttemptID mapTaskId) {
        }
    };
    Progress progress = new Progress();
    ShuffleSchedulerImpl<K, V> scheduler = new ShuffleSchedulerImpl<K, V>(job, status, null, null, progress, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter());
    TaskAttemptID attemptID0 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 0), 0);
    //adding the 1st interval, 40MB from 60s to 100s
    long bytes = (long) 40 * 1024 * 1024;
    scheduler.copySucceeded(attemptID0, new MapHost(null, null), bytes, 60000, 100000, output);
    Assert.assertEquals(copyMessage(1, 1, 1), progress.toString());
    TaskAttemptID attemptID1 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 1), 1);
    //adding the 2nd interval before the 1st interval, 50MB from 0s to 50s
    bytes = (long) 50 * 1024 * 1024;
    scheduler.copySucceeded(attemptID1, new MapHost(null, null), bytes, 0, 50000, output);
    Assert.assertEquals(copyMessage(2, 1, 1), progress.toString());
    TaskAttemptID attemptID2 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 2), 2);
    //adding the 3rd interval overlapping with the 1st and the 2nd interval
    //110MB from 25s to 80s
    bytes = (long) 110 * 1024 * 1024;
    scheduler.copySucceeded(attemptID2, new MapHost(null, null), bytes, 25000, 80000, output);
    Assert.assertEquals(copyMessage(3, 2, 2), progress.toString());
    TaskAttemptID attemptID3 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 3), 3);
    //adding the 4th interval just after the 2nd interval, 100MB from 100s to 300s
    bytes = (long) 100 * 1024 * 1024;
    scheduler.copySucceeded(attemptID3, new MapHost(null, null), bytes, 100000, 300000, output);
    Assert.assertEquals(copyMessage(4, 0.5, 1), progress.toString());
    TaskAttemptID attemptID4 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 4), 4);
    //adding the 5th interval between after 4th, 50MB from 350s to 400s
    bytes = (long) 50 * 1024 * 1024;
    scheduler.copySucceeded(attemptID4, new MapHost(null, null), bytes, 350000, 400000, output);
    Assert.assertEquals(copyMessage(5, 1, 1), progress.toString());
    TaskAttemptID attemptID5 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 5), 5);
    //adding the 6th interval between after 5th, 50MB from 450s to 500s
    bytes = (long) 50 * 1024 * 1024;
    scheduler.copySucceeded(attemptID5, new MapHost(null, null), bytes, 450000, 500000, output);
    Assert.assertEquals(copyMessage(6, 1, 1), progress.toString());
    TaskAttemptID attemptID6 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 6), 6);
    //adding the 7th interval between after 5th and 6th interval, 20MB from 320s to 340s
    bytes = (long) 20 * 1024 * 1024;
    scheduler.copySucceeded(attemptID6, new MapHost(null, null), bytes, 320000, 340000, output);
    Assert.assertEquals(copyMessage(7, 1, 1), progress.toString());
    TaskAttemptID attemptID7 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 7), 7);
    //adding the 8th interval overlapping with 4th, 5th, and 7th 30MB from 290s to 350s
    bytes = (long) 30 * 1024 * 1024;
    scheduler.copySucceeded(attemptID7, new MapHost(null, null), bytes, 290000, 350000, output);
    Assert.assertEquals(copyMessage(8, 0.5, 1), progress.toString());
    TaskAttemptID attemptID8 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 8), 8);
    //adding the 9th interval overlapping with 5th and 6th, 50MB from 400s to 450s
    bytes = (long) 50 * 1024 * 1024;
    scheduler.copySucceeded(attemptID8, new MapHost(null, null), bytes, 400000, 450000, output);
    Assert.assertEquals(copyMessage(9, 1, 1), progress.toString());
    TaskAttemptID attemptID9 = new TaskAttemptID(new org.apache.hadoop.mapred.TaskID(new JobID("test", 0), TaskType.MAP, 9), 9);
    //adding the 10th interval overlapping with all intervals, 500MB from 0s to 500s
    bytes = (long) 500 * 1024 * 1024;
    scheduler.copySucceeded(attemptID9, new MapHost(null, null), bytes, 0, 500000, output);
    Assert.assertEquals(copyMessage(10, 1, 2), progress.toString());
}

Example 13 with TaskAttemptID

use of org.apache.hadoop.mapred.TaskAttemptID in project hadoop by apache.

the class TestShuffleScheduler method testTipFailed.

@SuppressWarnings("rawtypes")
@Test
public void testTipFailed() throws Exception {
    JobConf job = new JobConf();
    job.setNumMapTasks(2);
    TaskStatus status = new TaskStatus() {

        @Override
        public boolean getIsMap() {
            return false;
        }

        @Override
        public void addFetchFailedMap(TaskAttemptID mapTaskId) {
        }
    };
    Progress progress = new Progress();
    TaskAttemptID reduceId = new TaskAttemptID("314159", 0, TaskType.REDUCE, 0, 0);
    ShuffleSchedulerImpl scheduler = new ShuffleSchedulerImpl(job, status, reduceId, null, progress, null, null, null);
    JobID jobId = new JobID();
    TaskID taskId1 = new TaskID(jobId, TaskType.REDUCE, 1);
    scheduler.tipFailed(taskId1);
    Assert.assertEquals("Progress should be 0.5", 0.5f, progress.getProgress(), 0.0f);
    Assert.assertFalse(scheduler.waitUntilDone(1));
    TaskID taskId0 = new TaskID(jobId, TaskType.REDUCE, 0);
    scheduler.tipFailed(taskId0);
    Assert.assertEquals("Progress should be 1.0", 1.0f, progress.getProgress(), 0.0f);
    Assert.assertTrue(scheduler.waitUntilDone(1));
}

Example 14 with TaskAttemptID

use of org.apache.hadoop.mapred.TaskAttemptID in project hadoop by apache.

the class TestPipeApplication method readStdOut.

private String readStdOut(JobConf conf) throws Exception {
    TaskAttemptID taskId = TaskAttemptID.forName(conf.get(MRJobConfig.TASK_ATTEMPT_ID));
    File stdOut = TaskLog.getTaskLogFile(taskId, false, TaskLog.LogName.STDOUT);
    return readFile(stdOut);
}

Example 15 with TaskAttemptID

use of org.apache.hadoop.mapred.TaskAttemptID in project hadoop by apache.

the class NativeMapOutputCollectorDelegator method init.

@SuppressWarnings("unchecked")
@Override
public void init(Context context) throws IOException, ClassNotFoundException {
    this.context = context;
    this.job = context.getJobConf();
    Platforms.init(job);
    if (job.getNumReduceTasks() == 0) {
        String message = "There is no reducer, no need to use native output collector";
        LOG.error(message);
        throw new InvalidJobConfException(message);
    }
    Class<?> comparatorClass = job.getClass(MRJobConfig.KEY_COMPARATOR, null, RawComparator.class);
    if (comparatorClass != null && !Platforms.define(comparatorClass)) {
        String message = "Native output collector doesn't support customized java comparator " + job.get(MRJobConfig.KEY_COMPARATOR);
        LOG.error(message);
        throw new InvalidJobConfException(message);
    }
    if (!QuickSort.class.getName().equals(job.get(Constants.MAP_SORT_CLASS))) {
        String message = "Native-Task doesn't support sort class " + job.get(Constants.MAP_SORT_CLASS);
        LOG.error(message);
        throw new InvalidJobConfException(message);
    }
    if (job.getBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, false) == true) {
        String message = "Native-Task doesn't support secure shuffle";
        LOG.error(message);
        throw new InvalidJobConfException(message);
    }
    final Class<?> keyCls = job.getMapOutputKeyClass();
    try {
        @SuppressWarnings("rawtypes") final INativeSerializer serializer = NativeSerialization.getInstance().getSerializer(keyCls);
        if (null == serializer) {
            String message = "Key type not supported. Cannot find serializer for " + keyCls.getName();
            LOG.error(message);
            throw new InvalidJobConfException(message);
        } else if (!Platforms.support(keyCls.getName(), serializer, job)) {
            String message = "Native output collector doesn't support this key, " + "this key is not comparable in native: " + keyCls.getName();
            LOG.error(message);
            throw new InvalidJobConfException(message);
        }
    } catch (final IOException e) {
        String message = "Cannot find serializer for " + keyCls.getName();
        LOG.error(message);
        throw new IOException(message);
    }
    final boolean ret = NativeRuntime.isNativeLibraryLoaded();
    if (ret) {
        if (job.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false)) {
            String codec = job.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC);
            if (!NativeRuntime.supportsCompressionCodec(codec.getBytes(Charsets.UTF_8))) {
                String message = "Native output collector doesn't support compression codec " + codec;
                LOG.error(message);
                throw new InvalidJobConfException(message);
            }
        }
        NativeRuntime.configure(job);
        final long updateInterval = job.getLong(Constants.NATIVE_STATUS_UPDATE_INTERVAL, Constants.NATIVE_STATUS_UPDATE_INTERVAL_DEFVAL);
        updater = new StatusReportChecker(context.getReporter(), updateInterval);
        updater.start();
    } else {
        String message = "NativeRuntime cannot be loaded, please check that " + "libnativetask.so is in hadoop library dir";
        LOG.error(message);
        throw new InvalidJobConfException(message);
    }
    this.handler = null;
    try {
        final Class<K> oKClass = (Class<K>) job.getMapOutputKeyClass();
        final Class<K> oVClass = (Class<K>) job.getMapOutputValueClass();
        final TaskAttemptID id = context.getMapTask().getTaskID();
        final TaskContext taskContext = new TaskContext(job, null, null, oKClass, oVClass, context.getReporter(), id);
        handler = NativeCollectorOnlyHandler.create(taskContext);
    } catch (final IOException e) {
        String message = "Native output collector cannot be loaded;";
        LOG.error(message);
        throw new IOException(message, e);
    }
    LOG.info("Native output collector can be successfully enabled!");
}