Examples with TimelineCollectorContext org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext used on opensource projects

Example 1 with TimelineCollectorContext

use of org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext in project hadoop by apache.

the class SimpleEntityWriterV2 method writeEntities.

protected void writeEntities(Configuration tlConf, TimelineCollectorManager manager, Context context) throws IOException {
    Configuration conf = context.getConfiguration();
    // simulate the app id with the task id
    int taskId = context.getTaskAttemptID().getTaskID().getId();
    long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
    ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
    // create the app level timeline collector
    AppLevelTimelineCollector collector = new AppLevelTimelineCollector(appId);
    manager.putIfAbsent(appId, collector);
    try {
        // set the context
        // flow id: job name, flow run id: timestamp, user id
        TimelineCollectorContext tlContext = collector.getTimelineEntityContext();
        tlContext.setFlowName(context.getJobName());
        tlContext.setFlowRunId(timestamp);
        tlContext.setUserId(context.getUser());
        final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);
        long totalTime = 0;
        final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
        final Random rand = new Random();
        final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
        final char[] payLoad = new char[kbs * 1024];
        for (int i = 0; i < testtimes; i++) {
            // Generate a fixed length random payload
            for (int xx = 0; xx < kbs * 1024; xx++) {
                int alphaNumIdx = rand.nextInt(ALPHA_NUMS.length);
                payLoad[xx] = ALPHA_NUMS[alphaNumIdx];
            }
            String entId = taskAttemptId + "_" + Integer.toString(i);
            final TimelineEntity entity = new TimelineEntity();
            entity.setId(entId);
            entity.setType("FOO_ATTEMPT");
            entity.addInfo("PERF_TEST", payLoad);
            // add an event
            TimelineEvent event = new TimelineEvent();
            event.setId("foo_event_id");
            event.setTimestamp(System.currentTimeMillis());
            event.addInfo("foo_event", "test");
            entity.addEvent(event);
            // add a metric
            TimelineMetric metric = new TimelineMetric();
            metric.setId("foo_metric");
            metric.addValue(System.currentTimeMillis(), 123456789L);
            entity.addMetric(metric);
            // add a config
            entity.addConfig("foo", "bar");
            TimelineEntities entities = new TimelineEntities();
            entities.addEntity(entity);
            // use the current user for this purpose
            UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
            long startWrite = System.nanoTime();
            try {
                collector.putEntities(entities, ugi);
            } catch (Exception e) {
                context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).increment(1);
                LOG.error("writing to the timeline service failed", e);
            }
            long endWrite = System.nanoTime();
            totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite - startWrite);
        }
        LOG.info("wrote " + testtimes + " entities (" + kbs * testtimes + " kB) in " + totalTime + " ms");
        context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).increment(totalTime);
        context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).increment(testtimes);
        context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).increment(kbs * testtimes);
    } finally {
        // clean up
        manager.remove(appId);
    }
}

Example 2 with TimelineCollectorContext

use of org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext in project hadoop by apache.

the class TestRMRestart method testRMRestartTimelineCollectorContext.

@Test(timeout = 60000)
public void testRMRestartTimelineCollectorContext() throws Exception {
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    MockRM rm1 = null;
    MockRM rm2 = null;
    try {
        rm1 = createMockRM(conf, memStore);
        rm1.start();
        MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
        nm1.registerNode();
        // submit an app.
        RMApp app = rm1.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", -1, null);
        // Check if app info has been saved.
        ApplicationStateData appState = rmAppState.get(app.getApplicationId());
        Assert.assertNotNull(appState);
        Assert.assertEquals(0, appState.getAttemptCount());
        Assert.assertEquals(appState.getApplicationSubmissionContext().getApplicationId(), app.getApplicationSubmissionContext().getApplicationId());
        // Allocate the AM
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt = app.getCurrentAppAttempt();
        ApplicationAttemptId attemptId1 = attempt.getAppAttemptId();
        rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
        ApplicationId appId = app.getApplicationId();
        TimelineCollectorContext contextBeforeRestart = rm1.getRMContext().getRMTimelineCollectorManager().get(appId).getTimelineEntityContext();
        // Restart RM.
        rm2 = createMockRM(conf, memStore);
        rm2.start();
        Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
        rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
        TimelineCollectorContext contextAfterRestart = rm2.getRMContext().getRMTimelineCollectorManager().get(appId).getTimelineEntityContext();
        Assert.assertEquals("Collector contexts for an app should be same " + "across restarts", contextBeforeRestart, contextAfterRestart);
    } finally {
        conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
        if (rm1 != null) {
            rm1.close();
        }
        if (rm2 != null) {
            rm2.close();
        }
    }
}

Example 3 with TimelineCollectorContext

use of org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext in project hadoop by apache.

the class RMTimelineCollectorManager method doPostPut.

@Override
protected void doPostPut(ApplicationId appId, TimelineCollector collector) {
    RMApp app = rmContext.getRMApps().get(appId);
    if (app == null) {
        throw new YarnRuntimeException("Unable to get the timeline collector context info for a " + "non-existing app " + appId);
    }
    String userId = app.getUser();
    TimelineCollectorContext context = collector.getTimelineEntityContext();
    if (userId != null && !userId.isEmpty()) {
        context.setUserId(userId);
    }
    // initialize the flow in the environment with default values for those
    // that do not specify the flow tags
    // flow name: app name (or app id if app name is missing),
    // flow version: "1", flow run id: start time
    context.setFlowName(TimelineUtils.generateDefaultFlowName(app.getName(), appId));
    context.setFlowVersion(TimelineUtils.DEFAULT_FLOW_VERSION);
    context.setFlowRunId(app.getStartTime());
    // the flow context is received via the application tags
    for (String tag : app.getApplicationTags()) {
        String[] parts = tag.split(":", 2);
        if (parts.length != 2 || parts[1].isEmpty()) {
            continue;
        }
        switch(parts[0].toUpperCase()) {
            case TimelineUtils.FLOW_NAME_TAG_PREFIX:
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Setting the flow name: " + parts[1]);
                }
                context.setFlowName(parts[1]);
                break;
            case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Setting the flow version: " + parts[1]);
                }
                context.setFlowVersion(parts[1]);
                break;
            case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Setting the flow run id: " + parts[1]);
                }
                context.setFlowRunId(Long.parseLong(parts[1]));
                break;
            default:
                break;
        }
    }
}

Example 4 with TimelineCollectorContext

use of org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext in project hadoop by apache.

the class JobHistoryFileReplayMapperV2 method writeEntities.

@Override
protected void writeEntities(Configuration tlConf, TimelineCollectorManager manager, Context context) throws IOException {
    JobHistoryFileReplayHelper helper = new JobHistoryFileReplayHelper(context);
    int replayMode = helper.getReplayMode();
    JobHistoryFileParser parser = helper.getParser();
    TimelineEntityConverterV2 converter = new TimelineEntityConverterV2();
    // collect the apps it needs to process
    Collection<JobFiles> jobs = helper.getJobFiles();
    if (jobs.isEmpty()) {
        LOG.info(context.getTaskAttemptID().getTaskID() + " will process no jobs");
    } else {
        LOG.info(context.getTaskAttemptID().getTaskID() + " will process " + jobs.size() + " jobs");
    }
    for (JobFiles job : jobs) {
        // process each job
        String jobIdStr = job.getJobId();
        // skip if either of the file is missing
        if (job.getJobConfFilePath() == null || job.getJobHistoryFilePath() == null) {
            LOG.info(jobIdStr + " missing either the job history file or the " + "configuration file. Skipping.");
            continue;
        }
        LOG.info("processing " + jobIdStr + "...");
        JobId jobId = TypeConverter.toYarn(JobID.forName(jobIdStr));
        ApplicationId appId = jobId.getAppId();
        // create the app level timeline collector and start it
        AppLevelTimelineCollector collector = new AppLevelTimelineCollector(appId);
        manager.putIfAbsent(appId, collector);
        try {
            // parse the job info and configuration
            JobInfo jobInfo = parser.parseHistoryFile(job.getJobHistoryFilePath());
            Configuration jobConf = parser.parseConfiguration(job.getJobConfFilePath());
            LOG.info("parsed the job history file and the configuration file " + "for job " + jobIdStr);
            // set the context
            // flow id: job name, flow run id: timestamp, user id
            TimelineCollectorContext tlContext = collector.getTimelineEntityContext();
            tlContext.setFlowName(jobInfo.getJobname());
            tlContext.setFlowRunId(jobInfo.getSubmitTime());
            tlContext.setUserId(jobInfo.getUsername());
            // create entities from job history and write them
            long totalTime = 0;
            List<TimelineEntity> entitySet = converter.createTimelineEntities(jobInfo, jobConf);
            LOG.info("converted them into timeline entities for job " + jobIdStr);
            // use the current user for this purpose
            UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
            long startWrite = System.nanoTime();
            try {
                switch(replayMode) {
                    case JobHistoryFileReplayHelper.WRITE_ALL_AT_ONCE:
                        writeAllEntities(collector, entitySet, ugi);
                        break;
                    case JobHistoryFileReplayHelper.WRITE_PER_ENTITY:
                        writePerEntity(collector, entitySet, ugi);
                        break;
                    default:
                        break;
                }
            } catch (Exception e) {
                context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).increment(1);
                LOG.error("writing to the timeline service failed", e);
            }
            long endWrite = System.nanoTime();
            totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite - startWrite);
            int numEntities = entitySet.size();
            LOG.info("wrote " + numEntities + " entities in " + totalTime + " ms");
            context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).increment(totalTime);
            context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).increment(numEntities);
        } finally {
            manager.remove(appId);
            // move it along
            context.progress();
        }
    }
}