Examples with TimelineEntity org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity used on opensource projects

Example 71 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class SimpleEntityWriterV2 method writeEntities.

protected void writeEntities(Configuration tlConf, TimelineCollectorManager manager, Context context) throws IOException {
    Configuration conf = context.getConfiguration();
    // simulate the app id with the task id
    int taskId = context.getTaskAttemptID().getTaskID().getId();
    long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
    ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
    // create the app level timeline collector
    AppLevelTimelineCollector collector = new AppLevelTimelineCollector(appId);
    manager.putIfAbsent(appId, collector);
    try {
        // set the context
        // flow id: job name, flow run id: timestamp, user id
        TimelineCollectorContext tlContext = collector.getTimelineEntityContext();
        tlContext.setFlowName(context.getJobName());
        tlContext.setFlowRunId(timestamp);
        tlContext.setUserId(context.getUser());
        final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);
        long totalTime = 0;
        final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
        final Random rand = new Random();
        final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
        final char[] payLoad = new char[kbs * 1024];
        for (int i = 0; i < testtimes; i++) {
            // Generate a fixed length random payload
            for (int xx = 0; xx < kbs * 1024; xx++) {
                int alphaNumIdx = rand.nextInt(ALPHA_NUMS.length);
                payLoad[xx] = ALPHA_NUMS[alphaNumIdx];
            }
            String entId = taskAttemptId + "_" + Integer.toString(i);
            final TimelineEntity entity = new TimelineEntity();
            entity.setId(entId);
            entity.setType("FOO_ATTEMPT");
            entity.addInfo("PERF_TEST", payLoad);
            // add an event
            TimelineEvent event = new TimelineEvent();
            event.setId("foo_event_id");
            event.setTimestamp(System.currentTimeMillis());
            event.addInfo("foo_event", "test");
            entity.addEvent(event);
            // add a metric
            TimelineMetric metric = new TimelineMetric();
            metric.setId("foo_metric");
            metric.addValue(System.currentTimeMillis(), 123456789L);
            entity.addMetric(metric);
            // add a config
            entity.addConfig("foo", "bar");
            TimelineEntities entities = new TimelineEntities();
            entities.addEntity(entity);
            // use the current user for this purpose
            UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
            long startWrite = System.nanoTime();
            try {
                collector.putEntities(entities, ugi);
            } catch (Exception e) {
                context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).increment(1);
                LOG.error("writing to the timeline service failed", e);
            }
            long endWrite = System.nanoTime();
            totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite - startWrite);
        }
        LOG.info("wrote " + testtimes + " entities (" + kbs * testtimes + " kB) in " + totalTime + " ms");
        context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).increment(totalTime);
        context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).increment(testtimes);
        context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).increment(kbs * testtimes);
    } finally {
        // clean up
        manager.remove(appId);
    }
}

Example 72 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TestSystemMetricsPublisherForV2 method verifyEntity.

private static void verifyEntity(File entityFile, long expectedEvents, String eventForCreatedTime, long expectedMetrics) throws IOException {
    BufferedReader reader = null;
    String strLine;
    long count = 0;
    long metricsCount = 0;
    try {
        reader = new BufferedReader(new FileReader(entityFile));
        while ((strLine = reader.readLine()) != null) {
            if (strLine.trim().length() > 0) {
                TimelineEntity entity = FileSystemTimelineReaderImpl.getTimelineRecordFromJSON(strLine.trim(), TimelineEntity.class);
                metricsCount = entity.getMetrics().size();
                for (TimelineEvent event : entity.getEvents()) {
                    if (event.getId().equals(eventForCreatedTime)) {
                        assertTrue(entity.getCreatedTime() > 0);
                        break;
                    }
                }
                count++;
            }
        }
    } finally {
        reader.close();
    }
    assertEquals("Expected " + expectedEvents + " events to be published", expectedEvents, count);
    assertEquals("Expected " + expectedMetrics + " metrics is incorrect", expectedMetrics, metricsCount);
}

Example 73 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TimelineReaderWebServices method getFlowRun.

/**
   * Return a single flow run for given UID which is a delimited string
   * containing clusterid, userid, flow name and flowrun id.
   *
   * @param req Servlet request.
   * @param res Servlet response.
   * @param uId a delimited string containing clusterid, userid, flow name and
   *     flowrun id which are extracted from UID and then used to query backend
   *     (Mandatory path param).
   * @param metricsToRetrieve If specified, defines which metrics to retrieve
   *     and send back in response.
   *
   * @return If successful, a HTTP 200(OK) response having a JSON representing a
   *     <cite>FlowRunEntity</cite> instance is returned. By default, all
   *     metrics for the flow run will be returned.<br>
   *     On failures,<br>
   *     If any problem occurs in parsing request or UID is incorrect,
   *     HTTP 400(Bad Request) is returned.<br>
   *     If flow run for the given flow run id cannot be found, HTTP 404
   *     (Not Found) is returned.<br>
   *     For all other errors while retrieving data, HTTP 500(Internal Server
   *     Error) is returned.
   */
@GET
@Path("/run-uid/{uid}/")
@Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8)
public TimelineEntity getFlowRun(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("uid") String uId, @QueryParam("metricstoretrieve") String metricsToRetrieve) {
    String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString());
    UserGroupInformation callerUGI = TimelineReaderWebServicesUtils.getUser(req);
    LOG.info("Received URL " + url + " from user " + TimelineReaderWebServicesUtils.getUserName(callerUGI));
    long startTime = Time.monotonicNow();
    init(res);
    TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
    TimelineEntity entity = null;
    try {
        TimelineReaderContext context = TimelineUIDConverter.FLOWRUN_UID.decodeUID(uId);
        if (context == null) {
            throw new BadRequestException("Incorrect UID " + uId);
        }
        context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
        entity = timelineReaderManager.getEntity(context, TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(null, metricsToRetrieve, null, null));
    } catch (Exception e) {
        handleException(e, url, startTime, "flowrunid");
    }
    long endTime = Time.monotonicNow();
    if (entity == null) {
        LOG.info("Processed URL " + url + " but flowrun not found (Took " + (endTime - startTime) + " ms.)");
        throw new NotFoundException("Flowrun with uid: " + uId + "is not found");
    }
    LOG.info("Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)");
    return entity;
}

Example 74 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class TimelineReaderWebServices method getApp.

/**
   * Return a single app for given cluster id and app id. If userid, flow name
   * and flowrun id which are optional query parameters are not specified, they
   * will be queried based on app id and cluster id from the flow context
   * information stored in underlying storage implementation.
   *
   * @param req Servlet request.
   * @param res Servlet response.
   * @param clusterId Cluster id to which the app to be queried belong to(
   *     Mandatory path param).
   * @param appId Application id to be queried(Mandatory path param).
   * @param flowName Flow name which should match for the app(Optional query
   *     param).
   * @param flowRunId Run id which should match for the app(Optional query
   *     param).
   * @param userId User id which should match for the app(Optional query param).
   * @param confsToRetrieve If specified, defines which configurations to
   *     retrieve and send back in response. These configs will be retrieved
   *     irrespective of whether configs are specified in fields to retrieve or
   *     not.
   * @param metricsToRetrieve If specified, defines which metrics to retrieve
   *     and send back in response. These metrics will be retrieved
   *     irrespective of whether metrics are specified in fields to retrieve or
   *     not.
   * @param fields Specifies which fields of the app entity object to retrieve,
   *     see {@link Field}. All fields will be retrieved if fields=ALL. If not
   *     specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
   *     app id and app created time is returned(Optional query param).
   * @param metricsLimit If specified, defines the number of metrics to return.
   *     Considered only if fields contains METRICS/ALL or metricsToRetrieve is
   *     specified. Ignored otherwise. The maximum possible value for
   *     metricsLimit can be {@link Integer#MAX_VALUE}. If it is not specified
   *     or has a value less than 1, and metrics have to be retrieved, then
   *     metricsLimit will be considered as 1 i.e. latest single value of
   *     metric(s) will be returned. (Optional query param).
   *
   * @return If successful, a HTTP 200(OK) response having a JSON representing a
   *     <cite>TimelineEntity</cite> instance is returned.<br>
   *     On failures,<br>
   *     If any problem occurs in parsing request, HTTP 400(Bad Request) is
   *     returned.<br>
   *     If flow context information cannot be retrieved or app for the given
   *     app id cannot be found, HTTP 404(Not Found) is returned.<br>
   *     For all other errors while retrieving data, HTTP 500(Internal Server
   *     Error) is returned.
   */
@GET
@Path("/clusters/{clusterid}/apps/{appid}/")
@Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8)
public TimelineEntity getApp(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("clusterid") String clusterId, @PathParam("appid") String appId, @QueryParam("flowname") String flowName, @QueryParam("flowrunid") String flowRunId, @QueryParam("userid") String userId, @QueryParam("confstoretrieve") String confsToRetrieve, @QueryParam("metricstoretrieve") String metricsToRetrieve, @QueryParam("fields") String fields, @QueryParam("metricslimit") String metricsLimit) {
    String url = req.getRequestURI() + (req.getQueryString() == null ? "" : QUERY_STRING_SEP + req.getQueryString());
    UserGroupInformation callerUGI = TimelineReaderWebServicesUtils.getUser(req);
    LOG.info("Received URL " + url + " from user " + TimelineReaderWebServicesUtils.getUserName(callerUGI));
    long startTime = Time.monotonicNow();
    init(res);
    TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
    TimelineEntity entity = null;
    try {
        entity = timelineReaderManager.getEntity(TimelineReaderWebServicesUtils.createTimelineReaderContext(clusterId, userId, flowName, flowRunId, appId, TimelineEntityType.YARN_APPLICATION.toString(), null), TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(confsToRetrieve, metricsToRetrieve, fields, metricsLimit));
    } catch (Exception e) {
        handleException(e, url, startTime, "flowrunid");
    }
    long endTime = Time.monotonicNow();
    if (entity == null) {
        LOG.info("Processed URL " + url + " but app not found" + " (Took " + (endTime - startTime) + " ms.)");
        throw new NotFoundException("App " + appId + " not found");
    }
    LOG.info("Processed URL " + url + " (Took " + (endTime - startTime) + " ms.)");
    return entity;
}

Example 75 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity in project hadoop by apache.

the class DataGeneratorForTest method loadApps.

static void loadApps(HBaseTestingUtility util) throws IOException {
    TimelineEntities te = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    String id = "application_1111111111_2222";
    entity.setId(id);
    entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
    Long cTime = 1425016502000L;
    entity.setCreatedTime(cTime);
    // add the info map in Timeline Entity
    Map<String, Object> infoMap = new HashMap<>();
    infoMap.put("infoMapKey1", "infoMapValue2");
    infoMap.put("infoMapKey2", 20);
    infoMap.put("infoMapKey3", 85.85);
    entity.addInfo(infoMap);
    // add the isRelatedToEntity info
    Set<String> isRelatedToSet = new HashSet<>();
    isRelatedToSet.add("relatedto1");
    Map<String, Set<String>> isRelatedTo = new HashMap<>();
    isRelatedTo.put("task", isRelatedToSet);
    entity.setIsRelatedToEntities(isRelatedTo);
    // add the relatesTo info
    Set<String> relatesToSet = new HashSet<>();
    relatesToSet.add("relatesto1");
    relatesToSet.add("relatesto3");
    Map<String, Set<String>> relatesTo = new HashMap<>();
    relatesTo.put("container", relatesToSet);
    Set<String> relatesToSet11 = new HashSet<>();
    relatesToSet11.add("relatesto4");
    relatesTo.put("container1", relatesToSet11);
    entity.setRelatesToEntities(relatesTo);
    // add some config entries
    Map<String, String> conf = new HashMap<>();
    conf.put("config_param1", "value1");
    conf.put("config_param2", "value2");
    conf.put("cfg_param1", "value3");
    entity.addConfigs(conf);
    // add metrics
    Set<TimelineMetric> metrics = new HashSet<>();
    TimelineMetric m1 = new TimelineMetric();
    m1.setId("MAP_SLOT_MILLIS");
    Map<Long, Number> metricValues = new HashMap<>();
    long ts = System.currentTimeMillis();
    metricValues.put(ts - 120000, 100000000);
    metricValues.put(ts - 100000, 200000000);
    metricValues.put(ts - 80000, 300000000);
    metricValues.put(ts - 60000, 400000000);
    metricValues.put(ts - 40000, 50000000000L);
    metricValues.put(ts - 20000, 60000000000L);
    m1.setType(Type.TIME_SERIES);
    m1.setValues(metricValues);
    metrics.add(m1);
    TimelineMetric m12 = new TimelineMetric();
    m12.setId("MAP1_BYTES");
    m12.addValue(ts, 50);
    metrics.add(m12);
    entity.addMetrics(metrics);
    TimelineEvent event = new TimelineEvent();
    event.setId("start_event");
    event.setTimestamp(ts);
    entity.addEvent(event);
    te.addEntity(entity);
    TimelineEntities te1 = new TimelineEntities();
    TimelineEntity entity1 = new TimelineEntity();
    String id1 = "application_1111111111_3333";
    entity1.setId(id1);
    entity1.setType(TimelineEntityType.YARN_APPLICATION.toString());
    entity1.setCreatedTime(cTime + 20L);
    // add the info map in Timeline Entity
    Map<String, Object> infoMap1 = new HashMap<>();
    infoMap1.put("infoMapKey1", "infoMapValue1");
    infoMap1.put("infoMapKey2", 10);
    entity1.addInfo(infoMap1);
    // add the isRelatedToEntity info
    Set<String> isRelatedToSet1 = new HashSet<>();
    isRelatedToSet1.add("relatedto3");
    isRelatedToSet1.add("relatedto5");
    Map<String, Set<String>> isRelatedTo1 = new HashMap<>();
    isRelatedTo1.put("task1", isRelatedToSet1);
    Set<String> isRelatedToSet11 = new HashSet<>();
    isRelatedToSet11.add("relatedto4");
    isRelatedTo1.put("task2", isRelatedToSet11);
    entity1.setIsRelatedToEntities(isRelatedTo1);
    // add the relatesTo info
    Set<String> relatesToSet1 = new HashSet<>();
    relatesToSet1.add("relatesto1");
    relatesToSet1.add("relatesto2");
    Map<String, Set<String>> relatesTo1 = new HashMap<>();
    relatesTo1.put("container", relatesToSet1);
    entity1.setRelatesToEntities(relatesTo1);
    // add some config entries
    Map<String, String> conf1 = new HashMap<>();
    conf1.put("cfg_param1", "value1");
    conf1.put("cfg_param2", "value2");
    entity1.addConfigs(conf1);
    // add metrics
    Set<TimelineMetric> metrics1 = new HashSet<>();
    TimelineMetric m2 = new TimelineMetric();
    m2.setId("MAP1_SLOT_MILLIS");
    Map<Long, Number> metricValues1 = new HashMap<>();
    long ts1 = System.currentTimeMillis();
    metricValues1.put(ts1 - 120000, 100000000);
    metricValues1.put(ts1 - 100000, 200000000);
    metricValues1.put(ts1 - 80000, 300000000);
    metricValues1.put(ts1 - 60000, 400000000);
    metricValues1.put(ts1 - 40000, 50000000000L);
    metricValues1.put(ts1 - 20000, 60000000000L);
    m2.setType(Type.TIME_SERIES);
    m2.setValues(metricValues1);
    metrics1.add(m2);
    entity1.addMetrics(metrics1);
    TimelineEvent event11 = new TimelineEvent();
    event11.setId("end_event");
    event11.setTimestamp(ts);
    entity1.addEvent(event11);
    TimelineEvent event12 = new TimelineEvent();
    event12.setId("update_event");
    event12.setTimestamp(ts - 10);
    entity1.addEvent(event12);
    te1.addEntity(entity1);
    TimelineEntities te2 = new TimelineEntities();
    TimelineEntity entity2 = new TimelineEntity();
    String id2 = "application_1111111111_4444";
    entity2.setId(id2);
    entity2.setType(TimelineEntityType.YARN_APPLICATION.toString());
    entity2.setCreatedTime(cTime + 40L);
    TimelineEvent event21 = new TimelineEvent();
    event21.setId("update_event");
    event21.setTimestamp(ts - 20);
    entity2.addEvent(event21);
    Set<String> isRelatedToSet2 = new HashSet<String>();
    isRelatedToSet2.add("relatedto3");
    Map<String, Set<String>> isRelatedTo2 = new HashMap<>();
    isRelatedTo2.put("task1", isRelatedToSet2);
    entity2.setIsRelatedToEntities(isRelatedTo2);
    Map<String, Set<String>> relatesTo3 = new HashMap<>();
    Set<String> relatesToSet14 = new HashSet<String>();
    relatesToSet14.add("relatesto7");
    relatesTo3.put("container2", relatesToSet14);
    entity2.setRelatesToEntities(relatesTo3);
    te2.addEntity(entity2);
    HBaseTimelineWriterImpl hbi = null;
    try {
        hbi = new HBaseTimelineWriterImpl();
        hbi.init(util.getConfiguration());
        hbi.start();
        String cluster = "cluster1";
        String user = "user1";
        String flow = "some_flow_name";
        String flowVersion = "AB7822C10F1111";
        long runid = 1002345678919L;
        String appName = "application_1111111111_2222";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
        appName = "application_1111111111_3333";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
        appName = "application_1111111111_4444";
        hbi.write(cluster, user, flow, flowVersion, runid, appName, te2);
        hbi.stop();
    } finally {
        if (hbi != null) {
            hbi.stop();
            hbi.close();
        }
    }
}