Examples with TimelineEntity org.apache.hadoop.yarn.api.records.timeline.TimelineEntity used on opensource projects

Example 1 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestLogInfo method testParseEntity.

@Test
public void testParseEntity() throws Exception {
    // Load test data
    TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
    EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME, TEST_ENTITY_FILE_NAME, UserGroupInformation.getLoginUser().getUserName());
    testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs);
    // Verify for the first batch
    PluginStoreTestUtils.verifyTestEntities(tdm);
    // Load new data
    TimelineEntity entityNew = PluginStoreTestUtils.createEntity("id_3", "type_3", 789l, null, null, null, null, "domain_id_1");
    TimelineEntities entityList = new TimelineEntities();
    entityList.addEntity(entityNew);
    writeEntitiesLeaveOpen(entityList, new Path(getTestRootPath(TEST_ATTEMPT_DIR_NAME), TEST_ENTITY_FILE_NAME));
    testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper, fs);
    // Verify the newly added data
    TimelineEntity entity3 = tdm.getEntity(entityNew.getEntityType(), entityNew.getEntityId(), EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser());
    assertNotNull(entity3);
    assertEquals("Failed to read out entity new", entityNew.getStartTime(), entity3.getStartTime());
    tdm.close();
}

Example 2 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class TestEntityGroupFSTimelineStore method testPluginRead.

@Test
public void testPluginRead() throws Exception {
    // Verify precondition
    assertEquals(EntityGroupPlugInForTest.class.getName(), store.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES));
    List<TimelineEntityGroupPlugin> currPlugins = store.getPlugins();
    for (TimelineEntityGroupPlugin plugin : currPlugins) {
        ClassLoader pluginClassLoader = plugin.getClass().getClassLoader();
        assertTrue("Should set up ApplicationClassLoader", pluginClassLoader instanceof ApplicationClassLoader);
        URL[] paths = ((URLClassLoader) pluginClassLoader).getURLs();
        boolean foundJAR = false;
        for (URL path : paths) {
            if (path.toString().contains(testJar.getAbsolutePath())) {
                foundJAR = true;
            }
        }
        assertTrue("Not found path " + testJar.getAbsolutePath() + " for plugin " + plugin.getClass().getName(), foundJAR);
    }
    // Load data and cache item, prepare timeline store by making a cache item
    EntityGroupFSTimelineStore.AppLogs appLogs = store.new AppLogs(mainTestAppId, mainTestAppDirPath, AppState.COMPLETED);
    EntityCacheItem cacheItem = new EntityCacheItem(EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId), config);
    cacheItem.setAppLogs(appLogs);
    store.setCachedLogs(EntityGroupPlugInForTest.getStandardTimelineGroupId(mainTestAppId), cacheItem);
    MutableCounterLong detailLogEntityRead = store.metrics.getGetEntityToDetailOps();
    MutableStat cacheRefresh = store.metrics.getCacheRefresh();
    long numEntityReadBefore = detailLogEntityRead.value();
    long cacheRefreshBefore = cacheRefresh.lastStat().numSamples();
    // Generate TDM
    TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithStore(config, store);
    // Verify single entity read
    TimelineEntity entity3 = tdm.getEntity("type_3", mainTestAppId.toString(), EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser());
    assertNotNull(entity3);
    assertEquals(entityNew.getStartTime(), entity3.getStartTime());
    // Verify multiple entities read
    NameValuePair primaryFilter = new NameValuePair(EntityGroupPlugInForTest.APP_ID_FILTER_NAME, mainTestAppId.toString());
    TimelineEntities entities = tdm.getEntities("type_3", primaryFilter, null, null, null, null, null, null, EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser());
    assertEquals(1, entities.getEntities().size());
    for (TimelineEntity entity : entities.getEntities()) {
        assertEquals(entityNew.getStartTime(), entity.getStartTime());
    }
    // Verify metrics
    assertEquals(numEntityReadBefore + 2L, detailLogEntityRead.value());
    assertEquals(cacheRefreshBefore + 1L, cacheRefresh.lastStat().numSamples());
}

Example 3 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class EntityGroupFSTimelineStore method getEntity.

@Override
public TimelineEntity getEntity(String entityId, String entityType, EnumSet<Field> fieldsToRetrieve) throws IOException {
    LOG.debug("getEntity type={} id={}", entityType, entityId);
    List<EntityCacheItem> relatedCacheItems = new ArrayList<>();
    List<TimelineStore> stores = getTimelineStoresForRead(entityId, entityType, relatedCacheItems);
    for (TimelineStore store : stores) {
        LOG.debug("Try timeline store {}:{} for the request", store.getName(), store.toString());
        TimelineEntity e = store.getEntity(entityId, entityType, fieldsToRetrieve);
        if (e != null) {
            return e;
        }
    }
    LOG.debug("getEntity: Found nothing");
    return null;
}

Example 4 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class PluginStoreTestUtils method verifyTestEntities.

static void verifyTestEntities(TimelineDataManager tdm) throws YarnException, IOException {
    TimelineEntity entity1 = tdm.getEntity("type_1", "id_1", EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser());
    TimelineEntity entity2 = tdm.getEntity("type_2", "id_2", EnumSet.allOf(TimelineReader.Field.class), UserGroupInformation.getLoginUser());
    assertNotNull(entity1);
    assertNotNull(entity2);
    assertEquals("Failed to read out entity 1", (Long) 123l, entity1.getStartTime());
    assertEquals("Failed to read out entity 2", (Long) 456l, entity2.getStartTime());
}

Example 5 with TimelineEntity

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntity in project hadoop by apache.

the class PluginStoreTestUtils method createEntity.

/**
   * Create a test entity
   */
static TimelineEntity createEntity(String entityId, String entityType, Long startTime, List<TimelineEvent> events, Map<String, Set<String>> relatedEntities, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, String domainId) {
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(entityId);
    entity.setEntityType(entityType);
    entity.setStartTime(startTime);
    entity.setEvents(events);
    if (relatedEntities != null) {
        for (Map.Entry<String, Set<String>> e : relatedEntities.entrySet()) {
            for (String v : e.getValue()) {
                entity.addRelatedEntity(e.getKey(), v);
            }
        }
    } else {
        entity.setRelatedEntities(null);
    }
    entity.setPrimaryFilters(primaryFilters);
    entity.setOtherInfo(otherInfo);
    entity.setDomainId(domainId);
    return entity;
}