Examples with TimelineEntities org.apache.hadoop.yarn.api.records.timeline.TimelineEntities used on opensource projects

Example 36 with TimelineEntities

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntities in project hadoop by apache.

the class TestTimelineWebServices method testGetEventsWithYarnACLsEnabled.

@Test
public void testGetEventsWithYarnACLsEnabled() {
    AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager);
    try {
        // Put entity [5, 5] in domain 1
        TimelineEntities entities = new TimelineEntities();
        TimelineEntity entity = new TimelineEntity();
        entity.setEntityId("test id 5");
        entity.setEntityType("test type 5");
        entity.setStartTime(System.currentTimeMillis());
        entity.setDomainId("domain_id_1");
        TimelineEvent event = new TimelineEvent();
        event.setEventType("event type 1");
        event.setTimestamp(System.currentTimeMillis());
        entity.addEvent(event);
        entities.addEntity(entity);
        WebResource r = resource();
        ClientResponse response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_1").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertEquals(0, putResponse.getErrors().size());
        // Put entity [5, 6] in domain 2
        entities = new TimelineEntities();
        entity = new TimelineEntity();
        entity.setEntityId("test id 6");
        entity.setEntityType("test type 5");
        entity.setStartTime(System.currentTimeMillis());
        entity.setDomainId("domain_id_2");
        event = new TimelineEvent();
        event.setEventType("event type 2");
        event.setTimestamp(System.currentTimeMillis());
        entity.addEvent(event);
        entities.addEntity(entity);
        r = resource();
        response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_3").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertEquals(0, putResponse.getErrors().size());
        // Query events belonging to the entities of type 4
        response = r.path("ws").path("v1").path("timeline").path("test type 5").path("events").queryParam("user.name", "reader_user_1").queryParam("entityId", "test id 5,test id 6").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        TimelineEvents events = response.getEntity(TimelineEvents.class);
        // Reader 1 should just have the access to the events of entity [5, 5]
        assertEquals(1, events.getAllEvents().size());
        assertEquals("test id 5", events.getAllEvents().get(0).getEntityId());
    } finally {
        timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
    }
}

Example 37 with TimelineEntities

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntities in project hadoop by apache.

the class TestTimelineWebServices method testPostEntitiesWithYarnACLsEnabled.

@Test
public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
    AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager);
    try {
        TimelineEntities entities = new TimelineEntities();
        TimelineEntity entity = new TimelineEntity();
        entity.setEntityId("test id 2");
        entity.setEntityType("test type 2");
        entity.setStartTime(System.currentTimeMillis());
        entity.setDomainId("domain_id_1");
        entities.addEntity(entity);
        WebResource r = resource();
        ClientResponse response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_1").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertNotNull(putResponse);
        Assert.assertEquals(0, putResponse.getErrors().size());
        // override/append timeline data in the same entity with different user
        response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_3").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertNotNull(putResponse);
        Assert.assertEquals(1, putResponse.getErrors().size());
        Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, putResponse.getErrors().get(0).getErrorCode());
        // Cross domain relationship will be rejected
        entities = new TimelineEntities();
        entity = new TimelineEntity();
        entity.setEntityId("test id 3");
        entity.setEntityType("test type 2");
        entity.setStartTime(System.currentTimeMillis());
        entity.setDomainId("domain_id_2");
        entity.setRelatedEntities(Collections.singletonMap("test type 2", Collections.singleton("test id 2")));
        entities.addEntity(entity);
        r = resource();
        response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_3").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertNotNull(putResponse);
        Assert.assertEquals(1, putResponse.getErrors().size());
        Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION, putResponse.getErrors().get(0).getErrorCode());
        // Make sure the entity has been added anyway even though the
        // relationship is been excluded
        response = r.path("ws").path("v1").path("timeline").path("test type 2").path("test id 3").queryParam("user.name", "reader_user_3").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        entity = response.getEntity(TimelineEntity.class);
        Assert.assertNotNull(entity);
        Assert.assertEquals("test id 3", entity.getEntityId());
        Assert.assertEquals("test type 2", entity.getEntityType());
    } finally {
        timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
    }
}

Example 38 with TimelineEntities

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntities in project hadoop by apache.

the class TestTimelineWebServices method testGetEntityWithYarnACLsEnabled.

@Test
public void testGetEntityWithYarnACLsEnabled() throws Exception {
    AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager);
    try {
        TimelineEntities entities = new TimelineEntities();
        TimelineEntity entity = new TimelineEntity();
        entity.setEntityId("test id 3");
        entity.setEntityType("test type 3");
        entity.setStartTime(System.currentTimeMillis());
        entity.setDomainId("domain_id_1");
        entities.addEntity(entity);
        WebResource r = resource();
        ClientResponse response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_1").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertEquals(0, putResponse.getErrors().size());
        // verify the system data will not be exposed
        // 1. No field specification
        response = r.path("ws").path("v1").path("timeline").path("test type 3").path("test id 3").queryParam("user.name", "reader_user_1").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        entity = response.getEntity(TimelineEntity.class);
        Assert.assertNull(entity.getPrimaryFilters().get(TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
        // 2. other field
        response = r.path("ws").path("v1").path("timeline").path("test type 3").path("test id 3").queryParam("fields", "relatedentities").queryParam("user.name", "reader_user_1").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        entity = response.getEntity(TimelineEntity.class);
        Assert.assertNull(entity.getPrimaryFilters().get(TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
        // 3. primaryfilters field
        response = r.path("ws").path("v1").path("timeline").path("test type 3").path("test id 3").queryParam("fields", "primaryfilters").queryParam("user.name", "reader_user_1").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        entity = response.getEntity(TimelineEntity.class);
        Assert.assertNull(entity.getPrimaryFilters().get(TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
        // get entity with other user
        response = r.path("ws").path("v1").path("timeline").path("test type 3").path("test id 3").queryParam("user.name", "reader_user_2").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo());
    } finally {
        timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
    }
}

Example 39 with TimelineEntities

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntities in project hadoop by apache.

the class TestTimelineWebServices method testGetEntitiesWithYarnACLsEnabled.

@Test
public void testGetEntitiesWithYarnACLsEnabled() {
    AdminACLsManager oldAdminACLsManager = timelineACLsManager.setAdminACLsManager(adminACLsManager);
    try {
        // Put entity [4, 4] in domain 1
        TimelineEntities entities = new TimelineEntities();
        TimelineEntity entity = new TimelineEntity();
        entity.setEntityId("test id 4");
        entity.setEntityType("test type 4");
        entity.setStartTime(System.currentTimeMillis());
        entity.setDomainId("domain_id_1");
        entities.addEntity(entity);
        WebResource r = resource();
        ClientResponse response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_1").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        TimelinePutResponse putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertEquals(0, putResponse.getErrors().size());
        // Put entity [4, 5] in domain 2
        entities = new TimelineEntities();
        entity = new TimelineEntity();
        entity.setEntityId("test id 5");
        entity.setEntityType("test type 4");
        entity.setStartTime(System.currentTimeMillis());
        entity.setDomainId("domain_id_2");
        entities.addEntity(entity);
        r = resource();
        response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "writer_user_3").accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        putResponse = response.getEntity(TimelinePutResponse.class);
        Assert.assertEquals(0, putResponse.getErrors().size());
        // Query entities of type 4
        response = r.path("ws").path("v1").path("timeline").queryParam("user.name", "reader_user_1").path("test type 4").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, response.getType().toString());
        entities = response.getEntity(TimelineEntities.class);
        // Reader 1 should just have the access to entity [4, 4]
        assertEquals(1, entities.getEntities().size());
        assertEquals("test type 4", entities.getEntities().get(0).getEntityType());
        assertEquals("test id 4", entities.getEntities().get(0).getEntityId());
    } finally {
        timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
    }
}

Example 40 with TimelineEntities

use of org.apache.hadoop.yarn.api.records.timeline.TimelineEntities in project hadoop by apache.

the class JHEventHandlerForSigtermTest method testTimelineEventHandling.

// Have JobHistoryEventHandler handle some events and make sure they get
// stored to the Timeline store
@Test(timeout = 50000)
public void testTimelineEventHandling() throws Exception {
    TestParams t = new TestParams(RunningAppContext.class, false);
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    long currentTime = System.currentTimeMillis();
    try (MiniYARNCluster yarnCluster = new MiniYARNCluster(TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1)) {
        yarnCluster.init(conf);
        yarnCluster.start();
        Configuration confJHEH = new YarnConfiguration(conf);
        confJHEH.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
        confJHEH.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":" + yarnCluster.getApplicationHistoryServer().getPort());
        JHEvenHandlerForTest jheh = new JHEvenHandlerForTest(t.mockAppContext, 0);
        jheh.init(confJHEH);
        jheh.start();
        TimelineStore ts = yarnCluster.getApplicationHistoryServer().getTimelineStore();
        handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent(t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1), currentTime - 10));
        TimelineEntities entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null);
        Assert.assertEquals(1, entities.getEntities().size());
        TimelineEntity tEntity = entities.getEntities().get(0);
        Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId());
        Assert.assertEquals(1, tEntity.getEvents().size());
        Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(0).getEventType());
        Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(0).getTimestamp());
        handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobSubmittedEvent(TypeConverter.fromYarn(t.jobId), "name", "user", 200, "/foo/job.xml", new HashMap<JobACL, AccessControlList>(), "default"), currentTime + 10));
        entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null);
        Assert.assertEquals(1, entities.getEntities().size());
        tEntity = entities.getEntities().get(0);
        Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId());
        Assert.assertEquals(2, tEntity.getEvents().size());
        Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(0).getEventType());
        Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(1).getEventType());
        Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(0).getTimestamp());
        Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(1).getTimestamp());
        handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobQueueChangeEvent(TypeConverter.fromYarn(t.jobId), "q2"), currentTime - 20));
        entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null);
        Assert.assertEquals(1, entities.getEntities().size());
        tEntity = entities.getEntities().get(0);
        Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId());
        Assert.assertEquals(3, tEntity.getEvents().size());
        Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(0).getEventType());
        Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(1).getEventType());
        Assert.assertEquals(EventType.JOB_QUEUE_CHANGED.toString(), tEntity.getEvents().get(2).getEventType());
        Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(0).getTimestamp());
        Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(1).getTimestamp());
        Assert.assertEquals(currentTime - 20, tEntity.getEvents().get(2).getTimestamp());
        handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()), currentTime));
        entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null);
        Assert.assertEquals(1, entities.getEntities().size());
        tEntity = entities.getEntities().get(0);
        Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId());
        Assert.assertEquals(4, tEntity.getEvents().size());
        Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(0).getEventType());
        Assert.assertEquals(EventType.JOB_FINISHED.toString(), tEntity.getEvents().get(1).getEventType());
        Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(2).getEventType());
        Assert.assertEquals(EventType.JOB_QUEUE_CHANGED.toString(), tEntity.getEvents().get(3).getEventType());
        Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(0).getTimestamp());
        Assert.assertEquals(currentTime, tEntity.getEvents().get(1).getTimestamp());
        Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(2).getTimestamp());
        Assert.assertEquals(currentTime - 20, tEntity.getEvents().get(3).getTimestamp());
        handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobUnsuccessfulCompletionEvent(TypeConverter.fromYarn(t.jobId), 0, 0, 0, JobStateInternal.KILLED.toString()), currentTime + 20));
        entities = ts.getEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null, null, null);
        Assert.assertEquals(1, entities.getEntities().size());
        tEntity = entities.getEntities().get(0);
        Assert.assertEquals(t.jobId.toString(), tEntity.getEntityId());
        Assert.assertEquals(5, tEntity.getEvents().size());
        Assert.assertEquals(EventType.JOB_KILLED.toString(), tEntity.getEvents().get(0).getEventType());
        Assert.assertEquals(EventType.JOB_SUBMITTED.toString(), tEntity.getEvents().get(1).getEventType());
        Assert.assertEquals(EventType.JOB_FINISHED.toString(), tEntity.getEvents().get(2).getEventType());
        Assert.assertEquals(EventType.AM_STARTED.toString(), tEntity.getEvents().get(3).getEventType());
        Assert.assertEquals(EventType.JOB_QUEUE_CHANGED.toString(), tEntity.getEvents().get(4).getEventType());
        Assert.assertEquals(currentTime + 20, tEntity.getEvents().get(0).getTimestamp());
        Assert.assertEquals(currentTime + 10, tEntity.getEvents().get(1).getTimestamp());
        Assert.assertEquals(currentTime, tEntity.getEvents().get(2).getTimestamp());
        Assert.assertEquals(currentTime - 10, tEntity.getEvents().get(3).getTimestamp());
        Assert.assertEquals(currentTime - 20, tEntity.getEvents().get(4).getTimestamp());
        handleEvent(jheh, new JobHistoryEvent(t.jobId, new TaskStartedEvent(t.taskID, 0, TaskType.MAP, "")));
        entities = ts.getEntities("MAPREDUCE_TASK", null, null, null, null, null, null, null, null, null);
        Assert.assertEquals(1, entities.getEntities().size());
        tEntity = entities.getEntities().get(0);
        Assert.assertEquals(t.taskID.toString(), tEntity.getEntityId());
        Assert.assertEquals(1, tEntity.getEvents().size());
        Assert.assertEquals(EventType.TASK_STARTED.toString(), tEntity.getEvents().get(0).getEventType());
        Assert.assertEquals(TaskType.MAP.toString(), tEntity.getEvents().get(0).getEventInfo().get("TASK_TYPE"));
        handleEvent(jheh, new JobHistoryEvent(t.jobId, new TaskStartedEvent(t.taskID, 0, TaskType.REDUCE, "")));
        entities = ts.getEntities("MAPREDUCE_TASK", null, null, null, null, null, null, null, null, null);
        Assert.assertEquals(1, entities.getEntities().size());
        tEntity = entities.getEntities().get(0);
        Assert.assertEquals(t.taskID.toString(), tEntity.getEntityId());
        Assert.assertEquals(2, tEntity.getEvents().size());
        Assert.assertEquals(EventType.TASK_STARTED.toString(), tEntity.getEvents().get(1).getEventType());
        Assert.assertEquals(TaskType.REDUCE.toString(), tEntity.getEvents().get(0).getEventInfo().get("TASK_TYPE"));
        Assert.assertEquals(TaskType.MAP.toString(), tEntity.getEvents().get(1).getEventInfo().get("TASK_TYPE"));
    }
}