Search in sources :

Example 66 with Set

use of java.util.Set in project hadoop by apache.

the class TestTimelineReaderWebServicesHBaseStorage method testGetEntitiesMetricFilters.

@Test
public void testGetEntitiesMetricFilters() throws Exception {
    Client client = createClient();
    try {
        // metricfilters=HDFS_BYTES_READ lt 60 OR HDFS_BYTES_READ eq 157
        URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=HDFS_BYTES_READ%20lt%2060%20OR%20" + "HDFS_BYTES_READ%20eq%20157");
        ClientResponse resp = getResponse(client, uri);
        Set<TimelineEntity> entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity1") || entity.getId().equals("entity2"));
        }
        // metricfilters=HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(0, entities.size());
        // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR
        // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122)
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + "%20140%20AND%20MAP11_SLOT_MILLIS%20le%20122)");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(1, entities.size());
        int metricCnt = 0;
        for (TimelineEntity entity : entities) {
            metricCnt += entity.getMetrics().size();
            assertTrue(entity.getId().equals("entity2"));
        }
        assertEquals(0, metricCnt);
        // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR
        // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122)
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + "%20140%20AND%20MAP11_SLOT_MILLIS%20le%20122)&fields=METRICS");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(1, entities.size());
        metricCnt = 0;
        for (TimelineEntity entity : entities) {
            metricCnt += entity.getMetrics().size();
            assertTrue(entity.getId().equals("entity2"));
        }
        assertEquals(3, metricCnt);
        // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR
        // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122)
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + "%20140%20AND%20MAP11_SLOT_MILLIS%20le%20122)&metricstoretrieve=" + "!(HDFS)");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(1, entities.size());
        metricCnt = 0;
        for (TimelineEntity entity : entities) {
            metricCnt += entity.getMetrics().size();
            assertTrue(entity.getId().equals("entity2"));
            for (TimelineMetric metric : entity.getMetrics()) {
                assertTrue(metric.getId().startsWith("MAP1"));
                assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric.getType());
            }
        }
        assertEquals(2, metricCnt);
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" + "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" + "%20140%20AND%20MAP11_SLOT_MILLIS%20le%20122)&metricstoretrieve=" + "!(HDFS)&metricslimit=10");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(1, entities.size());
        metricCnt = 0;
        for (TimelineEntity entity : entities) {
            metricCnt += entity.getMetrics().size();
            assertTrue(entity.getId().equals("entity2"));
            for (TimelineMetric metric : entity.getMetrics()) {
                assertTrue(metric.getId().startsWith("MAP1"));
                if (metric.getId().equals("MAP1_SLOT_MILLIS")) {
                    assertEquals(2, metric.getValues().size());
                    assertEquals(TimelineMetric.Type.TIME_SERIES, metric.getType());
                } else if (metric.getId().equals("MAP11_SLOT_MILLIS")) {
                    assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric.getType());
                } else {
                    fail("Unexpected metric id");
                }
            }
        }
        assertEquals(2, metricCnt);
        // Test for behavior when compare op is ne(not equals) vs ene
        // (exists and not equals). MAP11_SLOT_MILLIS does not exist for
        // entity1. For ne, both entity1 and entity2 will be returned. For ene,
        // only entity2 will be returned as we are checking for existence too.
        // metricfilters=MAP11_SLOT_MILLIS ne 100
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=MAP11_SLOT_MILLIS%20ne%20100");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity1") || entity.getId().equals("entity2"));
        }
        // metricfilters=MAP11_SLOT_MILLIS ene 100
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?metricfilters=MAP11_SLOT_MILLIS%20ene%20100");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(1, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity2"));
        }
    } finally {
        client.destroy();
    }
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) TimelineMetric(org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric) GenericType(com.sun.jersey.api.client.GenericType) Set(java.util.Set) HashSet(java.util.HashSet) Client(com.sun.jersey.api.client.Client) TimelineEntity(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity) URI(java.net.URI) Test(org.junit.Test)

Example 67 with Set

use of java.util.Set in project hadoop by apache.

the class TestTimelineReaderWebServicesHBaseStorage method testGetEntitiesRelationFilters.

@Test
public void testGetEntitiesRelationFilters() throws Exception {
    Client client = createClient();
    try {
        URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/apps/application_1111111111_1111/" + "entities/type1?isrelatedto=type3:entity31,type2:entity21:entity22");
        ClientResponse resp = getResponse(client, uri);
        Set<TimelineEntity> entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity1") || entity.getId().equals("entity2"));
        }
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?isrelatedto=!(type3:entity31,type2:entity21:entity22)");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(0, entities.size());
        // isrelatedto=!(type3:entity31,type2:entity21:entity22)OR type5:entity51,
        // type6:entity61:entity66
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?isrelatedto=!(type3:entity31,type2:entity21:entity22)%20OR%20" + "type5:entity51,type6:entity61:entity66");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(1, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity2"));
        }
        // isrelatedto=(!(type3:entity31,type2:entity21:entity22)OR type5:
        // entity51,type6:entity61:entity66) OR (type1:entity14,type2:entity21:
        // entity22 AND (type3:entity32:entity35,type4:entity42))
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?isrelatedto=(!(type3:entity31,type2:entity21:entity22)%20OR%20" + "type5:entity51,type6:entity61:entity66)%20OR%20(type1:entity14," + "type2:entity21:entity22%20AND%20(type3:entity32:entity35," + "type4:entity42))");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity1") || entity.getId().equals("entity2"));
        }
        // relatesto=!(type3:entity31,type2:entity21:entity22)OR type5:entity51,
        // type6:entity61:entity66
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?relatesto=!%20(type3:entity31,type2:entity21:entity22%20)%20OR%20" + "type5:entity51,type6:entity61:entity66");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(1, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity2"));
        }
        // relatesto=(!(type3:entity31,type2:entity21:entity22)OR type5:entity51,
        // type6:entity61:entity66) OR (type1:entity14,type2:entity21:entity22 AND
        // (type3:entity32:entity35 , type4:entity42))
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + "clusters/cluster1/apps/application_1111111111_1111/entities/type1" + "?relatesto=(!(%20type3:entity31,type2:entity21:entity22)%20OR%20" + "type5:entity51,type6:entity61:entity66%20)%20OR%20(type1:entity14," + "type2:entity21:entity22%20AND%20(type3:entity32:entity35%20,%20" + "type4:entity42))");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<TimelineEntity>>() {
        });
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (TimelineEntity entity : entities) {
            assertTrue(entity.getId().equals("entity1") || entity.getId().equals("entity2"));
        }
    } finally {
        client.destroy();
    }
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) GenericType(com.sun.jersey.api.client.GenericType) Set(java.util.Set) HashSet(java.util.HashSet) Client(com.sun.jersey.api.client.Client) TimelineEntity(org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity) URI(java.net.URI) Test(org.junit.Test)

Example 68 with Set

use of java.util.Set in project hadoop by apache.

the class TestTimelineReaderWebServicesHBaseStorage method testGetFlowRuns.

@Test
public void testGetFlowRuns() throws Exception {
    Client client = createClient();
    try {
        URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs");
        ClientResponse resp = getResponse(client, uri);
        Set<FlowRunEntity> entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>() {
        });
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", resp.getType().toString());
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (FlowRunEntity entity : entities) {
            assertTrue("Id, run id or start time does not match.", ((entity.getId().equals("user1@flow_name/1002345678919")) && (entity.getRunId() == 1002345678919L) && (entity.getStartTime() == 1425016501000L)) || ((entity.getId().equals("user1@flow_name/1002345678920")) && (entity.getRunId() == 1002345678920L) && (entity.getStartTime() == 1425016501034L)));
            assertEquals(0, entity.getMetrics().size());
        }
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" + "clusters/cluster1/users/user1/flows/flow_name/runs?limit=1");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>() {
        });
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", resp.getType().toString());
        assertNotNull(entities);
        assertEquals(1, entities.size());
        for (FlowRunEntity entity : entities) {
            assertTrue("Id, run id or start time does not match.", entity.getId().equals("user1@flow_name/1002345678920") && entity.getRunId() == 1002345678920L && entity.getStartTime() == 1425016501034L);
            assertEquals(0, entity.getMetrics().size());
        }
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "createdtimestart=1425016501030");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>() {
        });
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", resp.getType().toString());
        assertNotNull(entities);
        assertEquals(1, entities.size());
        for (FlowRunEntity entity : entities) {
            assertTrue("Id, run id or start time does not match.", entity.getId().equals("user1@flow_name/1002345678920") && entity.getRunId() == 1002345678920L && entity.getStartTime() == 1425016501034L);
            assertEquals(0, entity.getMetrics().size());
        }
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "createdtimestart=1425016500999&createdtimeend=1425016501035");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>() {
        });
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", resp.getType().toString());
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (FlowRunEntity entity : entities) {
            assertTrue("Id, run id or start time does not match.", ((entity.getId().equals("user1@flow_name/1002345678919")) && (entity.getRunId() == 1002345678919L) && (entity.getStartTime() == 1425016501000L)) || ((entity.getId().equals("user1@flow_name/1002345678920")) && (entity.getRunId() == 1002345678920L) && (entity.getStartTime() == 1425016501034L)));
            assertEquals(0, entity.getMetrics().size());
        }
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "createdtimeend=1425016501030");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>() {
        });
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", resp.getType().toString());
        assertNotNull(entities);
        assertEquals(1, entities.size());
        for (FlowRunEntity entity : entities) {
            assertTrue("Id, run id or start time does not match.", entity.getId().equals("user1@flow_name/1002345678919") && entity.getRunId() == 1002345678919L && entity.getStartTime() == 1425016501000L);
            assertEquals(0, entity.getMetrics().size());
        }
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "fields=metrics");
        resp = getResponse(client, uri);
        entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>() {
        });
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; charset=utf-8", resp.getType().toString());
        assertNotNull(entities);
        assertEquals(2, entities.size());
        for (FlowRunEntity entity : entities) {
            assertTrue("Id, run id or start time does not match.", ((entity.getId().equals("user1@flow_name/1002345678919")) && (entity.getRunId() == 1002345678919L) && (entity.getStartTime() == 1425016501000L) && (entity.getMetrics().size() == 3)) || ((entity.getId().equals("user1@flow_name/1002345678920")) && (entity.getRunId() == 1002345678920L) && (entity.getStartTime() == 1425016501034L) && (entity.getMetrics().size() == 1)));
        }
        // fields as CONFIGS will lead to a HTTP 400 as it makes no sense for
        // flow runs.
        uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" + "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" + "fields=CONFIGS");
        verifyHttpResponse(client, uri, Status.BAD_REQUEST);
    } finally {
        client.destroy();
    }
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) GenericType(com.sun.jersey.api.client.GenericType) Set(java.util.Set) HashSet(java.util.HashSet) FlowRunEntity(org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity) Client(com.sun.jersey.api.client.Client) URI(java.net.URI) Test(org.junit.Test)

Example 69 with Set

use of java.util.Set in project hadoop by apache.

the class TestEntityGroupFSTimelineStore method createTestFiles.

private void createTestFiles(ApplicationId appId, Path attemptDirPath) throws IOException {
    TimelineEntities entities = PluginStoreTestUtils.generateTestEntities();
    PluginStoreTestUtils.writeEntities(entities, new Path(attemptDirPath, TEST_SUMMARY_LOG_FILE_NAME), fs);
    Map<String, Set<Object>> primaryFilters = new HashMap<>();
    Set<Object> appSet = new HashSet<Object>();
    appSet.add(appId.toString());
    primaryFilters.put(EntityGroupPlugInForTest.APP_ID_FILTER_NAME, appSet);
    entityNew = PluginStoreTestUtils.createEntity(appId.toString(), "type_3", 789L, null, null, primaryFilters, null, "domain_id_1");
    TimelineEntities entityList = new TimelineEntities();
    entityList.addEntity(entityNew);
    PluginStoreTestUtils.writeEntities(entityList, new Path(attemptDirPath, mainEntityLogFileName), fs);
    FSDataOutputStream out = fs.create(new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
    out.close();
}
Also used : Path(org.apache.hadoop.fs.Path) HashSet(java.util.HashSet) EnumSet(java.util.EnumSet) Set(java.util.Set) TimelineEntities(org.apache.hadoop.yarn.api.records.timeline.TimelineEntities) HashMap(java.util.HashMap) FSDataOutputStream(org.apache.hadoop.fs.FSDataOutputStream) HashSet(java.util.HashSet)

Example 70 with Set

use of java.util.Set in project hadoop by apache.

the class PluginStoreTestUtils method createEntity.

/**
   * Create a test entity
   */
static TimelineEntity createEntity(String entityId, String entityType, Long startTime, List<TimelineEvent> events, Map<String, Set<String>> relatedEntities, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, String domainId) {
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(entityId);
    entity.setEntityType(entityType);
    entity.setStartTime(startTime);
    entity.setEvents(events);
    if (relatedEntities != null) {
        for (Map.Entry<String, Set<String>> e : relatedEntities.entrySet()) {
            for (String v : e.getValue()) {
                entity.addRelatedEntity(e.getKey(), v);
            }
        }
    } else {
        entity.setRelatedEntities(null);
    }
    entity.setPrimaryFilters(primaryFilters);
    entity.setOtherInfo(otherInfo);
    entity.setDomainId(domainId);
    return entity;
}
Also used : HashSet(java.util.HashSet) EnumSet(java.util.EnumSet) Set(java.util.Set) TimelineEntity(org.apache.hadoop.yarn.api.records.timeline.TimelineEntity) HashMap(java.util.HashMap) Map(java.util.Map)

Aggregations

Set (java.util.Set)6789 HashSet (java.util.HashSet)4372 HashMap (java.util.HashMap)2090 Map (java.util.Map)1865 Iterator (java.util.Iterator)1774 ArrayList (java.util.ArrayList)1113 List (java.util.List)980 Test (org.junit.Test)920 TreeSet (java.util.TreeSet)536 IOException (java.io.IOException)501 SSOException (com.iplanet.sso.SSOException)467 LinkedHashSet (java.util.LinkedHashSet)418 SMSException (com.sun.identity.sm.SMSException)347 IdRepoException (com.sun.identity.idm.IdRepoException)268 Collection (java.util.Collection)259 ImmutableSet (com.google.common.collect.ImmutableSet)256 File (java.io.File)245 SSOToken (com.iplanet.sso.SSOToken)226 Collectors (java.util.stream.Collectors)219 Test (org.testng.annotations.Test)209