Search in sources :

Example 51 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TestAMWebServicesJobs method testJobCounters.

@Test
public void testJobCounters() throws JSONException, Exception {
    WebResource r = resource();
    Map<JobId, Job> jobsMap = appContext.getAllJobs();
    for (JobId id : jobsMap.keySet()) {
        String jobId = MRApps.toString(id);
        ClientResponse response = r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId).path("counters").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("jobCounters");
        verifyAMJobCounters(info, jobsMap.get(id));
    }
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) JSONObject(org.codehaus.jettison.json.JSONObject) WebResource(com.sun.jersey.api.client.WebResource) Job(org.apache.hadoop.mapreduce.v2.app.job.Job) JobId(org.apache.hadoop.mapreduce.v2.api.records.JobId) Test(org.junit.Test)

Example 52 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TypeConverter method toYarn.

public static Counters toYarn(org.apache.hadoop.mapreduce.Counters counters) {
    if (counters == null) {
        return null;
    }
    Counters yCntrs = recordFactory.newRecordInstance(Counters.class);
    yCntrs.addAllCounterGroups(new HashMap<String, CounterGroup>());
    for (org.apache.hadoop.mapreduce.CounterGroup grp : counters) {
        CounterGroup yGrp = recordFactory.newRecordInstance(CounterGroup.class);
        yGrp.setName(grp.getName());
        yGrp.setDisplayName(grp.getDisplayName());
        yGrp.addAllCounters(new HashMap<String, Counter>());
        for (org.apache.hadoop.mapreduce.Counter cntr : grp) {
            Counter yCntr = recordFactory.newRecordInstance(Counter.class);
            yCntr.setName(cntr.getName());
            yCntr.setDisplayName(cntr.getDisplayName());
            yCntr.setValue(cntr.getValue());
            yGrp.setCounter(yCntr.getName(), yCntr);
        }
        yCntrs.setCounterGroup(yGrp.getName(), yGrp);
    }
    return yCntrs;
}
Also used : Counter(org.apache.hadoop.mapreduce.v2.api.records.Counter) CounterGroup(org.apache.hadoop.mapreduce.v2.api.records.CounterGroup) Counters(org.apache.hadoop.mapreduce.v2.api.records.Counters)

Example 53 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TestAMWebServicesTasks method testJobTaskCountersXML.

@Test
public void testJobTaskCountersXML() throws Exception {
    WebResource r = resource();
    Map<JobId, Job> jobsMap = appContext.getAllJobs();
    for (JobId id : jobsMap.keySet()) {
        String jobId = MRApps.toString(id);
        for (Task task : jobsMap.get(id).getTasks().values()) {
            String tid = MRApps.toString(task.getID());
            ClientResponse response = r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid).path("counters").accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
            assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
            String xml = response.getEntity(String.class);
            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
            DocumentBuilder db = dbf.newDocumentBuilder();
            InputSource is = new InputSource();
            is.setCharacterStream(new StringReader(xml));
            Document dom = db.parse(is);
            NodeList info = dom.getElementsByTagName("jobTaskCounters");
            verifyAMTaskCountersXML(info, task);
        }
    }
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) Task(org.apache.hadoop.mapreduce.v2.app.job.Task) InputSource(org.xml.sax.InputSource) DocumentBuilderFactory(javax.xml.parsers.DocumentBuilderFactory) NodeList(org.w3c.dom.NodeList) WebResource(com.sun.jersey.api.client.WebResource) Document(org.w3c.dom.Document) DocumentBuilder(javax.xml.parsers.DocumentBuilder) StringReader(java.io.StringReader) Job(org.apache.hadoop.mapreduce.v2.app.job.Job) JobId(org.apache.hadoop.mapreduce.v2.api.records.JobId) Test(org.junit.Test)

Example 54 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TestAMWebServicesTasks method testTaskIdCountersSlash.

@Test
public void testTaskIdCountersSlash() throws JSONException, Exception {
    WebResource r = resource();
    Map<JobId, Job> jobsMap = appContext.getAllJobs();
    for (JobId id : jobsMap.keySet()) {
        String jobId = MRApps.toString(id);
        for (Task task : jobsMap.get(id).getTasks().values()) {
            String tid = MRApps.toString(task.getID());
            ClientResponse response = r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId).path("tasks").path(tid).path("counters/").accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
            assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
            JSONObject json = response.getEntity(JSONObject.class);
            assertEquals("incorrect number of elements", 1, json.length());
            JSONObject info = json.getJSONObject("jobTaskCounters");
            verifyAMJobTaskCounters(info, task);
        }
    }
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) Task(org.apache.hadoop.mapreduce.v2.app.job.Task) JSONObject(org.codehaus.jettison.json.JSONObject) WebResource(com.sun.jersey.api.client.WebResource) Job(org.apache.hadoop.mapreduce.v2.app.job.Job) JobId(org.apache.hadoop.mapreduce.v2.api.records.JobId) Test(org.junit.Test)

Example 55 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TestAMWebServicesJobs method testJobCountersDefault.

@Test
public void testJobCountersDefault() throws JSONException, Exception {
    WebResource r = resource();
    Map<JobId, Job> jobsMap = appContext.getAllJobs();
    for (JobId id : jobsMap.keySet()) {
        String jobId = MRApps.toString(id);
        ClientResponse response = r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId).path("counters/").get(ClientResponse.class);
        assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, response.getType().toString());
        JSONObject json = response.getEntity(JSONObject.class);
        assertEquals("incorrect number of elements", 1, json.length());
        JSONObject info = json.getJSONObject("jobCounters");
        verifyAMJobCounters(info, jobsMap.get(id));
    }
}
Also used : ClientResponse(com.sun.jersey.api.client.ClientResponse) JSONObject(org.codehaus.jettison.json.JSONObject) WebResource(com.sun.jersey.api.client.WebResource) Job(org.apache.hadoop.mapreduce.v2.app.job.Job) JobId(org.apache.hadoop.mapreduce.v2.api.records.JobId) Test(org.junit.Test)

Aggregations

Job (org.apache.hadoop.mapreduce.v2.app.job.Job)36 Test (org.junit.Test)34 JobId (org.apache.hadoop.mapreduce.v2.api.records.JobId)29 Task (org.apache.hadoop.mapreduce.v2.app.job.Task)28 ClientResponse (com.sun.jersey.api.client.ClientResponse)21 WebResource (com.sun.jersey.api.client.WebResource)21 Counters (org.apache.hadoop.mapreduce.Counters)18 TaskAttempt (org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt)16 TaskAttemptId (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId)15 JSONObject (org.codehaus.jettison.json.JSONObject)15 TaskId (org.apache.hadoop.mapreduce.v2.api.records.TaskId)13 Configuration (org.apache.hadoop.conf.Configuration)9 Counters (org.apache.hadoop.mapreduce.v2.api.records.Counters)8 StringReader (java.io.StringReader)6 GET (javax.ws.rs.GET)6 Path (javax.ws.rs.Path)6 Produces (javax.ws.rs.Produces)6 DocumentBuilder (javax.xml.parsers.DocumentBuilder)6 DocumentBuilderFactory (javax.xml.parsers.DocumentBuilderFactory)6 CounterGroup (org.apache.hadoop.mapreduce.v2.api.records.CounterGroup)6