Examples with CounterGroup org.apache.tez.common.counters.CounterGroup used on opensource projects

Example 1 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class BaseInfo method getCounter.

/**
 * Get counter for a specific counter group name.
 * If counterGroupName is not mentioned, it would end up returning counter found in all
 * groups
 *
 * @param counterGroupName
 * @param counter
 * @return Map<String, TezCounter> tez counter at every counter group level
 */
public Map<String, TezCounter> getCounter(String counterGroupName, String counter) {
    // TODO: FS, TaskCounters are directly getting added as TezCounters always pass those.  Need a
    // way to get rid of these.
    Map<String, TezCounter> result = Maps.newHashMap();
    Iterator<String> iterator = tezCounters.getGroupNames().iterator();
    boolean found = false;
    while (iterator.hasNext()) {
        CounterGroup counterGroup = tezCounters.getGroup(iterator.next());
        if (counterGroupName != null) {
            String groupName = counterGroup.getName();
            if (groupName.equals(counterGroupName)) {
                found = true;
            }
        }
        // Explicitly mention that no need to create the counter if not present
        TezCounter tezCounter = counterGroup.getUnderlyingGroup().findCounter(counter, false);
        if (tezCounter != null) {
            result.put(counterGroup.getName(), tezCounter);
        }
        if (found) {
            // Retrieved counter specific to a counter group. Safe to exit.
            break;
        }
    }
    return result;
}

Example 2 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class Utils method parseTezCountersFromJSON.

/**
 * Parse tez counters from json
 *
 * @param jsonObject
 * @return TezCounters
 * @throws JSONException
 */
public static TezCounters parseTezCountersFromJSON(JSONObject jsonObject) throws JSONException {
    TezCounters counters = new TezCounters();
    if (jsonObject == null) {
        // empty counters.
        return counters;
    }
    final JSONArray counterGroupNodes = jsonObject.optJSONArray(Constants.COUNTER_GROUPS);
    if (counterGroupNodes != null) {
        for (int i = 0; i < counterGroupNodes.length(); i++) {
            JSONObject counterGroupNode = counterGroupNodes.optJSONObject(i);
            final String groupName = counterGroupNode.optString(Constants.COUNTER_GROUP_NAME);
            final String groupDisplayName = counterGroupNode.optString(Constants.COUNTER_GROUP_DISPLAY_NAME, groupName);
            CounterGroup group = counters.addGroup(groupName, groupDisplayName);
            final JSONArray counterNodes = counterGroupNode.optJSONArray(Constants.COUNTERS);
            // Parse counter nodes
            for (int j = 0; j < counterNodes.length(); j++) {
                JSONObject counterNode = counterNodes.optJSONObject(j);
                final String counterName = counterNode.getString(Constants.COUNTER_NAME);
                final String counterDisplayName = counterNode.optString(Constants.COUNTER_DISPLAY_NAME, counterName);
                final long counterValue = counterNode.getLong(Constants.COUNTER_VALUE);
                addCounter(group, counterName, counterDisplayName, counterValue);
            }
        }
    }
    return counters;
}

Example 3 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class TezTypeConverters method fromTez.

public static Counters fromTez(TezCounters tezCounters) {
    if (tezCounters == null) {
        return null;
    }
    Counters counters = new Counters();
    for (CounterGroup xGrp : tezCounters) {
        counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
        for (TezCounter xCounter : xGrp) {
            Counter counter = counters.findCounter(xGrp.getName(), xCounter.getName());
            counter.setValue(xCounter.getValue());
        }
    }
    return counters;
}

Example 4 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class TestTaskExecution2 method verifySysCounters.

private void verifySysCounters(TezCounters tezCounters, int minTaskCounterCount, int minFsCounterCount) {
    Preconditions.checkArgument((minTaskCounterCount > 0 && minFsCounterCount > 0) || (minTaskCounterCount <= 0 && minFsCounterCount <= 0), "Both targetCounter counts should be postitive or negative. A mix is not expected");
    int numTaskCounters = 0;
    int numFsCounters = 0;
    for (CounterGroup counterGroup : tezCounters) {
        if (counterGroup.getName().equals(TaskCounter.class.getName())) {
            for (TezCounter ignored : counterGroup) {
                numTaskCounters++;
            }
        } else if (counterGroup.getName().equals(FileSystemCounter.class.getName())) {
            for (TezCounter ignored : counterGroup) {
                numFsCounters++;
            }
        }
    }
    // If Target <=0, assert counter count is exactly 0
    if (minTaskCounterCount <= 0) {
        assertEquals(0, numTaskCounters);
        assertEquals(0, numFsCounters);
    } else {
        assertTrue(numTaskCounters >= minTaskCounterCount);
        assertTrue(numFsCounters >= minFsCounterCount);
    }
}

Example 5 with CounterGroup

use of org.apache.tez.common.counters.CounterGroup in project tez by apache.

the class TestMockDAGAppMaster method testBasicCounters.

@Test(timeout = 10000)
public void testBasicCounters() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null, false, false);
    tezClient.start();
    final String vAName = "A";
    final String vBName = "B";
    final String procCounterName = "Proc";
    final String globalCounterName = "Global";
    DAG dag = DAG.create("testBasicCounters");
    Vertex vA = Vertex.create(vAName, ProcessorDescriptor.create("Proc.class"), 10);
    Vertex vB = Vertex.create(vBName, ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA).addVertex(vB).addEdge(Edge.create(vA, vB, EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"))));
    TezCounters temp = new TezCounters();
    temp.findCounter(new String(globalCounterName), new String(globalCounterName)).increment(1);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutput out = new DataOutputStream(bos);
    temp.write(out);
    final byte[] payload = bos.toByteArray();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    mockApp.countersDelegate = new CountersDelegate() {

        @Override
        public TezCounters getCounters(TaskSpec taskSpec) {
            String vName = taskSpec.getVertexName();
            TezCounters counters = new TezCounters();
            final DataInputByteBuffer in = new DataInputByteBuffer();
            in.reset(ByteBuffer.wrap(payload));
            try {
                // this ensures that the serde code path is covered.
                // the internal merges of counters covers the constructor code path.
                counters.readFields(in);
            } catch (IOException e) {
                Assert.fail(e.getMessage());
            }
            counters.findCounter(vName, procCounterName).increment(1);
            for (OutputSpec output : taskSpec.getOutputs()) {
                counters.findCounter(vName, output.getDestinationVertexName()).increment(1);
            }
            for (InputSpec input : taskSpec.getInputs()) {
                counters.findCounter(vName, input.getSourceVertexName()).increment(1);
            }
            return counters;
        }
    };
    mockApp.doSleep = false;
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockLauncher.startScheduling(true);
    DAGStatus status = dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, status.getState());
    TezCounters counters = dagImpl.getAllCounters();
    String osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH);
    if (SystemUtils.IS_OS_LINUX) {
        Assert.assertTrue(counters.findCounter(DAGCounter.AM_CPU_MILLISECONDS).getValue() > 0);
    }
    // verify processor counters
    Assert.assertEquals(10, counters.findCounter(vAName, procCounterName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, procCounterName).getValue());
    // verify edge counters
    Assert.assertEquals(10, counters.findCounter(vAName, vBName).getValue());
    Assert.assertEquals(1, counters.findCounter(vBName, vAName).getValue());
    // verify global counters
    Assert.assertEquals(11, counters.findCounter(globalCounterName, globalCounterName).getValue());
    VertexImpl vAImpl = (VertexImpl) dagImpl.getVertex(vAName);
    VertexImpl vBImpl = (VertexImpl) dagImpl.getVertex(vBName);
    TezCounters vACounters = vAImpl.getAllCounters();
    TezCounters vBCounters = vBImpl.getAllCounters();
    String vACounterName = vACounters.findCounter(globalCounterName, globalCounterName).getName();
    String vBCounterName = vBCounters.findCounter(globalCounterName, globalCounterName).getName();
    if (vACounterName != vBCounterName) {
        Assert.fail("String counter name objects dont match despite interning.");
    }
    CounterGroup vaGroup = vACounters.getGroup(globalCounterName);
    String vaGrouName = vaGroup.getName();
    CounterGroup vBGroup = vBCounters.getGroup(globalCounterName);
    String vBGrouName = vBGroup.getName();
    if (vaGrouName != vBGrouName) {
        Assert.fail("String group name objects dont match despite interning.");
    }
    tezClient.stop();
}