Examples with OperatorStats org.apache.hadoop.hive.ql.stats.OperatorStats used on opensource projects

Example 1 with OperatorStats

use of org.apache.hadoop.hive.ql.stats.OperatorStats in project hive by apache.

the class TestCounterMapping method testMappingJoinLookup.

@Test
@Ignore("needs HiveFilter mapping")
public void testMappingJoinLookup() throws ParseException {
    IDriver driver = createDriver();
    PlanMapper pm0 = getMapperForQuery(driver, "select sum(tu.id_uv),sum(u) from tu join tv on (tu.id_uv = tv.id_uv) where u>1 and v>1");
    Iterator<LinkGroup> itG = pm0.iterateGroups();
    int checkedOperators = 0;
    while (itG.hasNext()) {
        LinkGroup g = itG.next();
        List<HiveFilter> hfs = g.getAll(HiveFilter.class);
        List<OperatorStats> oss = g.getAll(OperatorStats.class);
        List<FilterOperator> fos = g.getAll(FilterOperator.class);
        if (fos.size() > 0 && oss.size() > 0) {
            if (hfs.size() == 0) {
                fail("HiveFilter is not connected?");
            }
            OperatorStats os = oss.get(0);
            if (!(os.getOutputRecords() == 3 || os.getOutputRecords() == 6)) {
                fail("nonexpected number of records produced");
            }
            checkedOperators++;
        }
    }
    assertEquals(2, checkedOperators);
}

Example 2 with OperatorStats

use of org.apache.hadoop.hive.ql.stats.OperatorStats in project hive by apache.

the class TestReOptimization method testStatsAreSetInReopt.

@Test
public void testStatsAreSetInReopt() throws Exception {
    IDriver driver = createDriver("overlay,reoptimize");
    String query = "select assert_true_oom(${hiveconf:zzz} > sum(u*v)) from tu join tv on (tu.id_uv=tv.id_uv) where u<10 and v>1";
    PlanMapper pm = getMapperForQuery(driver, query);
    Iterator<LinkGroup> itG = pm.iterateGroups();
    int checkedOperators = 0;
    while (itG.hasNext()) {
        LinkGroup g = itG.next();
        List<FilterOperator> fos = g.getAll(FilterOperator.class);
        List<OperatorStats> oss = g.getAll(OperatorStats.class);
        if (fos.size() > 0 && oss.size() > 0) {
            fos.sort(TestCounterMapping.OPERATOR_ID_COMPARATOR.reversed());
            FilterOperator fo = fos.get(0);
            OperatorStats os = oss.get(0);
            Statistics stats = fo.getStatistics();
            assertEquals(os.getOutputRecords(), stats.getNumRows());
            if (!(os.getOutputRecords() == 3 || os.getOutputRecords() == 6)) {
                fail("nonexpected number of records produced");
            }
            checkedOperators++;
        }
    }
    assertEquals(2, checkedOperators);
}

Example 3 with OperatorStats

use of org.apache.hadoop.hive.ql.stats.OperatorStats in project hive by apache.

the class StatsRulesProcFactory method applyRuntimeStats.

private static Statistics applyRuntimeStats(Context context, Statistics stats, Operator<?> op) {
    if (!context.getRuntimeStatsSource().isPresent()) {
        return stats;
    }
    RuntimeStatsSource rss = context.getRuntimeStatsSource().get();
    Optional<OperatorStats> os = rss.lookup(op);
    if (!os.isPresent()) {
        return stats;
    }
    LOG.debug("using runtime stats for {}; {}", op, os.get());
    Statistics outStats = stats.clone();
    outStats = outStats.scaleToRowCount(os.get().getOutputRecords(), false);
    outStats.setRuntimeStats(true);
    return outStats;
}

Example 4 with OperatorStats

use of org.apache.hadoop.hive.ql.stats.OperatorStats in project hive by apache.

the class SimpleRuntimeStatsSource method lookup.

@Override
public Optional<OperatorStats> lookup(Operator<?> op) {
    try {
        OpTreeSignature sig = OpTreeSignature.of(op);
        List<OperatorStats> v = pm.lookupAll(OperatorStats.class, sig);
        if (v.size() > 0) {
            return Optional.of(v.get(0));
        }
        return Optional.empty();
    } catch (NoSuchElementException | IllegalArgumentException iae) {
        return Optional.empty();
    }
}