Examples with EquivGroup org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.EquivGroup used on opensource projects

Example 1 with EquivGroup

use of org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.EquivGroup in project hive by apache.

the class StatsSources method extractStatsFromPlanMapper.

private static ImmutableList<PersistedRuntimeStats> extractStatsFromPlanMapper(PlanMapper pm) {
    Builder<PersistedRuntimeStats> li = ImmutableList.builder();
    Iterator<EquivGroup> it = pm.iterateGroups();
    while (it.hasNext()) {
        EquivGroup e = it.next();
        List<OperatorStats> stat = e.getAll(OperatorStats.class);
        List<OpTreeSignature> sig = e.getAll(OpTreeSignature.class);
        List<RelTreeSignature> rSig = e.getAll(RelTreeSignature.class);
        if (stat.size() > 1 || sig.size() > 1) {
            StringBuffer sb = new StringBuffer();
            sb.append(String.format("expected(stat-sig) 1-1, got {}-{} ;", stat.size(), sig.size()));
            for (OperatorStats s : stat) {
                sb.append(s);
                sb.append(";");
            }
            for (OpTreeSignature s : sig) {
                sb.append(s);
                sb.append(";");
            }
            LOG.debug(sb.toString());
        }
        if (stat.size() < 1 || sig.size() < 1) {
            continue;
        }
        if (e.getAll(OperatorStats.IncorrectRuntimeStatsMarker.class).size() > 0) {
            LOG.debug("Ignoring {}, marked with OperatorStats.IncorrectRuntimeStatsMarker", sig.get(0));
            continue;
        }
        if (e.getAll(OperatorStats.MayNotUseForRelNodes.class).size() > 0) {
            rSig = new ArrayList<>();
        }
        li.add(new PersistedRuntimeStats(first(sig), first(stat), first(rSig)));
    }
    return li.build();
}

Example 2 with EquivGroup

use of org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.EquivGroup in project hive by apache.

the class TestCounterMapping method testMappingJoinLookup.

@Test
@Ignore("needs HiveFilter mapping")
public void testMappingJoinLookup() throws ParseException, CommandProcessorException {
    IDriver driver = createDriver();
    PlanMapper pm0 = getMapperForQuery(driver, "select sum(tu.id_uv), sum(u)\n" + "from tu join tv on (tu.id_uv = tv.id_uv)\n" + "where u > 1 and v > 1");
    Iterator<EquivGroup> itG = pm0.iterateGroups();
    int checkedOperators = 0;
    while (itG.hasNext()) {
        EquivGroup g = itG.next();
        List<HiveFilter> hfs = g.getAll(HiveFilter.class);
        List<OperatorStats> oss = g.getAll(OperatorStats.class);
        List<FilterOperator> fos = g.getAll(FilterOperator.class);
        if (fos.size() > 0 && oss.size() > 0) {
            if (hfs.size() == 0) {
                fail("HiveFilter is not connected?");
            }
            OperatorStats os = oss.get(0);
            if (!(os.getOutputRecords() == 3 || os.getOutputRecords() == 6)) {
                fail("nonexpected number of records produced");
            }
            checkedOperators++;
        }
    }
    assertEquals(2, checkedOperators);
}

Example 3 with EquivGroup

use of org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.EquivGroup in project hive by apache.

the class TestReOptimization method testStatsAreSetInReopt.

@Test
public void testStatsAreSetInReopt() throws Exception {
    IDriver driver = createDriver("overlay,reoptimize");
    String query = "select assert_true_oom(${hiveconf:zzz} > sum(u*v))" + " from tu join tv on (tu.id_uv=tv.id_uv)" + " where u<10 and v>1";
    PlanMapper pm = getMapperForQuery(driver, query);
    Iterator<EquivGroup> itG = pm.iterateGroups();
    int checkedOperators = 0;
    while (itG.hasNext()) {
        EquivGroup g = itG.next();
        List<FilterOperator> fos = g.getAll(FilterOperator.class);
        List<OperatorStats> oss = g.getAll(OperatorStats.class);
        if (fos.size() > 0 && oss.size() > 0) {
            fos.sort(TestCounterMapping.OPERATOR_ID_COMPARATOR.reversed());
            FilterOperator fo = fos.get(0);
            OperatorStats os = oss.get(0);
            Statistics stats = fo.getStatistics();
            assertEquals(os.getOutputRecords(), stats.getNumRows());
            if (!(os.getOutputRecords() == 3 || os.getOutputRecords() == 6)) {
                fail("nonexpected number of records produced");
            }
            checkedOperators++;
        }
    }
    assertEquals(2, checkedOperators);
}

Example 4 with EquivGroup

use of org.apache.hadoop.hive.ql.plan.mapper.PlanMapper.EquivGroup in project hive by apache.

the class TestReOptimization method testReOptimizationCanSendBackStatsToCBO.

@Test
public void testReOptimizationCanSendBackStatsToCBO() throws Exception {
    IDriver driver = createDriver("overlay,reoptimize");
    // @formatter:off
    String query = "select assert_true_oom(${hiveconf:zzz} > sum(u*v*w)) from tu\n" + "        join tv on (tu.id_uv=tv.id_uv)\n" + "        join tw on (tu.id_uw=tw.id_uw)\n" + "        where w>9 and u>1 and v>3";
    // @formatter:on
    PlanMapper pm = getMapperForQuery(driver, query);
    Iterator<EquivGroup> itG = pm.iterateGroups();
    int checkedOperators = 0;
    while (itG.hasNext()) {
        EquivGroup g = itG.next();
        List<FilterOperator> fos = g.getAll(FilterOperator.class);
        List<OperatorStats> oss = g.getAll(OperatorStats.class);
        List<HiveFilter> hfs = g.getAll(HiveFilter.class);
        if (fos.size() > 0 && oss.size() > 0 && hfs.size() > 0) {
            fos.sort(TestCounterMapping.OPERATOR_ID_COMPARATOR.reversed());
            HiveFilter hf = hfs.get(0);
            FilterOperator fo = fos.get(0);
            OperatorStats os = oss.get(0);
            long cntFilter = RelMetadataQuery.instance().getRowCount(hf).longValue();
            if (fo.getStatistics() != null) {
            // in case the join order is changed the subTree-s are not matching anymore because an RS is present in the condition
            // assertEquals(os.getOutputRecords(), fo.getStatistics().getNumRows());
            }
            assertEquals(os.getOutputRecords(), cntFilter);
            checkedOperators++;
        }
    }
    assertEquals(3, checkedOperators);
}