Examples with Trigger org.apache.hadoop.hive.ql.wm.Trigger used on opensource projects

Example 16 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersMoveWorkloadManager method testTriggerMoveEscapeKill.

@Test(timeout = 60000)
public void testTriggerMoveEscapeKill() throws Exception {
    Expression moveExpression = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
    Expression killExpression = ExpressionFactory.fromString("EXECUTION_TIME > 5000");
    Trigger moveTrigger = new ExecutionTrigger("move_big_read", moveExpression, new Action(Action.Type.MOVE_TO_POOL, "ETL"));
    Trigger killTrigger = new ExecutionTrigger("slow_query_kill", killExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(moveTrigger, killTrigger), Lists.newArrayList());
    String query = "select sleep(t1.under_col, 1), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col==t2.under_col";
    List<String> setCmds = new ArrayList<>();
    setCmds.add("set hive.tez.session.events.print.summary=json");
    setCmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    List<String> errCaptureExpect = new ArrayList<>();
    errCaptureExpect.add("Workload Manager Events Summary");
    errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00");
    errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00");
    errCaptureExpect.add("\"eventType\" : \"GET\"");
    errCaptureExpect.add("\"eventType\" : \"MOVE\"");
    errCaptureExpect.add("\"eventType\" : \"RETURN\"");
    errCaptureExpect.add("\"name\" : \"move_big_read\"");
    errCaptureExpect.add("\"name\" : \"slow_query_kill\"");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger + " violated");
    errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"EXECUTION_TIME\" ]");
    runQueryWithTrigger(query, setCmds, null, errCaptureExpect);
}

Example 17 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersMoveWorkloadManager method testTriggerMoveBackKill.

@Test(timeout = 60000)
public void testTriggerMoveBackKill() throws Exception {
    Expression moveExpression1 = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
    Expression moveExpression2 = ExpressionFactory.fromString("SHUFFLE_BYTES > 200");
    Expression killExpression = ExpressionFactory.fromString("EXECUTION_TIME > 2000");
    Trigger moveTrigger1 = new ExecutionTrigger("move_big_read", moveExpression1, new Action(Action.Type.MOVE_TO_POOL, "ETL"));
    Trigger moveTrigger2 = new ExecutionTrigger("move_high", moveExpression2, new Action(Action.Type.MOVE_TO_POOL, "BI"));
    Trigger killTrigger = new ExecutionTrigger("slow_query_kill", killExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(moveTrigger1, killTrigger), Lists.newArrayList(moveTrigger2));
    String query = "select sleep(t1.under_col, 1), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    List<String> setCmds = new ArrayList<>();
    setCmds.add("set hive.tez.session.events.print.summary=json");
    setCmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    List<String> errCaptureExpect = new ArrayList<>();
    errCaptureExpect.add("Workload Manager Events Summary");
    errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00");
    errCaptureExpect.add("Event: MOVE Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00");
    errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00");
    errCaptureExpect.add("\"eventType\" : \"GET\"");
    errCaptureExpect.add("\"eventType\" : \"MOVE\"");
    errCaptureExpect.add("\"eventType\" : \"MOVE\"");
    errCaptureExpect.add("\"eventType\" : \"KILL\"");
    errCaptureExpect.add("\"eventType\" : \"RETURN\"");
    errCaptureExpect.add("\"name\" : \"move_big_read\"");
    errCaptureExpect.add("\"name\" : \"slow_query_kill\"");
    errCaptureExpect.add("\"name\" : \"move_high\"");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger1 + " violated");
    // violation in ETL queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger2 + " violated");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated");
    errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"EXECUTION_TIME\", \"SHUFFLE_BYTES\" ]");
    runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect);
}

Example 18 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersMoveWorkloadManager method testTriggerMoveAndKill.

@Test(timeout = 60000)
public void testTriggerMoveAndKill() throws Exception {
    Expression moveExpression = ExpressionFactory.fromString("EXECUTION_TIME > 1sec");
    Expression killExpression = ExpressionFactory.fromString("EXECUTION_TIME > 5000ms");
    Trigger moveTrigger = new ExecutionTrigger("slow_query_move", moveExpression, new Action(Action.Type.MOVE_TO_POOL, "ETL"));
    Trigger killTrigger = new ExecutionTrigger("slow_query_kill", killExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(moveTrigger), Lists.newArrayList(killTrigger));
    String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    List<String> setCmds = new ArrayList<>();
    setCmds.add("set hive.tez.session.events.print.summary=json");
    setCmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    List<String> errCaptureExpect = new ArrayList<>();
    errCaptureExpect.add("Workload Manager Events Summary");
    errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00");
    errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00");
    errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00");
    errCaptureExpect.add("\"eventType\" : \"GET\"");
    errCaptureExpect.add("\"eventType\" : \"MOVE\"");
    errCaptureExpect.add("\"eventType\" : \"KILL\"");
    errCaptureExpect.add("\"eventType\" : \"RETURN\"");
    errCaptureExpect.add("\"name\" : \"slow_query_move\"");
    errCaptureExpect.add("\"name\" : \"slow_query_kill\"");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger + " violated");
    // violation in ETL queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated");
    errCaptureExpect.add("\"subscribedCounters\" : [ \"EXECUTION_TIME\" ]");
    runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect);
}

Example 19 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersMoveWorkloadManager method setupTriggers.

private void setupTriggers(final List<Trigger> biTriggers, final List<Trigger> etlTriggers) throws Exception {
    WorkloadManager wm = WorkloadManager.getInstance();
    WMPool biPool = pool("BI", 1, 0.8f);
    WMPool etlPool = pool("ETL", 1, 0.2f);
    WMFullResourcePlan plan = new WMFullResourcePlan(plan(), Lists.newArrayList(biPool, etlPool));
    plan.getPlan().setDefaultPoolPath("BI");
    for (Trigger trigger : biTriggers) {
        plan.addToTriggers(wmTriggerFromTrigger(trigger));
        plan.addToPoolTriggers(new WMPoolTrigger("BI", trigger.getName()));
    }
    for (Trigger trigger : etlTriggers) {
        plan.addToTriggers(wmTriggerFromTrigger(trigger));
        plan.addToPoolTriggers(new WMPoolTrigger("ETL", trigger.getName()));
    }
    wm.updateResourcePlanAsync(plan).get(10, TimeUnit.SECONDS);
}

Example 20 with Trigger

use of org.apache.hadoop.hive.ql.wm.Trigger in project hive by apache.

the class TestTriggersMoveWorkloadManager method testTriggerMoveConflictKill.

// TODO: disabling this test as tez publishes counters only after task completion which will cause write side counters
// to be not validated correctly (DAG will be completed before validation)
// @Test(timeout = 60000)
// public void testTriggerMoveKill() throws Exception {
// Expression moveExpression1 = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
// Expression moveExpression2 = ExpressionFactory.fromString("HDFS_BYTES_WRITTEN > 200");
// Trigger moveTrigger1 = new ExecutionTrigger("move_big_read", moveExpression1,
// new Action(Action.Type.MOVE_TO_POOL, "ETL"));
// Trigger killTrigger = new ExecutionTrigger("big_write_kill", moveExpression2,
// new Action(Action.Type.KILL_QUERY));
// setupTriggers(Lists.newArrayList(moveTrigger1), Lists.newArrayList(killTrigger));
// String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName +
// " t2 on t1.under_col>=t2.under_col order by t1.under_col, t1.value";
// List<String> setCmds = new ArrayList<>();
// setCmds.add("set hive.tez.session.events.print.summary=json");
// setCmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
// setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
// List<String> errCaptureExpect = new ArrayList<>();
// errCaptureExpect.add("Workload Manager Events Summary");
// errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00");
// errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00");
// errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00");
// errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00");
// errCaptureExpect.add("\"eventType\" : \"GET\"");
// errCaptureExpect.add("\"eventType\" : \"MOVE\"");
// errCaptureExpect.add("\"eventType\" : \"KILL\"");
// errCaptureExpect.add("\"eventType\" : \"RETURN\"");
// errCaptureExpect.add("\"name\" : \"move_big_read\"");
// errCaptureExpect.add("\"name\" : \"big_write_kill\"");
// // violation in BI queue
// errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger1 + " violated");
// // violation in ETL queue
// errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated");
// errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"HDFS_BYTES_WRITTEN\" ]");
// runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect);
// }
@Test(timeout = 60000)
public void testTriggerMoveConflictKill() throws Exception {
    Expression moveExpression = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
    Expression killExpression = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
    Trigger moveTrigger = new ExecutionTrigger("move_big_read", moveExpression, new Action(Action.Type.MOVE_TO_POOL, "ETL"));
    Trigger killTrigger = new ExecutionTrigger("kill_big_read", killExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(moveTrigger, killTrigger), Lists.newArrayList());
    String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    List<String> setCmds = new ArrayList<>();
    setCmds.add("set hive.tez.session.events.print.summary=json");
    setCmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    List<String> errCaptureExpect = new ArrayList<>();
    errCaptureExpect.add("Workload Manager Events Summary");
    errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00");
    errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00");
    errCaptureExpect.add("\"eventType\" : \"GET\"");
    errCaptureExpect.add("\"eventType\" : \"KILL\"");
    errCaptureExpect.add("\"eventType\" : \"RETURN\"");
    errCaptureExpect.add("\"name\" : \"move_big_read\"");
    errCaptureExpect.add("\"name\" : \"kill_big_read\"");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated");
    errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\" ]");
    runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect);
}