Examples with ExecutionTrigger org.apache.hadoop.hive.ql.wm.ExecutionTrigger used on opensource projects

Example 11 with ExecutionTrigger

use of org.apache.hadoop.hive.ql.wm.ExecutionTrigger in project hive by apache.

the class TestTriggersTezSessionPoolManager method testTriggerVertexRawInputSplitsNoKill.

@Test(timeout = 60000)
public void testTriggerVertexRawInputSplitsNoKill() throws Exception {
    // Map 1 - 55 splits
    // Map 3 - 55 splits
    Expression expression = ExpressionFactory.fromString("VERTEX_RAW_INPUT_SPLITS > 100");
    Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, getConfigs(), null);
}

Example 12 with ExecutionTrigger

use of org.apache.hadoop.hive.ql.wm.ExecutionTrigger in project hive by apache.

the class TestTriggersTezSessionPoolManager method testTriggerVertexRawInputSplitsKill.

@Test(timeout = 60000)
public void testTriggerVertexRawInputSplitsKill() throws Exception {
    // Map 1 - 55 splits
    // Map 3 - 55 splits
    Expression expression = ExpressionFactory.fromString("VERTEX_RAW_INPUT_SPLITS > 50");
    Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, getConfigs(), "Query was cancelled");
}

Example 13 with ExecutionTrigger

use of org.apache.hadoop.hive.ql.wm.ExecutionTrigger in project hive by apache.

the class TestTriggersTezSessionPoolManager method testMultipleTriggers2.

@Test(timeout = 60000)
public void testMultipleTriggers2() throws Exception {
    Expression shuffleExpression = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
    Trigger shuffleTrigger = new ExecutionTrigger("big_shuffle", shuffleExpression, new Action(Action.Type.KILL_QUERY));
    Expression execTimeExpression = ExpressionFactory.fromString("EXECUTION_TIME > 100000");
    Trigger execTimeTrigger = new ExecutionTrigger("slow_query", execTimeExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(shuffleTrigger, execTimeTrigger));
    String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, null, shuffleTrigger + " violated");
}

Example 14 with ExecutionTrigger

use of org.apache.hadoop.hive.ql.wm.ExecutionTrigger in project hive by apache.

the class TestTriggersMoveWorkloadManager method testTriggerMoveEscapeKill.

@Test(timeout = 60000)
public void testTriggerMoveEscapeKill() throws Exception {
    Expression moveExpression = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
    Expression killExpression = ExpressionFactory.fromString("EXECUTION_TIME > 5000");
    Trigger moveTrigger = new ExecutionTrigger("move_big_read", moveExpression, new Action(Action.Type.MOVE_TO_POOL, "ETL"));
    Trigger killTrigger = new ExecutionTrigger("slow_query_kill", killExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(moveTrigger, killTrigger), Lists.newArrayList());
    String query = "select sleep(t1.under_col, 1), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col==t2.under_col";
    List<String> setCmds = new ArrayList<>();
    setCmds.add("set hive.tez.session.events.print.summary=json");
    setCmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    List<String> errCaptureExpect = new ArrayList<>();
    errCaptureExpect.add("Workload Manager Events Summary");
    errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00");
    errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00");
    errCaptureExpect.add("\"eventType\" : \"GET\"");
    errCaptureExpect.add("\"eventType\" : \"MOVE\"");
    errCaptureExpect.add("\"eventType\" : \"RETURN\"");
    errCaptureExpect.add("\"name\" : \"move_big_read\"");
    errCaptureExpect.add("\"name\" : \"slow_query_kill\"");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger + " violated");
    errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"EXECUTION_TIME\" ]");
    runQueryWithTrigger(query, setCmds, null, errCaptureExpect);
}

Example 15 with ExecutionTrigger

use of org.apache.hadoop.hive.ql.wm.ExecutionTrigger in project hive by apache.

the class TestTriggersMoveWorkloadManager method testTriggerMoveBackKill.

@Test(timeout = 60000)
public void testTriggerMoveBackKill() throws Exception {
    Expression moveExpression1 = ExpressionFactory.fromString("HDFS_BYTES_READ > 100");
    Expression moveExpression2 = ExpressionFactory.fromString("SHUFFLE_BYTES > 200");
    Expression killExpression = ExpressionFactory.fromString("EXECUTION_TIME > 2000");
    Trigger moveTrigger1 = new ExecutionTrigger("move_big_read", moveExpression1, new Action(Action.Type.MOVE_TO_POOL, "ETL"));
    Trigger moveTrigger2 = new ExecutionTrigger("move_high", moveExpression2, new Action(Action.Type.MOVE_TO_POOL, "BI"));
    Trigger killTrigger = new ExecutionTrigger("slow_query_kill", killExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(moveTrigger1, killTrigger), Lists.newArrayList(moveTrigger2));
    String query = "select sleep(t1.under_col, 1), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    List<String> setCmds = new ArrayList<>();
    setCmds.add("set hive.tez.session.events.print.summary=json");
    setCmds.add("set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    setCmds.add("set hive.exec.failure.hooks=org.apache.hadoop.hive.ql.hooks.PostExecWMEventsSummaryPrinter");
    List<String> errCaptureExpect = new ArrayList<>();
    errCaptureExpect.add("Workload Manager Events Summary");
    errCaptureExpect.add("Event: GET Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: MOVE Pool: ETL Cluster %: 20.00");
    errCaptureExpect.add("Event: MOVE Pool: BI Cluster %: 80.00");
    errCaptureExpect.add("Event: KILL Pool: null Cluster %: 0.00");
    errCaptureExpect.add("Event: RETURN Pool: null Cluster %: 0.00");
    errCaptureExpect.add("\"eventType\" : \"GET\"");
    errCaptureExpect.add("\"eventType\" : \"MOVE\"");
    errCaptureExpect.add("\"eventType\" : \"MOVE\"");
    errCaptureExpect.add("\"eventType\" : \"KILL\"");
    errCaptureExpect.add("\"eventType\" : \"RETURN\"");
    errCaptureExpect.add("\"name\" : \"move_big_read\"");
    errCaptureExpect.add("\"name\" : \"slow_query_kill\"");
    errCaptureExpect.add("\"name\" : \"move_high\"");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger1 + " violated");
    // violation in ETL queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + moveTrigger2 + " violated");
    // violation in BI queue
    errCaptureExpect.add("\"violationMsg\" : \"Trigger " + killTrigger + " violated");
    errCaptureExpect.add("\"subscribedCounters\" : [ \"HDFS_BYTES_READ\", \"EXECUTION_TIME\", \"SHUFFLE_BYTES\" ]");
    runQueryWithTrigger(query, setCmds, killTrigger + " violated", errCaptureExpect);
}