Search in sources :

Example 21 with Expression

use of org.apache.hadoop.hive.ql.wm.Expression in project hive by apache.

the class TestTriggersTezSessionPoolManager method testTriggerHighShuffleBytes.

@Test(timeout = 60000)
public void testTriggerHighShuffleBytes() throws Exception {
    Expression expression = ExpressionFactory.fromString("SHUFFLE_BYTES > 100");
    Trigger trigger = new ExecutionTrigger("big_shuffle", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    List<String> cmds = new ArrayList<>();
    cmds.add("set hive.auto.convert.join=false");
    // to slow down the reducer so that SHUFFLE_BYTES publishing and validation can happen, adding sleep between
    // multiple reduce stages
    String query = "select count(distinct t.under_col), sleep(t.under_col, 10) from (select t1.under_col from " + tableName + " t1 " + "join " + tableName + " t2 on t1.under_col=t2.under_col order by sleep(t1.under_col, 0))" + " t group by t.under_col";
    runQueryWithTrigger(query, cmds, trigger + " violated");
}
Also used : ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Action(org.apache.hadoop.hive.ql.wm.Action) WMTrigger(org.apache.hadoop.hive.metastore.api.WMTrigger) ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Trigger(org.apache.hadoop.hive.ql.wm.Trigger) Expression(org.apache.hadoop.hive.ql.wm.Expression) ArrayList(java.util.ArrayList) Test(org.junit.Test)

Example 22 with Expression

use of org.apache.hadoop.hive.ql.wm.Expression in project hive by apache.

the class TestTriggersTezSessionPoolManager method testMultipleTriggers1.

@Test(timeout = 60000)
public void testMultipleTriggers1() throws Exception {
    Expression shuffleExpression = ExpressionFactory.fromString("HDFS_BYTES_READ > 1000000");
    Trigger shuffleTrigger = new ExecutionTrigger("big_shuffle", shuffleExpression, new Action(Action.Type.KILL_QUERY));
    Expression execTimeExpression = ExpressionFactory.fromString("EXECUTION_TIME > 1000");
    Trigger execTimeTrigger = new ExecutionTrigger("slow_query", execTimeExpression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(shuffleTrigger, execTimeTrigger));
    String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, null, execTimeTrigger + " violated");
}
Also used : ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Action(org.apache.hadoop.hive.ql.wm.Action) WMTrigger(org.apache.hadoop.hive.metastore.api.WMTrigger) ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Trigger(org.apache.hadoop.hive.ql.wm.Trigger) Expression(org.apache.hadoop.hive.ql.wm.Expression) Test(org.junit.Test)

Example 23 with Expression

use of org.apache.hadoop.hive.ql.wm.Expression in project hive by apache.

the class TestTriggersTezSessionPoolManager method testTriggerCustomCreatedDynamicPartitionsMultiInsert.

@Test(timeout = 60000)
public void testTriggerCustomCreatedDynamicPartitionsMultiInsert() throws Exception {
    List<String> cmds = getConfigs();
    cmds.add("drop table src2");
    cmds.add("drop table src3");
    cmds.add("create table src2 (key int) partitioned by (value string)");
    cmds.add("create table src3 (key int) partitioned by (value string)");
    String query = "from " + tableName + " insert overwrite table src2 partition (value) select * where under_col < 100 " + " insert overwrite table src3 partition (value) select * where under_col >= 100 and under_col < 200";
    Expression expression = ExpressionFactory.fromString("CREATED_DYNAMIC_PARTITIONS > 70");
    Trigger trigger = new ExecutionTrigger("high_partitions", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    runQueryWithTrigger(query, cmds, trigger + " violated");
}
Also used : ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Action(org.apache.hadoop.hive.ql.wm.Action) WMTrigger(org.apache.hadoop.hive.metastore.api.WMTrigger) ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Trigger(org.apache.hadoop.hive.ql.wm.Trigger) Expression(org.apache.hadoop.hive.ql.wm.Expression) Test(org.junit.Test)

Example 24 with Expression

use of org.apache.hadoop.hive.ql.wm.Expression in project hive by apache.

the class TestTriggersTezSessionPoolManager method testTriggerShortQueryElapsedTime.

@Test(timeout = 60000)
public void testTriggerShortQueryElapsedTime() throws Exception {
    Expression expression = ExpressionFactory.fromString("ELAPSED_TIME > 100");
    Trigger trigger = new ExecutionTrigger("slow_query", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    String query = "select sleep(t1.under_col, 500), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, null, trigger + " violated");
}
Also used : ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Action(org.apache.hadoop.hive.ql.wm.Action) WMTrigger(org.apache.hadoop.hive.metastore.api.WMTrigger) ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Trigger(org.apache.hadoop.hive.ql.wm.Trigger) Expression(org.apache.hadoop.hive.ql.wm.Expression) Test(org.junit.Test)

Example 25 with Expression

use of org.apache.hadoop.hive.ql.wm.Expression in project hive by apache.

the class TestTriggersTezSessionPoolManager method testTriggerSlowQueryElapsedTime.

@Test(timeout = 60000)
public void testTriggerSlowQueryElapsedTime() throws Exception {
    Expression expression = ExpressionFactory.fromString("ELAPSED_TIME > 20000");
    Trigger trigger = new ExecutionTrigger("slow_query", expression, new Action(Action.Type.KILL_QUERY));
    setupTriggers(Lists.newArrayList(trigger));
    String query = "select sleep(t1.under_col, 500), t1.value from " + tableName + " t1 join " + tableName + " t2 on t1.under_col>=t2.under_col";
    runQueryWithTrigger(query, null, trigger + " violated");
}
Also used : ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Action(org.apache.hadoop.hive.ql.wm.Action) WMTrigger(org.apache.hadoop.hive.metastore.api.WMTrigger) ExecutionTrigger(org.apache.hadoop.hive.ql.wm.ExecutionTrigger) Trigger(org.apache.hadoop.hive.ql.wm.Trigger) Expression(org.apache.hadoop.hive.ql.wm.Expression) Test(org.junit.Test)

Aggregations

Action (org.apache.hadoop.hive.ql.wm.Action)28 ExecutionTrigger (org.apache.hadoop.hive.ql.wm.ExecutionTrigger)28 Expression (org.apache.hadoop.hive.ql.wm.Expression)28 Trigger (org.apache.hadoop.hive.ql.wm.Trigger)28 Test (org.junit.Test)28 WMTrigger (org.apache.hadoop.hive.metastore.api.WMTrigger)24 ArrayList (java.util.ArrayList)5 WMPoolTrigger (org.apache.hadoop.hive.metastore.api.WMPoolTrigger)4