Examples with CommandProcessorException org.apache.hadoop.hive.ql.processors.CommandProcessorException used on opensource projects

Example 41 with CommandProcessorException

use of org.apache.hadoop.hive.ql.processors.CommandProcessorException in project hive by apache.

the class TestDbTxnManager2 method testDropTable.

private void testDropTable(boolean blocking) throws Exception {
    dropTable(new String[] { "tab_acid" });
    FileSystem fs = FileSystem.get(conf);
    HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED, !blocking);
    HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_LOCKS_PARTITION_THRESHOLD, 1);
    driver = Mockito.spy(driver);
    HiveConf.setBoolVar(driver2.getConf(), HiveConf.ConfVars.HIVE_ACID_CREATE_TABLE_USE_SUFFIX, !blocking);
    driver2 = Mockito.spy(driver2);
    driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + "stored as orc TBLPROPERTIES ('transactional'='true')");
    driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')");
    driver.compileAndRespond("select * from tab_acid");
    List<String> res = new ArrayList<>();
    driver.lockAndRespond();
    List<ShowLocksResponseElement> locks = getLocks();
    Assert.assertEquals("Unexpected lock count", 1, locks.size());
    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    DbTxnManager txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    swapTxnManager(txnMgr2);
    driver2.compileAndRespond("drop table if exists tab_acid");
    if (blocking) {
        txnMgr2.acquireLocks(driver2.getPlan(), ctx, null, false);
        locks = getLocks();
        ShowLocksResponseElement checkLock = checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_acid", null, locks);
        swapTxnManager(txnMgr);
        Mockito.doNothing().when(driver).lockAndRespond();
        driver.run();
        driver.getFetchTask().fetch(res);
        swapTxnManager(txnMgr2);
        FieldSetter.setField(txnMgr2, txnMgr2.getClass().getDeclaredField("numStatements"), 0);
        txnMgr2.getMS().unlock(checkLock.getLockid());
    }
    driver2.lockAndRespond();
    locks = getLocks();
    Assert.assertEquals("Unexpected lock count", blocking ? 1 : 2, locks.size());
    checkLock(blocking ? LockType.EXCLUSIVE : LockType.EXCL_WRITE, LockState.ACQUIRED, "default", "tab_acid", null, locks);
    Mockito.doNothing().when(driver2).lockAndRespond();
    driver2.run();
    if (!blocking) {
        swapTxnManager(txnMgr);
        Mockito.doNothing().when(driver).lockAndRespond();
        driver.run();
    }
    Mockito.reset(driver, driver2);
    FileStatus[] stat = fs.listStatus(new Path(getWarehouseDir()), t -> t.getName().matches("tab_acid" + (blocking ? "" : SOFT_DELETE_TABLE_PATTERN)));
    if ((blocking ? 0 : 1) != stat.length) {
        Assert.fail("Table data was " + (blocking ? "not" : "") + "removed from FS");
    }
    driver.getFetchTask().fetch(res);
    Assert.assertEquals("Expecting 2 rows and found " + res.size(), 2, res.size());
    try {
        driver.run("select * from tab_acid");
    } catch (CommandProcessorException ex) {
        Assert.assertEquals(ErrorMsg.INVALID_TABLE.getErrorCode(), ex.getResponseCode());
    }
    // re-create table with the same name
    driver.run("create table if not exists tab_acid (a int, b int) partitioned by (p string) " + "stored as orc TBLPROPERTIES ('transactional'='true')");
    driver.run("insert into tab_acid partition(p) (a,b,p) values(1,2,'foo'),(3,4,'bar')");
    driver.run("select * from tab_acid ");
    res = new ArrayList<>();
    driver.getFetchTask().fetch(res);
    Assert.assertEquals("Expecting 2 rows and found " + res.size(), 2, res.size());
}

Example 42 with CommandProcessorException

use of org.apache.hadoop.hive.ql.processors.CommandProcessorException in project hive by apache.

the class TestSparkInvalidFileFormat method readTextFileAsParquet.

@Test
public void readTextFileAsParquet() throws IOException, CommandProcessorException {
    HiveConf conf = new HiveConf();
    conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, SQLStdHiveAuthorizerFactory.class.getName());
    conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
    conf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "spark");
    conf.set("spark.master", "local");
    FileSystem fs = FileSystem.getLocal(conf);
    Path tmpDir = new Path("TestSparkInvalidFileFormat-tmp");
    File testFile = new File(conf.get("test.data.files"), "kv1.txt");
    SessionState.start(conf);
    IDriver driver = null;
    try {
        driver = DriverFactory.newDriver(conf);
        driver.run("CREATE TABLE test_table (key STRING, value STRING)");
        driver.run("LOAD DATA LOCAL INPATH '" + testFile + "' INTO TABLE test_table");
        driver.run("ALTER TABLE test_table SET FILEFORMAT parquet");
        try {
            driver.run("SELECT * FROM test_table ORDER BY key LIMIT 10");
            assert false;
        } catch (CommandProcessorException e) {
            Assert.assertTrue(e.getCause() instanceof HiveException);
            Assert.assertTrue(e.getCause().getMessage().contains("Spark job failed due to task failures"));
            Assert.assertTrue(e.getCause().getMessage().contains("kv1.txt is not a Parquet file. expected " + "magic number at tail [80, 65, 82, 49] but found [95, 57, 55, 10]"));
        }
    } finally {
        if (driver != null) {
            driver.run("DROP TABLE IF EXISTS test_table");
            driver.destroy();
        }
        if (fs.exists(tmpDir)) {
            fs.delete(tmpDir, true);
        }
    }
}

Example 43 with CommandProcessorException

use of org.apache.hadoop.hive.ql.processors.CommandProcessorException in project hive by apache.

the class GenericUDTFGetSplits method createPlanFragment.

private PlanFragment createPlanFragment(String query, ApplicationId splitsAppId) throws HiveException {
    HiveConf conf = new HiveConf(SessionState.get().getConf());
    HiveConf.setVar(conf, ConfVars.HIVEFETCHTASKCONVERSION, "none");
    HiveConf.setVar(conf, ConfVars.HIVEQUERYRESULTFILEFORMAT, PlanUtils.LLAP_OUTPUT_FORMAT_KEY);
    String originalMode = HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_MODE);
    HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, "llap");
    HiveConf.setBoolVar(conf, ConfVars.HIVE_TEZ_GENERATE_CONSISTENT_SPLITS, true);
    HiveConf.setBoolVar(conf, ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS, true);
    conf.setBoolean(TezSplitGrouper.TEZ_GROUPING_NODE_LOCAL_ONLY, true);
    // Tez/LLAP requires RPC query plan
    HiveConf.setBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN, true);
    HiveConf.setBoolVar(conf, ConfVars.HIVE_QUERY_RESULTS_CACHE_ENABLED, false);
    if (schemaSplitOnly) {
        // Schema only
        try {
            List<FieldSchema> fieldSchemas = ParseUtils.parseQueryAndGetSchema(conf, query);
            Schema schema = new Schema(convertSchema(fieldSchemas));
            return new PlanFragment(null, schema, null);
        } catch (ParseException e) {
            throw new HiveException(e);
        }
    }
    try {
        jc = DagUtils.getInstance().createConfiguration(conf);
    } catch (IOException e) {
        throw new HiveException(e);
    }
    // Instantiate Driver to compile the query passed in.
    // This UDF is running as part of an existing query, which may already be using the
    // SessionState TxnManager. If this new Driver also tries to use the same TxnManager
    // then this may mess up the existing state of the TxnManager.
    // So initialize the new Driver with a new TxnManager so that it does not use the
    // Session TxnManager that is already in use.
    HiveTxnManager txnManager = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
    Driver driver = new Driver(new QueryState.Builder().withHiveConf(conf).nonIsolated().build(), null, txnManager);
    DriverCleanup driverCleanup = new DriverCleanup(driver, txnManager, splitsAppId.toString());
    boolean needsCleanup = true;
    try {
        try {
            driver.compileAndRespond(query, false);
        } catch (CommandProcessorException e) {
            throw new HiveException("Failed to compile query", e);
        }
        QueryPlan plan = driver.getPlan();
        limitQuery = plan.getQueryProperties().getOuterQueryLimit() != -1;
        List<Task<?>> roots = plan.getRootTasks();
        Schema schema = convertSchema(plan.getResultSchema());
        boolean fetchTask = plan.getFetchTask() != null;
        TezWork tezWork;
        if (roots == null || roots.size() != 1 || !(roots.get(0) instanceof TezTask)) {
            // fetch task query
            if (fetchTask) {
                tezWork = null;
            } else {
                throw new HiveException("Was expecting a single TezTask or FetchTask.");
            }
        } else {
            tezWork = ((TezTask) roots.get(0)).getWork();
        }
        // return more than "n" rows. Therefore, a limit query needs to be materialized.
        if (tezWork == null || tezWork.getAllWork().size() != 1 || limitQuery) {
            String tableName = "table_" + UUID.randomUUID().toString().replaceAll("-", "");
            String storageFormatString = getTempTableStorageFormatString(conf);
            String ctas = "create temporary table " + tableName + " " + storageFormatString + " as " + query;
            LOG.info("Materializing the query for LLAPIF; CTAS: " + ctas);
            driver.releaseLocksAndCommitOrRollback(false);
            driver.releaseResources();
            HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, originalMode);
            try {
                driver.run(ctas);
            } catch (CommandProcessorException e) {
                throw new HiveException("Failed to create temp table [" + tableName + "]", e);
            }
            HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, "llap");
            query = "select * from " + tableName;
            try {
                driver.compileAndRespond(query, true);
            } catch (CommandProcessorException e) {
                throw new HiveException("Failed to select from table [" + tableName + "]", e);
            }
            plan = driver.getPlan();
            roots = plan.getRootTasks();
            schema = convertSchema(plan.getResultSchema());
            if (roots == null || roots.size() != 1 || !(roots.get(0) instanceof TezTask)) {
                throw new HiveException("Was expecting a single TezTask.");
            }
            tezWork = ((TezTask) roots.get(0)).getWork();
        } else {
            // The read will have READ_COMMITTED level semantics.
            try {
                driver.lockAndRespond();
            } catch (CommandProcessorException cpr1) {
                throw new HiveException("Failed to acquire locks", cpr1);
            }
            // Attach the resources to the session cleanup.
            SessionState.get().addCleanupItem(driverCleanup);
            needsCleanup = false;
        }
        // Pass the ValidTxnList and ValidTxnWriteIdList snapshot configurations corresponding to the input query
        HiveConf driverConf = driver.getConf();
        String validTxnString = driverConf.get(ValidTxnList.VALID_TXNS_KEY);
        if (validTxnString != null) {
            jc.set(ValidTxnList.VALID_TXNS_KEY, validTxnString);
        }
        String validWriteIdString = driverConf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY);
        if (validWriteIdString != null) {
            assert validTxnString != null;
            jc.set(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY, validWriteIdString);
        }
        return new PlanFragment(tezWork, schema, jc);
    } finally {
        if (needsCleanup) {
            if (driverCleanup != null) {
                try {
                    driverCleanup.close();
                } catch (IOException err) {
                    throw new HiveException(err);
                }
            } else if (driver != null) {
                driver.close();
                driver.destroy();
            }
        }
    }
}

Example 44 with CommandProcessorException

use of org.apache.hadoop.hive.ql.processors.CommandProcessorException in project hive by apache.

the class TestQueryHooks method testQueryLifeTimeWithCompileError.

@Test
public void testQueryLifeTimeWithCompileError() throws Exception {
    String query = "select * from foo";
    ArgumentMatcher<QueryLifeTimeHookContext> argMatcher = new QueryLifeTimeHookContextMatcher(query);
    QueryLifeTimeHook mockHook = mock(QueryLifeTimeHook.class);
    Driver driver = createDriver();
    driver.getHookRunner().addLifeTimeHook(mockHook);
    try {
        driver.run(query);
        Assert.fail("Expected compilation to fail");
    } catch (CommandProcessorException e) {
    // we expect to get here
    }
    verify(mockHook).beforeCompile(argThat(argMatcher));
    verify(mockHook).afterCompile(argThat(argMatcher), eq(true));
    verify(mockHook, never()).beforeExecution(any());
    verify(mockHook, never()).afterExecution(any(), anyBoolean());
}

Example 45 with CommandProcessorException

use of org.apache.hadoop.hive.ql.processors.CommandProcessorException in project hive by apache.

the class TestMSCKRepairOnAcid method testAddPartitionHighWriteIdException.

/**
 * A new partition copied under a table containing only deltas, but the table already contains allocated writes,
 * and the restored partition has higher writeId.
 * @throws Exception ex
 */
@Test
public void testAddPartitionHighWriteIdException() throws Exception {
    runStatementOnDriver("drop table if exists " + acidTblPartMsck);
    // Insert few rows
    runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition(p) values(1,1,'p1'),(2,2,'p1'),(3,3,'p1')");
    runStatementOnDriver("insert into " + Table.ACIDTBLPART + " partition(p) values(1,2,'p2'),(2,3,'p2'),(3,4,'p2')");
    List<String> r = runStatementOnDriver("select a, b from " + Table.ACIDTBLPART + " order by a, b");
    int[][] expected = { { 1, 1 }, { 1, 2 }, { 2, 2 }, { 2, 3 }, { 3, 3 }, { 3, 4 } };
    Assert.assertEquals(stringifyValues(expected), r);
    // Create target table
    runStatementOnDriver("create table " + acidTblPartMsck + " (a int, b int) partitioned by (p string) clustered by (a) into 2 buckets" + " stored as orc TBLPROPERTIES ('transactional'='true')");
    // Insert data in p1 so we allocate writeId in the msck table
    runStatementOnDriver("insert into " + acidTblPartMsck + " partition(p) values(1,1,'p1'),(2,2,'p1'),(3,3,'p1')");
    // copy files on fs
    FileSystem fs = FileSystem.get(hiveConf);
    FileUtil.copy(fs, new Path(getWarehouseDir() + "/" + Table.ACIDTBLPART.toString().toLowerCase() + "/p=p2"), fs, new Path(getWarehouseDir(), acidTblPartMsck), false, false, hiveConf);
    // One partition written, one copied
    FileStatus[] fileStatuses = fs.listStatus(new Path(getWarehouseDir(), acidTblPartMsck));
    Assert.assertEquals(2, fileStatuses.length);
    // call msk repair, it should fail, since it will find a delta folder with writeId 2
    // that is higher than the allocated max in the table
    CommandProcessorException e = runStatementOnDriverNegative("msck repair table " + acidTblPartMsck);
    Assert.assertEquals(-1, e.getErrorCode());
    runStatementOnDriver("drop table if exists " + acidTblPartMsck);
}