Examples with HCatException org.apache.hive.hcatalog.common.HCatException used on opensource projects

Example 21 with HCatException

use of org.apache.hive.hcatalog.common.HCatException in project hive by apache.

the class HCatBaseStorer method convertPigSchemaToHCatSchema.

/** Constructs HCatSchema from pigSchema. Passed tableSchema is the existing
   * schema of the table in metastore.
   */
protected HCatSchema convertPigSchemaToHCatSchema(Schema pigSchema, HCatSchema tableSchema) throws FrontendException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("convertPigSchemaToHCatSchema(pigSchema,tblSchema)=(" + pigSchema + "," + tableSchema + ")");
    }
    List<HCatFieldSchema> fieldSchemas = new ArrayList<HCatFieldSchema>(pigSchema.size());
    for (FieldSchema fSchema : pigSchema.getFields()) {
        try {
            HCatFieldSchema hcatFieldSchema = getColFromSchema(fSchema.alias, tableSchema);
            //if writing to a partitioned table, then pigSchema will have more columns than tableSchema
            //partition columns are not part of tableSchema... e.g. TestHCatStorer#testPartColsInData()
            //        HCatUtil.assertNotNull(hcatFieldSchema, "Nothing matching '" + fSchema.alias + "' found " +
            //                "in target table schema", LOG);
            fieldSchemas.add(getHCatFSFromPigFS(fSchema, hcatFieldSchema, pigSchema, tableSchema));
        } catch (HCatException he) {
            throw new FrontendException(he.getMessage(), PigHCatUtil.PIG_EXCEPTION_CODE, he);
        }
    }
    HCatSchema s = new HCatSchema(fieldSchemas);
    LOG.debug("convertPigSchemaToHCatSchema(computed)=(" + s + ")");
    return s;
}

Example 22 with HCatException

use of org.apache.hive.hcatalog.common.HCatException in project hive by apache.

the class TestHCatDynamicPartitioned method _testHCatDynamicPartitionMaxPartitions.

//TODO 1.0 miniCluster is slow this test times out, make it work
// renaming test to make test framework skip it
public void _testHCatDynamicPartitionMaxPartitions() throws Exception {
    HiveConf hc = new HiveConf(this.getClass());
    int maxParts = hiveConf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTS);
    LOG.info("Max partitions allowed = {}", maxParts);
    IOException exc = null;
    try {
        generateWriteRecords(maxParts + 5, maxParts + 2, 10);
        runMRCreate(null, dataColumns, writeRecords, maxParts + 5, false);
    } catch (IOException e) {
        exc = e;
    }
    if (HCatConstants.HCAT_IS_DYNAMIC_MAX_PTN_CHECK_ENABLED) {
        assertTrue(exc != null);
        assertTrue(exc instanceof HCatException);
        assertEquals(ErrorType.ERROR_TOO_MANY_DYNAMIC_PTNS, ((HCatException) exc).getErrorType());
    } else {
        assertTrue(exc == null);
        runMRRead(maxParts + 5);
    }
}

Example 23 with HCatException

use of org.apache.hive.hcatalog.common.HCatException in project hive by apache.

the class TestHCatNonPartitioned method testHCatNonPartitionedTable.

@Test
public void testHCatNonPartitionedTable() throws Exception {
    Map<String, String> partitionMap = new HashMap<String, String>();
    runMRCreate(null, partitionColumns, writeRecords, 10, true);
    //Test for duplicate publish -- this will either fail on job creation time
    // and throw an exception, or will fail at runtime, and fail the job.
    IOException exc = null;
    try {
        Job j = runMRCreate(null, partitionColumns, writeRecords, 20, true);
        assertEquals(!isTableImmutable(), j.isSuccessful());
    } catch (IOException e) {
        exc = e;
        assertTrue(exc instanceof HCatException);
        assertEquals(ErrorType.ERROR_NON_EMPTY_TABLE, ((HCatException) exc).getErrorType());
    }
    if (!isTableImmutable()) {
        assertNull(exc);
    }
    //Test for publish with invalid partition key name
    exc = null;
    partitionMap.clear();
    partitionMap.put("px", "p1value2");
    try {
        Job j = runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true);
        assertFalse(j.isSuccessful());
    } catch (IOException e) {
        exc = e;
        assertTrue(exc != null);
        assertTrue(exc instanceof HCatException);
        assertEquals(ErrorType.ERROR_INVALID_PARTITION_VALUES, ((HCatException) exc).getErrorType());
    }
    //Read should get 10 rows if immutable, 30 if mutable
    if (isTableImmutable()) {
        runMRRead(10);
    } else {
        runMRRead(30);
    }
    hiveReadTest();
}

Example 24 with HCatException

use of org.apache.hive.hcatalog.common.HCatException in project hive by apache.

the class TestHCatPartitioned method tableSchemaTest.

//test that new columns gets added to table schema
private void tableSchemaTest() throws Exception {
    HCatSchema tableSchema = getTableSchema();
    assertEquals(4, tableSchema.getFields().size());
    //Update partition schema to have 3 fields
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", serdeConstants.STRING_TYPE_NAME, "")));
    writeRecords = new ArrayList<HCatRecord>();
    for (int i = 0; i < 20; i++) {
        List<Object> objList = new ArrayList<Object>();
        objList.add(i);
        objList.add("strvalue" + i);
        objList.add("str2value" + i);
        writeRecords.add(new DefaultHCatRecord(objList));
    }
    Map<String, String> partitionMap = new HashMap<String, String>();
    partitionMap.put("part1", "p1value5");
    partitionMap.put("part0", "505");
    runMRCreate(partitionMap, partitionColumns, writeRecords, 10, true);
    tableSchema = getTableSchema();
    //assert that c3 has got added to table schema
    assertEquals(5, tableSchema.getFields().size());
    assertEquals("c1", tableSchema.getFields().get(0).getName());
    assertEquals("c2", tableSchema.getFields().get(1).getName());
    assertEquals("c3", tableSchema.getFields().get(2).getName());
    assertEquals("part1", tableSchema.getFields().get(3).getName());
    assertEquals("part0", tableSchema.getFields().get(4).getName());
    //Test that changing column data type fails
    partitionMap.clear();
    partitionMap.put("part1", "p1value6");
    partitionMap.put("part0", "506");
    partitionColumns = new ArrayList<HCatFieldSchema>();
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", serdeConstants.INT_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c2", serdeConstants.INT_TYPE_NAME, "")));
    IOException exc = null;
    try {
        runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true);
    } catch (IOException e) {
        exc = e;
    }
    assertTrue(exc != null);
    assertTrue(exc instanceof HCatException);
    assertEquals(ErrorType.ERROR_SCHEMA_TYPE_MISMATCH, ((HCatException) exc).getErrorType());
    //Test that partition key is not allowed in data
    partitionColumns = new ArrayList<HCatFieldSchema>();
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", serdeConstants.INT_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c2", serdeConstants.STRING_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", serdeConstants.STRING_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("part1", serdeConstants.STRING_TYPE_NAME, "")));
    List<HCatRecord> recordsContainingPartitionCols = new ArrayList<HCatRecord>(20);
    for (int i = 0; i < 20; i++) {
        List<Object> objList = new ArrayList<Object>();
        objList.add(i);
        objList.add("c2value" + i);
        objList.add("c3value" + i);
        objList.add("p1value6");
        recordsContainingPartitionCols.add(new DefaultHCatRecord(objList));
    }
    exc = null;
    try {
        runMRCreate(partitionMap, partitionColumns, recordsContainingPartitionCols, 20, true);
    } catch (IOException e) {
        exc = e;
    }
    List<HCatRecord> records = runMRRead(20, "part1 = \"p1value6\"");
    assertEquals(20, records.size());
    records = runMRRead(20, "part0 = \"506\"");
    assertEquals(20, records.size());
    Integer i = 0;
    for (HCatRecord rec : records) {
        assertEquals(5, rec.size());
        assertEquals(rec.get(0), i);
        assertEquals(rec.get(1), "c2value" + i);
        assertEquals(rec.get(2), "c3value" + i);
        assertEquals(rec.get(3), "p1value6");
        assertEquals(rec.get(4), 506);
        i++;
    }
}

Example 25 with HCatException

use of org.apache.hive.hcatalog.common.HCatException in project hive by apache.

the class TestHCatDynamicPartitioned method runHCatDynamicPartitionedTable.

protected void runHCatDynamicPartitionedTable(boolean asSingleMapTask, String customDynamicPathPattern) throws Exception {
    generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
    runMRCreate(null, dataColumns, writeRecords, NUM_RECORDS, true, asSingleMapTask, customDynamicPathPattern);
    runMRRead(NUM_RECORDS);
    //Read with partition filter
    runMRRead(4, "p1 = \"0\"");
    runMRRead(8, "p1 = \"1\" or p1 = \"3\"");
    runMRRead(4, "p1 = \"4\"");
    // read from hive to test
    String query = "select * from " + tableName;
    int retCode = driver.run(query).getResponseCode();
    if (retCode != 0) {
        throw new Exception("Error " + retCode + " running query " + query);
    }
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
    assertEquals(NUM_RECORDS, res.size());
    //Test for duplicate publish
    IOException exc = null;
    try {
        generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
        Job job = runMRCreate(null, dataColumns, writeRecords, NUM_RECORDS, false, true, customDynamicPathPattern);
        if (HCatUtil.isHadoop23()) {
            Assert.assertTrue(job.isSuccessful() == false);
        }
    } catch (IOException e) {
        exc = e;
    }
    if (!HCatUtil.isHadoop23()) {
        assertTrue(exc != null);
        assertTrue(exc instanceof HCatException);
        assertTrue("Got exception of type [" + ((HCatException) exc).getErrorType().toString() + "] Expected ERROR_PUBLISHING_PARTITION or ERROR_MOVE_FAILED " + "or ERROR_DUPLICATE_PARTITION", (ErrorType.ERROR_PUBLISHING_PARTITION == ((HCatException) exc).getErrorType()) || (ErrorType.ERROR_MOVE_FAILED == ((HCatException) exc).getErrorType()) || (ErrorType.ERROR_DUPLICATE_PARTITION == ((HCatException) exc).getErrorType()));
    }
    query = "show partitions " + tableName;
    retCode = driver.run(query).getResponseCode();
    if (retCode != 0) {
        throw new Exception("Error " + retCode + " running query " + query);
    }
    res = new ArrayList<String>();
    driver.getResults(res);
    assertEquals(NUM_PARTITIONS, res.size());
    query = "select * from " + tableName;
    retCode = driver.run(query).getResponseCode();
    if (retCode != 0) {
        throw new Exception("Error " + retCode + " running query " + query);
    }
    res = new ArrayList<String>();
    driver.getResults(res);
    assertEquals(NUM_RECORDS, res.size());
    query = "select count(*) from " + tableName;
    retCode = driver.run(query).getResponseCode();
    if (retCode != 0) {
        throw new Exception("Error " + retCode + " running query " + query);
    }
    res = new ArrayList<String>();
    driver.getResults(res);
    assertEquals(1, res.size());
    assertEquals("20", res.get(0));
    query = "select count(*) from " + tableName + " where p1=1";
    retCode = driver.run(query).getResponseCode();
    if (retCode != 0) {
        throw new Exception("Error " + retCode + " running query " + query);
    }
    res = new ArrayList<String>();
    driver.getResults(res);
    assertEquals(1, res.size());
    assertEquals("4", res.get(0));
}