Examples with HCatSchema org.apache.hive.hcatalog.data.schema.HCatSchema used on opensource projects

Example 16 with HCatSchema

use of org.apache.hive.hcatalog.data.schema.HCatSchema in project hive by apache.

the class TestDefaultHCatRecord method testGetSetByType1.

/**
   * Test get and set calls with type
   * @throws HCatException
   */
public void testGetSetByType1() throws HCatException {
    HCatRecord inpRec = getHCatRecords()[0];
    HCatRecord newRec = new DefaultHCatRecord(inpRec.size());
    HCatSchema hsch = HCatSchemaUtils.getHCatSchema("a:tinyint,b:smallint,c:int,d:bigint,e:float,f:double,g:boolean,h:string,i:binary,j:string");
    newRec.setByte("a", hsch, inpRec.getByte("a", hsch));
    newRec.setShort("b", hsch, inpRec.getShort("b", hsch));
    newRec.setInteger("c", hsch, inpRec.getInteger("c", hsch));
    newRec.setLong("d", hsch, inpRec.getLong("d", hsch));
    newRec.setFloat("e", hsch, inpRec.getFloat("e", hsch));
    newRec.setDouble("f", hsch, inpRec.getDouble("f", hsch));
    newRec.setBoolean("g", hsch, inpRec.getBoolean("g", hsch));
    newRec.setString("h", hsch, inpRec.getString("h", hsch));
    newRec.setByteArray("i", hsch, inpRec.getByteArray("i", hsch));
    newRec.setString("j", hsch, inpRec.getString("j", hsch));
    Assert.assertTrue(HCatDataCheckUtil.recordsEqual(newRec, inpRec));
}

Example 17 with HCatSchema

use of org.apache.hive.hcatalog.data.schema.HCatSchema in project hive by apache.

the class TestDefaultHCatRecord method testGetSetByType2.

/**
   * Test get and set calls with type
   * @throws HCatException
   */
public void testGetSetByType2() throws HCatException {
    HCatRecord inpRec = getGetSet2InpRec();
    HCatRecord newRec = new DefaultHCatRecord(inpRec.size());
    HCatSchema hsch = HCatSchemaUtils.getHCatSchema("a:binary,b:map<string,string>,c:array<int>,d:struct<i:int>");
    newRec.setByteArray("a", hsch, inpRec.getByteArray("a", hsch));
    newRec.setMap("b", hsch, inpRec.getMap("b", hsch));
    newRec.setList("c", hsch, inpRec.getList("c", hsch));
    newRec.setStruct("d", hsch, inpRec.getStruct("d", hsch));
    Assert.assertTrue(HCatDataCheckUtil.recordsEqual(newRec, inpRec));
}

Example 18 with HCatSchema

use of org.apache.hive.hcatalog.data.schema.HCatSchema in project hive by apache.

the class TestHCatPartitionPublish method runMRCreateFail.

void runMRCreateFail(String dbName, String tableName, Map<String, String> partitionValues, List<HCatFieldSchema> columns) throws Exception {
    Job job = new Job(mrConf, "hcat mapreduce write fail test");
    job.setJarByClass(this.getClass());
    job.setMapperClass(TestHCatPartitionPublish.MapFail.class);
    // input/output settings
    job.setInputFormatClass(TextInputFormat.class);
    Path path = new Path(fs.getWorkingDirectory(), "mapred/testHCatMapReduceInput");
    // The write count does not matter, as the map will fail in its first
    // call.
    createInputFile(path, 5);
    TextInputFormat.setInputPaths(job, path);
    job.setOutputFormatClass(HCatOutputFormat.class);
    OutputJobInfo outputJobInfo = OutputJobInfo.create(dbName, tableName, partitionValues);
    HCatOutputFormat.setOutput(job, outputJobInfo);
    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(DefaultHCatRecord.class);
    job.setNumReduceTasks(0);
    HCatOutputFormat.setSchema(job, new HCatSchema(columns));
    boolean success = job.waitForCompletion(true);
    Assert.assertTrue(success == false);
}

Example 19 with HCatSchema

use of org.apache.hive.hcatalog.data.schema.HCatSchema in project hive by apache.

the class TestHCatPartitioned method tableSchemaTest.

//test that new columns gets added to table schema
private void tableSchemaTest() throws Exception {
    HCatSchema tableSchema = getTableSchema();
    assertEquals(4, tableSchema.getFields().size());
    //Update partition schema to have 3 fields
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", serdeConstants.STRING_TYPE_NAME, "")));
    writeRecords = new ArrayList<HCatRecord>();
    for (int i = 0; i < 20; i++) {
        List<Object> objList = new ArrayList<Object>();
        objList.add(i);
        objList.add("strvalue" + i);
        objList.add("str2value" + i);
        writeRecords.add(new DefaultHCatRecord(objList));
    }
    Map<String, String> partitionMap = new HashMap<String, String>();
    partitionMap.put("part1", "p1value5");
    partitionMap.put("part0", "505");
    runMRCreate(partitionMap, partitionColumns, writeRecords, 10, true);
    tableSchema = getTableSchema();
    //assert that c3 has got added to table schema
    assertEquals(5, tableSchema.getFields().size());
    assertEquals("c1", tableSchema.getFields().get(0).getName());
    assertEquals("c2", tableSchema.getFields().get(1).getName());
    assertEquals("c3", tableSchema.getFields().get(2).getName());
    assertEquals("part1", tableSchema.getFields().get(3).getName());
    assertEquals("part0", tableSchema.getFields().get(4).getName());
    //Test that changing column data type fails
    partitionMap.clear();
    partitionMap.put("part1", "p1value6");
    partitionMap.put("part0", "506");
    partitionColumns = new ArrayList<HCatFieldSchema>();
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", serdeConstants.INT_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c2", serdeConstants.INT_TYPE_NAME, "")));
    IOException exc = null;
    try {
        runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true);
    } catch (IOException e) {
        exc = e;
    }
    assertTrue(exc != null);
    assertTrue(exc instanceof HCatException);
    assertEquals(ErrorType.ERROR_SCHEMA_TYPE_MISMATCH, ((HCatException) exc).getErrorType());
    //Test that partition key is not allowed in data
    partitionColumns = new ArrayList<HCatFieldSchema>();
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", serdeConstants.INT_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c2", serdeConstants.STRING_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", serdeConstants.STRING_TYPE_NAME, "")));
    partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("part1", serdeConstants.STRING_TYPE_NAME, "")));
    List<HCatRecord> recordsContainingPartitionCols = new ArrayList<HCatRecord>(20);
    for (int i = 0; i < 20; i++) {
        List<Object> objList = new ArrayList<Object>();
        objList.add(i);
        objList.add("c2value" + i);
        objList.add("c3value" + i);
        objList.add("p1value6");
        recordsContainingPartitionCols.add(new DefaultHCatRecord(objList));
    }
    exc = null;
    try {
        runMRCreate(partitionMap, partitionColumns, recordsContainingPartitionCols, 20, true);
    } catch (IOException e) {
        exc = e;
    }
    List<HCatRecord> records = runMRRead(20, "part1 = \"p1value6\"");
    assertEquals(20, records.size());
    records = runMRRead(20, "part0 = \"506\"");
    assertEquals(20, records.size());
    Integer i = 0;
    for (HCatRecord rec : records) {
        assertEquals(5, rec.size());
        assertEquals(rec.get(0), i);
        assertEquals(rec.get(1), "c2value" + i);
        assertEquals(rec.get(2), "c3value" + i);
        assertEquals(rec.get(3), "p1value6");
        assertEquals(rec.get(4), 506);
        i++;
    }
}

Example 20 with HCatSchema

use of org.apache.hive.hcatalog.data.schema.HCatSchema in project flink by apache.

the class HCatInputFormatBase method getFields.

/**
	 * Specifies the fields which are returned by the InputFormat and their order.
	 *
	 * @param fields The fields and their order which are returned by the InputFormat.
	 * @return This InputFormat with specified return fields.
	 * @throws java.io.IOException
	 */
public HCatInputFormatBase<T> getFields(String... fields) throws IOException {
    // build output schema
    ArrayList<HCatFieldSchema> fieldSchemas = new ArrayList<HCatFieldSchema>(fields.length);
    for (String field : fields) {
        fieldSchemas.add(this.outputSchema.get(field));
    }
    this.outputSchema = new HCatSchema(fieldSchemas);
    // update output schema configuration
    configuration.set("mapreduce.lib.hcat.output.schema", HCatUtil.serialize(outputSchema));
    return this;
}