Examples with FieldSchema org.apache.hadoop.hive.metastore.api.FieldSchema used on opensource projects

Example 1 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHive method testIndex.

/**
   * Tests creating a simple index on a simple table.
   *
   * @throws Throwable
   */
public void testIndex() throws Throwable {
    try {
        // create a simple table
        String tableName = "table_for_testindex";
        String qTableName = MetaStoreUtils.DEFAULT_DATABASE_NAME + "." + tableName;
        try {
            hm.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
        } catch (HiveException e) {
            e.printStackTrace();
            assertTrue("Unable to drop table", false);
        }
        Table tbl = new Table(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName);
        List<FieldSchema> fields = tbl.getCols();
        fields.add(new FieldSchema("col1", serdeConstants.INT_TYPE_NAME, "int -- first column"));
        fields.add(new FieldSchema("col2", serdeConstants.STRING_TYPE_NAME, "string -- second column"));
        fields.add(new FieldSchema("col3", serdeConstants.DOUBLE_TYPE_NAME, "double -- thrift column"));
        tbl.setFields(fields);
        tbl.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class);
        tbl.setInputFormatClass(SequenceFileInputFormat.class);
        // create table
        try {
            hm.createTable(tbl);
        } catch (HiveException e) {
            e.printStackTrace();
            assertTrue("Unable to create table: " + tableName, false);
        }
        // Create a simple index
        String indexName = "index_on_table_for_testindex";
        String indexHandlerClass = HiveIndex.IndexType.COMPACT_SUMMARY_TABLE.getHandlerClsName();
        List<String> indexedCols = new ArrayList<String>();
        indexedCols.add("col1");
        String indexTableName = "index_on_table_for_testindex_table";
        String qIndexTableName = MetaStoreUtils.DEFAULT_DATABASE_NAME + "." + indexTableName;
        boolean deferredRebuild = true;
        String inputFormat = SequenceFileInputFormat.class.getName();
        String outputFormat = SequenceFileOutputFormat.class.getName();
        String serde = null;
        String storageHandler = null;
        String location = null;
        String collItemDelim = null;
        String fieldDelim = null;
        String fieldEscape = null;
        String lineDelim = null;
        String mapKeyDelim = null;
        String indexComment = null;
        Map<String, String> indexProps = null;
        Map<String, String> tableProps = null;
        Map<String, String> serdeProps = new HashMap<String, String>();
        hm.createIndex(qTableName, indexName, indexHandlerClass, indexedCols, qIndexTableName, deferredRebuild, inputFormat, outputFormat, serde, storageHandler, location, indexProps, tableProps, serdeProps, collItemDelim, fieldDelim, fieldEscape, lineDelim, mapKeyDelim, indexComment);
        // Retrieve and validate the index
        Index index = null;
        try {
            index = hm.getIndex(tableName, indexName);
            assertNotNull("Unable to fetch index", index);
            index.validate();
            assertEquals("Index names don't match for index: " + indexName, indexName, index.getIndexName());
            assertEquals("Table names don't match for index: " + indexName, tableName, index.getOrigTableName());
            assertEquals("Index table names didn't match for index: " + indexName, indexTableName, index.getIndexTableName());
            assertEquals("Index handler classes didn't match for index: " + indexName, indexHandlerClass, index.getIndexHandlerClass());
            assertEquals("Deferred rebuild didn't match for index: " + indexName, deferredRebuild, index.isDeferredRebuild());
        } catch (HiveException e) {
            System.err.println(StringUtils.stringifyException(e));
            assertTrue("Unable to fetch index correctly: " + indexName, false);
        }
        // Drop index
        try {
            hm.dropIndex(MetaStoreUtils.DEFAULT_DATABASE_NAME, tableName, indexName, false, true);
        } catch (HiveException e) {
            System.err.println(StringUtils.stringifyException(e));
            assertTrue("Unable to drop index: " + indexName, false);
        }
        boolean dropIndexException = false;
        try {
            hm.getIndex(tableName, indexName);
        } catch (HiveException e) {
            // Expected since it was just dropped
            dropIndexException = true;
        }
        assertTrue("Unable to drop index: " + indexName, dropIndexException);
        // Drop table
        try {
            hm.dropTable(tableName);
            Table droppedTable = hm.getTable(tableName, false);
            assertNull("Unable to drop table " + tableName, droppedTable);
        } catch (HiveException e) {
            System.err.println(StringUtils.stringifyException(e));
            assertTrue("Unable to drop table: " + tableName, false);
        }
    } catch (Throwable e) {
        System.err.println(StringUtils.stringifyException(e));
        System.err.println("testIndex failed");
        throw e;
    }
}

Example 2 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHiveMetaStoreChecker method setUp.

@Override
protected void setUp() throws Exception {
    super.setUp();
    hive = Hive.get();
    hive.getConf().setIntVar(HiveConf.ConfVars.METASTORE_FS_HANDLER_THREADS_COUNT, 15);
    hive.getConf().set(HiveConf.ConfVars.HIVE_MSCK_PATH_VALIDATION.varname, "throw");
    checker = new HiveMetaStoreChecker(hive);
    partCols = new ArrayList<FieldSchema>();
    partCols.add(new FieldSchema(partDateName, serdeConstants.STRING_TYPE_NAME, ""));
    partCols.add(new FieldSchema(partCityName, serdeConstants.STRING_TYPE_NAME, ""));
    parts = new ArrayList<Map<String, String>>();
    Map<String, String> part1 = new HashMap<String, String>();
    part1.put(partDateName, "2008-01-01");
    part1.put(partCityName, "london");
    parts.add(part1);
    Map<String, String> part2 = new HashMap<String, String>();
    part2.put(partDateName, "2008-01-02");
    part2.put(partCityName, "stockholm");
    parts.add(part2);
    //cleanup just in case something is left over from previous run
    dropDbTable();
}

Example 3 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class TestHiveMetaStoreChecker method createPartitionsDirectoriesOnFS.

/**
   * Creates partition sub-directories for a given table on the file system. Used to test the
   * use-cases when partitions for the table are not present in the metastore db
   *
   * @param table - Table which provides the base locations and partition specs for creating the
   *          sub-directories
   * @param numPartitions - Number of partitions to be created
   * @throws IOException
   */
private void createPartitionsDirectoriesOnFS(Table table, int numPartitions) throws IOException {
    String path = table.getDataLocation().toString();
    fs = table.getPath().getFileSystem(hive.getConf());
    int numPartKeys = table.getPartitionKeys().size();
    for (int i = 0; i < numPartitions; i++) {
        StringBuilder partPath = new StringBuilder(path);
        partPath.append(Path.SEPARATOR);
        for (int j = 0; j < numPartKeys; j++) {
            FieldSchema field = table.getPartitionKeys().get(j);
            partPath.append(field.getName());
            partPath.append('=');
            partPath.append("val_");
            partPath.append(i);
            if (j < (numPartKeys - 1)) {
                partPath.append(Path.SEPARATOR);
            }
        }
        createDirectory(partPath.toString());
    }
}

Example 4 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class SQLOperation method getSerDe.

private AbstractSerDe getSerDe() throws SQLException {
    if (serde != null) {
        return serde;
    }
    try {
        List<FieldSchema> fieldSchemas = mResultSchema.getFieldSchemas();
        StringBuilder namesSb = new StringBuilder();
        StringBuilder typesSb = new StringBuilder();
        if (fieldSchemas != null && !fieldSchemas.isEmpty()) {
            for (int pos = 0; pos < fieldSchemas.size(); pos++) {
                if (pos != 0) {
                    namesSb.append(",");
                    typesSb.append(",");
                }
                namesSb.append(fieldSchemas.get(pos).getName());
                typesSb.append(fieldSchemas.get(pos).getType());
            }
        }
        String names = namesSb.toString();
        String types = typesSb.toString();
        serde = new LazySimpleSerDe();
        Properties props = new Properties();
        if (names.length() > 0) {
            LOG.debug("Column names: " + names);
            props.setProperty(serdeConstants.LIST_COLUMNS, names);
        }
        if (types.length() > 0) {
            LOG.debug("Column types: " + types);
            props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types);
        }
        SerDeUtils.initializeSerDe(serde, queryState.getConf(), props, null);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new SQLException("Could not create ResultSet: " + ex.getMessage(), ex);
    }
    return serde;
}

Example 5 with FieldSchema

use of org.apache.hadoop.hive.metastore.api.FieldSchema in project hive by apache.

the class DelimitedInputWriter method getCols.

private ArrayList<String> getCols(Table table) {
    List<FieldSchema> cols = table.getSd().getCols();
    ArrayList<String> colNames = new ArrayList<String>(cols.size());
    for (FieldSchema col : cols) {
        colNames.add(col.getName().toLowerCase());
    }
    return colNames;
}