Examples with SerDeInfo org.apache.hadoop.hive.metastore.api.SerDeInfo used on opensource projects

Example 81 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestHBaseStoreCached method alterTable.

@Test
public void alterTable() throws Exception {
    String tableName = "alttable";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    store.createTable(table);
    startTime += 10;
    table.setLastAccessTime(startTime);
    store.alterTable("default", tableName, table);
    Table t = store.getTable("default", tableName);
    Assert.assertEquals(1, t.getSd().getColsSize());
    Assert.assertEquals("col1", t.getSd().getCols().get(0).getName());
    Assert.assertEquals("int", t.getSd().getCols().get(0).getType());
    Assert.assertEquals("nocomment", t.getSd().getCols().get(0).getComment());
    Assert.assertEquals("serde", t.getSd().getSerdeInfo().getName());
    Assert.assertEquals("seriallib", t.getSd().getSerdeInfo().getSerializationLib());
    Assert.assertEquals("file:/tmp", t.getSd().getLocation());
    Assert.assertEquals("input", t.getSd().getInputFormat());
    Assert.assertEquals("output", t.getSd().getOutputFormat());
    Assert.assertEquals("me", t.getOwner());
    Assert.assertEquals("default", t.getDbName());
    Assert.assertEquals(tableName, t.getTableName());
    Assert.assertEquals(startTime, t.getLastAccessTime());
}

Example 82 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestSharedStorageDescriptor method changeOnSerde.

@Test
public void changeOnSerde() {
    StorageDescriptor sd = new StorageDescriptor();
    SerDeInfo serde = new SerDeInfo();
    serde.setName("serde");
    sd.setSerdeInfo(serde);
    SharedStorageDescriptor ssd = new SharedStorageDescriptor();
    ssd.setShared(sd);
    ssd.getSerdeInfo().setName("different");
    Assert.assertFalse(sd.getSerdeInfo() == ssd.getSerdeInfo());
    Assert.assertEquals("serde", serde.getName());
    Assert.assertEquals("different", ssd.getSerdeInfo().getName());
    Assert.assertEquals("serde", sd.getSerdeInfo().getName());
}

Example 83 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestHBaseStoreCached method dropTable.

@Test
public void dropTable() throws Exception {
    String tableName = "dtable";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
    Table table = new Table(tableName, "default", "me", startTime, startTime, 0, sd, null, emptyParameters, null, null, null);
    store.createTable(table);
    Table t = store.getTable("default", tableName);
    Assert.assertNotNull(t);
    store.dropTable("default", tableName);
    Assert.assertNull(store.getTable("default", tableName));
}

Example 84 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class TestHBaseStore method getPartitions.

@Test
public void getPartitions() throws Exception {
    String tableName = "manyParts";
    int startTime = (int) (System.currentTimeMillis() / 1000);
    List<FieldSchema> cols = new ArrayList<FieldSchema>();
    cols.add(new FieldSchema("col1", "int", "nocomment"));
    SerDeInfo serde = new SerDeInfo("serde", "seriallib", null);
    StorageDescriptor sd = new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 0, serde, null, null, emptyParameters);
    List<FieldSchema> partCols = new ArrayList<FieldSchema>();
    partCols.add(new FieldSchema("pc", "string", ""));
    Table table = new Table(tableName, DB, "me", startTime, startTime, 0, sd, partCols, emptyParameters, null, null, null);
    store.createTable(table);
    List<String> partVals = Arrays.asList("alan", "bob", "carl", "doug", "ethan");
    for (String val : partVals) {
        List<String> vals = new ArrayList<String>();
        vals.add(val);
        StorageDescriptor psd = new StorageDescriptor(sd);
        psd.setLocation("file:/tmp/pc=" + val);
        Partition part = new Partition(vals, DB, tableName, startTime, startTime, psd, emptyParameters);
        store.addPartition(part);
        Partition p = store.getPartition(DB, tableName, vals);
        Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
    }
    List<Partition> parts = store.getPartitions(DB, tableName, -1);
    Assert.assertEquals(5, parts.size());
    String[] pv = new String[5];
    for (int i = 0; i < 5; i++) pv[i] = parts.get(i).getValues().get(0);
    Arrays.sort(pv);
    Assert.assertArrayEquals(pv, partVals.toArray(new String[5]));
}

Example 85 with SerDeInfo

use of org.apache.hadoop.hive.metastore.api.SerDeInfo in project hive by apache.

the class Hive method createIndex.

/**
   *
   * @param tableName
   *          table name
   * @param indexName
   *          index name
   * @param indexHandlerClass
   *          index handler class
   * @param indexedCols
   *          index columns
   * @param indexTblName
   *          index table's name
   * @param deferredRebuild
   *          referred build index table's data
   * @param inputFormat
   *          input format
   * @param outputFormat
   *          output format
   * @param serde
   * @param storageHandler
   *          index table's storage handler
   * @param location
   *          location
   * @param idxProps
   *          idx
   * @param serdeProps
   *          serde properties
   * @param collItemDelim
   * @param fieldDelim
   * @param fieldEscape
   * @param lineDelim
   * @param mapKeyDelim
   * @throws HiveException
   */
public void createIndex(String tableName, String indexName, String indexHandlerClass, List<String> indexedCols, String indexTblName, boolean deferredRebuild, String inputFormat, String outputFormat, String serde, String storageHandler, String location, Map<String, String> idxProps, Map<String, String> tblProps, Map<String, String> serdeProps, String collItemDelim, String fieldDelim, String fieldEscape, String lineDelim, String mapKeyDelim, String indexComment) throws HiveException {
    try {
        String tdname = Utilities.getDatabaseName(tableName);
        String idname = Utilities.getDatabaseName(indexTblName);
        if (!idname.equals(tdname)) {
            throw new HiveException("Index on different database (" + idname + ") from base table (" + tdname + ") is not supported.");
        }
        Index old_index = null;
        try {
            old_index = getIndex(tableName, indexName);
        } catch (Exception e) {
        }
        if (old_index != null) {
            throw new HiveException("Index " + indexName + " already exists on table " + tableName);
        }
        org.apache.hadoop.hive.metastore.api.Table baseTbl = getTable(tableName).getTTable();
        if (TableType.VIRTUAL_VIEW.toString().equals(baseTbl.getTableType())) {
            throw new HiveException("tableName=" + tableName + " is a VIRTUAL VIEW. Index on VIRTUAL VIEW is not supported.");
        }
        if (baseTbl.isTemporary()) {
            throw new HiveException("tableName=" + tableName + " is a TEMPORARY TABLE. Index on TEMPORARY TABLE is not supported.");
        }
        org.apache.hadoop.hive.metastore.api.Table temp = null;
        try {
            temp = getTable(indexTblName).getTTable();
        } catch (Exception e) {
        }
        if (temp != null) {
            throw new HiveException("Table name " + indexTblName + " already exists. Choose another name.");
        }
        SerDeInfo serdeInfo = new SerDeInfo();
        serdeInfo.setName(indexTblName);
        if (serde != null) {
            serdeInfo.setSerializationLib(serde);
        } else {
            if (storageHandler == null) {
                serdeInfo.setSerializationLib(org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.class.getName());
            } else {
                HiveStorageHandler sh = HiveUtils.getStorageHandler(getConf(), storageHandler);
                String serDeClassName = sh.getSerDeClass().getName();
                serdeInfo.setSerializationLib(serDeClassName);
            }
        }
        serdeInfo.setParameters(new HashMap<String, String>());
        if (fieldDelim != null) {
            serdeInfo.getParameters().put(FIELD_DELIM, fieldDelim);
            serdeInfo.getParameters().put(SERIALIZATION_FORMAT, fieldDelim);
        }
        if (fieldEscape != null) {
            serdeInfo.getParameters().put(ESCAPE_CHAR, fieldEscape);
        }
        if (collItemDelim != null) {
            serdeInfo.getParameters().put(COLLECTION_DELIM, collItemDelim);
        }
        if (mapKeyDelim != null) {
            serdeInfo.getParameters().put(MAPKEY_DELIM, mapKeyDelim);
        }
        if (lineDelim != null) {
            serdeInfo.getParameters().put(LINE_DELIM, lineDelim);
        }
        if (serdeProps != null) {
            Iterator<Entry<String, String>> iter = serdeProps.entrySet().iterator();
            while (iter.hasNext()) {
                Entry<String, String> m = iter.next();
                serdeInfo.getParameters().put(m.getKey(), m.getValue());
            }
        }
        List<FieldSchema> indexTblCols = new ArrayList<FieldSchema>();
        List<Order> sortCols = new ArrayList<Order>();
        int k = 0;
        Table metaBaseTbl = new Table(baseTbl);
        // Even though we are storing these in metastore, get regular columns. Indexes on lengthy
        // types from e.g. Avro schema will just fail to create the index table (by design).
        List<FieldSchema> cols = metaBaseTbl.getCols();
        for (int i = 0; i < cols.size(); i++) {
            FieldSchema col = cols.get(i);
            if (indexedCols.contains(col.getName())) {
                indexTblCols.add(col);
                sortCols.add(new Order(col.getName(), 1));
                k++;
            }
        }
        if (k != indexedCols.size()) {
            throw new RuntimeException("Check the index columns, they should appear in the table being indexed.");
        }
        int time = (int) (System.currentTimeMillis() / 1000);
        org.apache.hadoop.hive.metastore.api.Table tt = null;
        HiveIndexHandler indexHandler = HiveUtils.getIndexHandler(this.getConf(), indexHandlerClass);
        String itname = Utilities.getTableName(indexTblName);
        if (indexHandler.usesIndexTable()) {
            tt = new org.apache.hadoop.hive.ql.metadata.Table(idname, itname).getTTable();
            List<FieldSchema> partKeys = baseTbl.getPartitionKeys();
            tt.setPartitionKeys(partKeys);
            tt.setTableType(TableType.INDEX_TABLE.toString());
            if (tblProps != null) {
                for (Entry<String, String> prop : tblProps.entrySet()) {
                    tt.putToParameters(prop.getKey(), prop.getValue());
                }
            }
            SessionState ss = SessionState.get();
            CreateTableAutomaticGrant grants;
            if (ss != null && ((grants = ss.getCreateTableGrants()) != null)) {
                PrincipalPrivilegeSet principalPrivs = new PrincipalPrivilegeSet();
                principalPrivs.setUserPrivileges(grants.getUserGrants());
                principalPrivs.setGroupPrivileges(grants.getGroupGrants());
                principalPrivs.setRolePrivileges(grants.getRoleGrants());
                tt.setPrivileges(principalPrivs);
            }
        }
        if (!deferredRebuild) {
            throw new RuntimeException("Please specify deferred rebuild using \" WITH DEFERRED REBUILD \".");
        }
        StorageDescriptor indexSd = new StorageDescriptor(indexTblCols, location, inputFormat, outputFormat, false, /*compressed - not used*/
        -1, /*numBuckets - default is -1 when the table has no buckets*/
        serdeInfo, null, /*bucketCols*/
        sortCols, null);
        String ttname = Utilities.getTableName(tableName);
        Index indexDesc = new Index(indexName, indexHandlerClass, tdname, ttname, time, time, itname, indexSd, new HashMap<String, String>(), deferredRebuild);
        if (indexComment != null) {
            indexDesc.getParameters().put("comment", indexComment);
        }
        if (idxProps != null) {
            indexDesc.getParameters().putAll(idxProps);
        }
        indexHandler.analyzeIndexDefinition(baseTbl, indexDesc, tt);
        this.getMSC().createIndex(indexDesc, tt);
    } catch (Exception e) {
        throw new HiveException(e);
    }
}