Examples with TableInfo co.cask.cdap.proto.TableInfo used on opensource projects

Example 1 with TableInfo

use of co.cask.cdap.proto.TableInfo in project cdap by caskdata.

the class HiveExploreServiceTestRun method testTableInfo.

private void testTableInfo(DatasetId datasetId, @Nullable String customDatabase, @Nullable String customTable) throws Exception {
    ExploreProperties.Builder builder = ExploreProperties.builder();
    if (customDatabase != null) {
        builder.setExploreDatabaseName(customDatabase);
    }
    if (customTable != null) {
        builder.setExploreTableName(customTable);
    }
    datasetFramework.addInstance("keyStructValueTable", datasetId, builder.build());
    String tableName = customTable != null ? customTable : "dataset_" + datasetId.getDataset().toLowerCase();
    String databaseName = customDatabase != null ? customDatabase : "cdap_" + datasetId.getNamespace().toLowerCase();
    try {
        TableInfo info = exploreService.getTableInfo(datasetId.getNamespace(), customDatabase, tableName);
        TableInfo clientInfo = exploreClient.getTableInfo(datasetId.getNamespace(), customDatabase, tableName);
        Assert.assertEquals(databaseName, info.getDbName());
        Assert.assertEquals(tableName, info.getTableName());
        Assert.assertEquals(info, clientInfo);
    } finally {
        datasetFramework.deleteInstance(datasetId);
    }
}

Example 2 with TableInfo

use of co.cask.cdap.proto.TableInfo in project cdap by caskdata.

the class HiveExploreServiceTestRun method getDatasetSchemaTest.

@Test
public void getDatasetSchemaTest() throws Exception {
    TableInfo tableInfo = exploreService.getTableInfo(NAMESPACE_ID.getNamespace(), MY_TABLE_NAME);
    Assert.assertEquals(new TableInfo(MY_TABLE_NAME, NAMESPACE_DATABASE, System.getProperty("user.name"), tableInfo.getCreationTime(), 0, 0, ImmutableList.<TableInfo.ColumnInfo>of(), tableInfo.getParameters(), "EXTERNAL_TABLE", ImmutableList.of(new TableInfo.ColumnInfo("key", "string", null), new TableInfo.ColumnInfo("value", "struct<name:string,ints:array<int>>", null)), tableInfo.getLocation(), null, null, false, -1, DatasetSerDe.class.getName(), ImmutableMap.of("serialization.format", "1", Constants.Explore.DATASET_NAME, MY_TABLE.getEntityName(), Constants.Explore.DATASET_NAMESPACE, NAMESPACE_ID.getNamespace()), true), tableInfo);
    Assert.assertEquals(DatasetStorageHandler.class.getName(), tableInfo.getParameters().get("storage_handler"));
    tableInfo = exploreService.getTableInfo(NAMESPACE_ID.getNamespace(), MY_TABLE_NAME);
    Assert.assertEquals(new TableInfo(MY_TABLE_NAME, NAMESPACE_DATABASE, System.getProperty("user.name"), tableInfo.getCreationTime(), 0, 0, ImmutableList.<TableInfo.ColumnInfo>of(), tableInfo.getParameters(), "EXTERNAL_TABLE", ImmutableList.of(new TableInfo.ColumnInfo("key", "string", null), new TableInfo.ColumnInfo("value", "struct<name:string,ints:array<int>>", null)), tableInfo.getLocation(), null, null, false, -1, DatasetSerDe.class.getName(), ImmutableMap.of("serialization.format", "1", Constants.Explore.DATASET_NAME, MY_TABLE.getEntityName(), Constants.Explore.DATASET_NAMESPACE, NAMESPACE_ID.getNamespace()), true), tableInfo);
    Assert.assertEquals(DatasetStorageHandler.class.getName(), tableInfo.getParameters().get("storage_handler"));
    try {
        exploreService.getTableInfo("foo", "foobar");
        Assert.fail("Should throw TableNotFoundException on table foobar");
    } catch (TableNotFoundException e) {
    // Expected
    }
    try {
        exploreService.getTableInfo("foo", MY_TABLE_NAME);
        Assert.fail("Should throw TableNotFoundException on table foo.my_table");
    } catch (TableNotFoundException e) {
    // Expected
    }
    // Get info of a Hive table
    exploreClient.submit(NAMESPACE_ID, "create table test (first INT, second STRING) " + "ROW FORMAT DELIMITED FIELDS TERMINATED BY '\\t'").get();
    tableInfo = exploreService.getTableInfo(NAMESPACE_ID.getNamespace(), "test");
    Assert.assertEquals(new TableInfo("test", NAMESPACE_DATABASE, System.getProperty("user.name"), tableInfo.getCreationTime(), 0, 0, ImmutableList.<TableInfo.ColumnInfo>of(), tableInfo.getParameters(), "MANAGED_TABLE", ImmutableList.of(new TableInfo.ColumnInfo("first", "int", null), new TableInfo.ColumnInfo("second", "string", null)), tableInfo.getLocation(), "org.apache.hadoop.mapred.TextInputFormat", "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", false, -1, "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", ImmutableMap.of("serialization.format", "\t", "field.delim", "\t"), false), tableInfo);
    exploreClient.submit(NAMESPACE_ID, "drop table if exists test").get();
    // Get info of a partitioned table
    exploreClient.submit(NAMESPACE_ID, "CREATE TABLE page_view(viewTime INT, userid BIGINT, page_url STRING, referrer_url STRING, " + "ip STRING COMMENT \"IP Address of the User\") COMMENT \"This is the page view table\" " + "PARTITIONED BY(dt STRING, country STRING) STORED AS SEQUENCEFILE").get();
    tableInfo = exploreService.getTableInfo(NAMESPACE_ID.getNamespace(), "page_view");
    Assert.assertEquals(new TableInfo("page_view", NAMESPACE_DATABASE, System.getProperty("user.name"), tableInfo.getCreationTime(), 0, 0, ImmutableList.of(new TableInfo.ColumnInfo("dt", "string", null), new TableInfo.ColumnInfo("country", "string", null)), tableInfo.getParameters(), "MANAGED_TABLE", ImmutableList.of(new TableInfo.ColumnInfo("viewtime", "int", null), new TableInfo.ColumnInfo("userid", "bigint", null), new TableInfo.ColumnInfo("page_url", "string", null), new TableInfo.ColumnInfo("referrer_url", "string", null), new TableInfo.ColumnInfo("ip", "string", "IP Address of the User"), new TableInfo.ColumnInfo("dt", "string", null), new TableInfo.ColumnInfo("country", "string", null)), tableInfo.getLocation(), "org.apache.hadoop.mapred.SequenceFileInputFormat", "org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat", false, -1, "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", ImmutableMap.of("serialization.format", "1"), false), tableInfo);
    Assert.assertEquals("This is the page view table", tableInfo.getParameters().get("comment"));
    exploreClient.submit(NAMESPACE_ID, "drop table if exists page_view").get();
}

Example 3 with TableInfo

use of co.cask.cdap.proto.TableInfo in project cdap by caskdata.

the class BaseHiveExploreService method getTableInfo.

@Override
public TableInfo getTableInfo(String namespace, @Nullable String databaseName, String table) throws ExploreException, TableNotFoundException {
    startAndWait();
    // TODO check if the database user is allowed to access if security is enabled
    try {
        String db = databaseName != null ? databaseName : getHiveDatabase(namespace);
        Table tableInfo = getMetaStoreClient().getTable(db, table);
        List<FieldSchema> tableFields = tableInfo.getSd().getCols();
        // in the storage descriptor. If columns are missing, do a separate call for schema.
        if (tableFields == null || tableFields.isEmpty()) {
            // don't call .getSchema()... class not found exception if we do in the thrift code...
            tableFields = getMetaStoreClient().getFields(db, table);
        }
        ImmutableList.Builder<TableInfo.ColumnInfo> schemaBuilder = ImmutableList.builder();
        Set<String> fieldNames = Sets.newHashSet();
        for (FieldSchema column : tableFields) {
            schemaBuilder.add(new TableInfo.ColumnInfo(column.getName(), column.getType(), column.getComment()));
            fieldNames.add(column.getName());
        }
        ImmutableList.Builder<TableInfo.ColumnInfo> partitionKeysBuilder = ImmutableList.builder();
        for (FieldSchema column : tableInfo.getPartitionKeys()) {
            TableInfo.ColumnInfo columnInfo = new TableInfo.ColumnInfo(column.getName(), column.getType(), column.getComment());
            partitionKeysBuilder.add(columnInfo);
            // since they show up when you do a 'describe <table>' command.
            if (!fieldNames.contains(column.getName())) {
                schemaBuilder.add(columnInfo);
            }
        }
        // its a cdap generated table if it uses our storage handler, or if a property is set on the table.
        String cdapName = null;
        Map<String, String> tableParameters = tableInfo.getParameters();
        if (tableParameters != null) {
            cdapName = tableParameters.get(Constants.Explore.CDAP_NAME);
        }
        // tables created after CDAP 2.6 should set the "cdap.name" property, but older ones
        // do not. So also check if it uses a cdap storage handler.
        String storageHandler = tableInfo.getParameters().get("storage_handler");
        boolean isDatasetTable = cdapName != null || DatasetStorageHandler.class.getName().equals(storageHandler) || StreamStorageHandler.class.getName().equals(storageHandler);
        return new TableInfo(tableInfo.getTableName(), tableInfo.getDbName(), tableInfo.getOwner(), (long) tableInfo.getCreateTime() * 1000, (long) tableInfo.getLastAccessTime() * 1000, tableInfo.getRetention(), partitionKeysBuilder.build(), tableInfo.getParameters(), tableInfo.getTableType(), schemaBuilder.build(), tableInfo.getSd().getLocation(), tableInfo.getSd().getInputFormat(), tableInfo.getSd().getOutputFormat(), tableInfo.getSd().isCompressed(), tableInfo.getSd().getNumBuckets(), tableInfo.getSd().getSerdeInfo().getSerializationLib(), tableInfo.getSd().getSerdeInfo().getParameters(), isDatasetTable);
    } catch (NoSuchObjectException e) {
        throw new TableNotFoundException(e);
    } catch (TException e) {
        throw new ExploreException(e);
    }
}