Examples with TableIdentifier org.apache.iceberg.catalog.TableIdentifier used on opensource projects

Example 71 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testAlterTableAddColumns.

@Test
public void testAlterTableAddColumns() throws Exception {
    TableIdentifier identifier = TableIdentifier.of("default", "customers");
    testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, SPEC, FileFormat.PARQUET, ImmutableList.of());
    shell.executeStatement("ALTER TABLE default.customers ADD COLUMNS " + "(newintcol int, newstringcol string COMMENT 'Column with description')");
    verifyAlterTableAddColumnsTests();
}

Example 72 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testSetPartitionTransform.

@Test
public void testSetPartitionTransform() {
    Schema schema = new Schema(optional(1, "id", Types.LongType.get()), optional(2, "year_field", Types.DateType.get()), optional(3, "month_field", Types.TimestampType.withZone()), optional(4, "day_field", Types.TimestampType.withoutZone()), optional(5, "hour_field", Types.TimestampType.withoutZone()), optional(6, "truncate_field", Types.StringType.get()), optional(7, "bucket_field", Types.StringType.get()), optional(8, "identity_field", Types.StringType.get()));
    TableIdentifier identifier = TableIdentifier.of("default", "part_test");
    shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " PARTITIONED BY SPEC (year(year_field), hour(hour_field), " + "truncate(2, truncate_field), bucket(2, bucket_field), identity_field)" + " STORED BY ICEBERG " + testTables.locationForCreateTableSQL(identifier) + "TBLPROPERTIES ('" + InputFormatConfig.TABLE_SCHEMA + "'='" + SchemaParser.toJson(schema) + "', " + "'" + InputFormatConfig.CATALOG_NAME + "'='" + testTables.catalogName() + "')");
    PartitionSpec spec = PartitionSpec.builderFor(schema).year("year_field").hour("hour_field").truncate("truncate_field", 2).bucket("bucket_field", 2).identity("identity_field").build();
    Table table = testTables.loadTable(identifier);
    Assert.assertEquals(spec, table.spec());
    shell.executeStatement("ALTER TABLE default.part_test SET PARTITION SPEC(year(year_field), month(month_field), " + "day(day_field))");
    spec = PartitionSpec.builderFor(schema).withSpecId(1).year("year_field").alwaysNull("hour_field", "hour_field_hour").alwaysNull("truncate_field", "truncate_field_trunc").alwaysNull("bucket_field", "bucket_field_bucket").alwaysNull("identity_field", "identity_field").month("month_field").day("day_field").build();
    table.refresh();
    Assert.assertEquals(spec, table.spec());
}

Example 73 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testAlterTableReplaceColumns.

@Test
public void testAlterTableReplaceColumns() throws TException, InterruptedException {
    TableIdentifier identifier = TableIdentifier.of("default", "customers");
    Schema schema = new Schema(optional(1, "customer_id", Types.IntegerType.get()), optional(2, "first_name", Types.StringType.get(), "This is first name"), optional(3, "last_name", Types.StringType.get(), "This is last name"), optional(4, "address", Types.StructType.of(optional(5, "city", Types.StringType.get()), optional(6, "street", Types.StringType.get())), null));
    testTables.createTable(shell, identifier.name(), schema, SPEC, FileFormat.PARQUET, ImmutableList.of());
    shell.executeStatement("ALTER TABLE default.customers REPLACE COLUMNS " + "(customer_id int, last_name string COMMENT 'This is last name', " + "address struct<city:string,street:string>)");
    org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
    org.apache.hadoop.hive.metastore.api.Table hmsTable = shell.metastore().getTable("default", "customers");
    List<FieldSchema> icebergSchema = HiveSchemaUtil.convert(icebergTable.schema());
    List<FieldSchema> hmsSchema = hmsTable.getSd().getCols();
    List<FieldSchema> expectedSchema = Lists.newArrayList(new FieldSchema("customer_id", "int", null), // first_name column is dropped
    new FieldSchema("last_name", "string", "This is last name"), new FieldSchema("address", "struct<city:string,street:string>", null));
    Assert.assertEquals(expectedSchema, icebergSchema);
    Assert.assertEquals(expectedSchema, hmsSchema);
}

Example 74 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testCreatePartitionedTableWithColumnComments.

@Test
public void testCreatePartitionedTableWithColumnComments() {
    TableIdentifier identifier = TableIdentifier.of("default", "partitioned_with_comment_table");
    String[] expectedDoc = new String[] { "int column", "string column", null, "partition column", null };
    shell.executeStatement("CREATE EXTERNAL TABLE partitioned_with_comment_table (" + "t_int INT COMMENT 'int column',  " + "t_string STRING COMMENT 'string column', " + "t_string_2 STRING) " + "PARTITIONED BY (t_string_3 STRING COMMENT 'partition column', t_string_4 STRING) " + "STORED BY ICEBERG " + testTables.locationForCreateTableSQL(identifier) + testTables.propertiesForCreateTableSQL(ImmutableMap.of()));
    org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
    List<Object[]> rows = shell.executeStatement("DESCRIBE default.partitioned_with_comment_table");
    List<Types.NestedField> columns = icebergTable.schema().columns();
    // The partition transform information is 3 extra lines, and 2 more line for the columns
    Assert.assertEquals(columns.size() + 5, rows.size());
    for (int i = 0; i < columns.size(); i++) {
        Types.NestedField field = columns.get(i);
        Assert.assertArrayEquals(new Object[] { field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(), field.doc() != null ? field.doc() : "from deserializer" }, rows.get(i));
        Assert.assertEquals(expectedDoc[i], field.doc());
    }
}

Example 75 with TableIdentifier

use of org.apache.iceberg.catalog.TableIdentifier in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testDropTableWithAppendedData.

@Test
public void testDropTableWithAppendedData() throws IOException {
    TableIdentifier identifier = TableIdentifier.of("default", "customers");
    testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, SPEC, FileFormat.PARQUET, ImmutableList.of());
    org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
    testTables.appendIcebergTable(shell.getHiveConf(), icebergTable, FileFormat.PARQUET, null, HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS);
    shell.executeStatement("DROP TABLE customers");
}