Examples with Schema org.apache.iceberg.Schema used on opensource projects

Example 66 with Schema

use of org.apache.iceberg.Schema in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testSetPartitionTransformCaseSensitive.

@Test
public void testSetPartitionTransformCaseSensitive() {
    Schema schema = new Schema(optional(1, "id", Types.LongType.get()), optional(2, "truncate_field", Types.StringType.get()), optional(3, "bucket_field", Types.StringType.get()));
    TableIdentifier identifier = TableIdentifier.of("default", "part_test");
    shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " PARTITIONED BY SPEC (truncate(2, truncate_field), bucket(2, bucket_field))" + " STORED BY ICEBERG " + testTables.locationForCreateTableSQL(identifier) + "TBLPROPERTIES ('" + InputFormatConfig.TABLE_SCHEMA + "'='" + SchemaParser.toJson(schema) + "', " + "'" + InputFormatConfig.CATALOG_NAME + "'='" + testTables.catalogName() + "')");
    PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("truncate_field", 2).bucket("bucket_field", 2).build();
    Table table = testTables.loadTable(identifier);
    Assert.assertEquals(spec, table.spec());
    shell.executeStatement("ALTER TABLE default.part_test " + "SET PARTITION SPEC (truncaTe(3, truncate_Field), buCket(3, bUckeT_field))");
    spec = PartitionSpec.builderFor(schema).withSpecId(1).alwaysNull("truncate_field", "truncate_field_trunc").alwaysNull("bucket_field", "bucket_field_bucket").truncate("truncate_field", 3, "truncate_field_trunc_3").bucket("bucket_field", 3, "bucket_field_bucket_3").build();
    table.refresh();
    Assert.assertEquals(spec, table.spec());
}

Example 67 with Schema

use of org.apache.iceberg.Schema in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testCreateTableWithAllSupportedTypes.

@Test
public void testCreateTableWithAllSupportedTypes() {
    TableIdentifier identifier = TableIdentifier.of("default", "all_types");
    Schema allSupportedSchema = new Schema(optional(1, "t_float", Types.FloatType.get()), optional(2, "t_double", Types.DoubleType.get()), optional(3, "t_boolean", Types.BooleanType.get()), optional(4, "t_int", Types.IntegerType.get()), optional(5, "t_bigint", Types.LongType.get()), optional(6, "t_binary", Types.BinaryType.get()), optional(7, "t_string", Types.StringType.get()), optional(8, "t_timestamp", Types.TimestampType.withoutZone()), optional(9, "t_date", Types.DateType.get()), optional(10, "t_decimal", Types.DecimalType.of(3, 2)));
    // Intentionally adding some mixed letters to test that we handle them correctly
    shell.executeStatement("CREATE EXTERNAL TABLE all_types (" + "t_Float FLOaT, t_dOuble DOUBLE, t_boolean BOOLEAN, t_int INT, t_bigint BIGINT, t_binary BINARY, " + "t_string STRING, t_timestamp TIMESTAMP, t_date DATE, t_decimal DECIMAL(3,2)) " + "STORED BY ICEBERG " + testTables.locationForCreateTableSQL(identifier) + testTables.propertiesForCreateTableSQL(ImmutableMap.of()));
    // Check the Iceberg table data
    org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
    Assert.assertEquals(allSupportedSchema.asStruct(), icebergTable.schema().asStruct());
}

Example 68 with Schema

use of org.apache.iceberg.Schema in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testAlterTableReplaceColumnsFailsWhenNotOnlyDropping.

@Test
public void testAlterTableReplaceColumnsFailsWhenNotOnlyDropping() {
    TableIdentifier identifier = TableIdentifier.of("default", "customers");
    Schema schema = new Schema(optional(1, "customer_id", Types.IntegerType.get()), optional(2, "first_name", Types.StringType.get(), "This is first name"), optional(3, "last_name", Types.StringType.get(), "This is last name"), optional(4, "address", Types.StructType.of(optional(5, "city", Types.StringType.get()), optional(6, "street", Types.StringType.get())), null));
    testTables.createTable(shell, identifier.name(), schema, SPEC, FileFormat.PARQUET, ImmutableList.of());
    // check unsupported operations
    String[] commands = { // type promotion
    "ALTER TABLE default.customers REPLACE COLUMNS (customer_id bigint, first_name string COMMENT 'This is " + "first name', last_name string COMMENT 'This is last name', address struct<city:string,street:string>)", // delete a comment
    "ALTER TABLE default.customers REPLACE COLUMNS (customer_id int, first_name string, " + "last_name string COMMENT 'This is last name', address struct<city:string,street:string>)", // change a comment
    "ALTER TABLE default.customers REPLACE COLUMNS (customer_id int, first_name string COMMENT 'New docs', " + "last_name string COMMENT 'This is last name', address struct<city:string,street:string>)", // reorder columns
    "ALTER TABLE default.customers REPLACE COLUMNS (customer_id int, last_name string COMMENT 'This is " + "last name', first_name string COMMENT 'This is first name', address struct<city:string,street:string>)", // add new column
    "ALTER TABLE default.customers REPLACE COLUMNS (customer_id int, first_name string COMMENT 'This is " + "first name', last_name string COMMENT 'This is last name', address struct<city:string,street:string>, " + "new_col timestamp)", // dropping a column + reordering columns
    "ALTER TABLE default.customers REPLACE COLUMNS (last_name string COMMENT 'This is " + "last name', first_name string COMMENT 'This is first name', address struct<city:string,street:string>)" };
    for (String command : commands) {
        AssertHelpers.assertThrows("", IllegalArgumentException.class, "Unsupported operation to use REPLACE COLUMNS", () -> shell.executeStatement(command));
    }
    // check no-op case too
    String command = "ALTER TABLE default.customers REPLACE COLUMNS (customer_id int, first_name string COMMENT 'This" + " is first name', last_name string COMMENT 'This is last name', address struct<city:string,street:string>)";
    AssertHelpers.assertThrows("", IllegalArgumentException.class, "No schema change detected", () -> shell.executeStatement(command));
}

Example 69 with Schema

use of org.apache.iceberg.Schema in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testAlterTableChangeColumnNameAndComment.

@Test
public void testAlterTableChangeColumnNameAndComment() throws TException, InterruptedException {
    TableIdentifier identifier = TableIdentifier.of("default", "customers");
    Schema schema = new Schema(optional(1, "customer_id", Types.IntegerType.get()), optional(2, "last_name", Types.StringType.get(), "This is last name"));
    testTables.createTable(shell, identifier.name(), schema, SPEC, FileFormat.PARQUET, ImmutableList.of());
    shell.executeStatement("ALTER TABLE default.customers CHANGE COLUMN " + "last_name family_name string COMMENT 'This is family name'");
    org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
    org.apache.hadoop.hive.metastore.api.Table hmsTable = shell.metastore().getTable("default", "customers");
    List<FieldSchema> icebergSchema = HiveSchemaUtil.convert(icebergTable.schema());
    List<FieldSchema> hmsSchema = hmsTable.getSd().getCols();
    List<FieldSchema> expectedSchema = Lists.newArrayList(new FieldSchema("customer_id", "int", null), new FieldSchema("family_name", "string", "This is family name"));
    Assert.assertEquals(expectedSchema, icebergSchema);
    if (testTableType != TestTables.TestTableType.HIVE_CATALOG) {
        expectedSchema.stream().filter(fs -> fs.getComment() == null).forEach(fs -> fs.setComment("from deserializer"));
    }
    Assert.assertEquals(expectedSchema, hmsSchema);
}

Example 70 with Schema

use of org.apache.iceberg.Schema in project hive by apache.

the class TestHiveIcebergStorageHandlerNoScan method testSetPartitionTransform.

@Test
public void testSetPartitionTransform() {
    Schema schema = new Schema(optional(1, "id", Types.LongType.get()), optional(2, "year_field", Types.DateType.get()), optional(3, "month_field", Types.TimestampType.withZone()), optional(4, "day_field", Types.TimestampType.withoutZone()), optional(5, "hour_field", Types.TimestampType.withoutZone()), optional(6, "truncate_field", Types.StringType.get()), optional(7, "bucket_field", Types.StringType.get()), optional(8, "identity_field", Types.StringType.get()));
    TableIdentifier identifier = TableIdentifier.of("default", "part_test");
    shell.executeStatement("CREATE EXTERNAL TABLE " + identifier + " PARTITIONED BY SPEC (year(year_field), hour(hour_field), " + "truncate(2, truncate_field), bucket(2, bucket_field), identity_field)" + " STORED BY ICEBERG " + testTables.locationForCreateTableSQL(identifier) + "TBLPROPERTIES ('" + InputFormatConfig.TABLE_SCHEMA + "'='" + SchemaParser.toJson(schema) + "', " + "'" + InputFormatConfig.CATALOG_NAME + "'='" + testTables.catalogName() + "')");
    PartitionSpec spec = PartitionSpec.builderFor(schema).year("year_field").hour("hour_field").truncate("truncate_field", 2).bucket("bucket_field", 2).identity("identity_field").build();
    Table table = testTables.loadTable(identifier);
    Assert.assertEquals(spec, table.spec());
    shell.executeStatement("ALTER TABLE default.part_test SET PARTITION SPEC(year(year_field), month(month_field), " + "day(day_field))");
    spec = PartitionSpec.builderFor(schema).withSpecId(1).year("year_field").alwaysNull("hour_field", "hour_field_hour").alwaysNull("truncate_field", "truncate_field_trunc").alwaysNull("bucket_field", "bucket_field_bucket").alwaysNull("identity_field", "identity_field").month("month_field").day("day_field").build();
    table.refresh();
    Assert.assertEquals(spec, table.spec());
}