Examples with MetadataTableType org.apache.iceberg.MetadataTableType used on opensource projects

Example 11 with MetadataTableType

use of org.apache.iceberg.MetadataTableType in project iceberg by apache.

the class TestMetadataTablesWithPartitionEvolution method testEntriesMetadataTable.

@Test
public void testEntriesMetadataTable() throws ParseException {
    sql("CREATE TABLE %s (id bigint NOT NULL, category string, data string) USING iceberg", tableName);
    initTable();
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables while the current spec is still unpartitioned
    for (MetadataTableType tableType : Arrays.asList(ENTRIES, ALL_ENTRIES)) {
        Dataset<Row> df = loadMetadataTable(tableType);
        StructType dataFileType = (StructType) df.schema().apply("data_file").dataType();
        Assert.assertTrue("Partition must be skipped", dataFileType.getFieldIndex("").isEmpty());
    }
    Table table = validationCatalog.loadTable(tableIdent);
    table.updateSpec().addField("data").commit();
    sql("REFRESH TABLE %s", tableName);
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables after adding the first partition column
    for (MetadataTableType tableType : Arrays.asList(ENTRIES, ALL_ENTRIES)) {
        assertPartitions(ImmutableList.of(row(new Object[] { null }), row("b1")), "STRUCT<data:STRING>", tableType);
    }
    table.updateSpec().addField(Expressions.bucket("category", 8)).commit();
    sql("REFRESH TABLE %s", tableName);
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables after adding the second partition column
    for (MetadataTableType tableType : Arrays.asList(ENTRIES, ALL_ENTRIES)) {
        assertPartitions(ImmutableList.of(row(null, null), row("b1", null), row("b1", 2)), "STRUCT<data:STRING,category_bucket_8:INT>", tableType);
    }
    table.updateSpec().removeField("data").commit();
    sql("REFRESH TABLE %s", tableName);
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables after dropping the first partition column
    for (MetadataTableType tableType : Arrays.asList(ENTRIES, ALL_ENTRIES)) {
        assertPartitions(ImmutableList.of(row(null, null), row(null, 2), row("b1", null), row("b1", 2)), "STRUCT<data:STRING,category_bucket_8:INT>", tableType);
    }
    table.updateSpec().renameField("category_bucket_8", "category_bucket_8_another_name").commit();
    sql("REFRESH TABLE %s", tableName);
    // verify the metadata tables after renaming the second partition column
    for (MetadataTableType tableType : Arrays.asList(ENTRIES, ALL_ENTRIES)) {
        assertPartitions(ImmutableList.of(row(null, null), row(null, 2), row("b1", null), row("b1", 2)), "STRUCT<data:STRING,category_bucket_8_another_name:INT>", tableType);
    }
}

Example 12 with MetadataTableType

use of org.apache.iceberg.MetadataTableType in project iceberg by apache.

the class HadoopTables method parseMetadataType.

/**
 * Try to resolve a metadata table, which we encode as URI fragments
 * e.g. hdfs:///warehouse/my_table#snapshots
 * @param location Path to parse
 * @return A base table name and MetadataTableType if a type is found, null if not
 */
private Pair<String, MetadataTableType> parseMetadataType(String location) {
    int hashIndex = location.lastIndexOf('#');
    if (hashIndex != -1 && !location.endsWith("#")) {
        String baseTable = location.substring(0, hashIndex);
        String metaTable = location.substring(hashIndex + 1);
        MetadataTableType type = MetadataTableType.from(metaTable);
        return (type == null) ? null : Pair.of(baseTable, type);
    } else {
        return null;
    }
}