Examples with MetadataTableType org.apache.iceberg.MetadataTableType used on opensource projects

Example 6 with MetadataTableType

use of org.apache.iceberg.MetadataTableType in project iceberg by apache.

the class TestTableSerialization method testSerializableMetadataTableKryoSerialization.

@Test
public void testSerializableMetadataTableKryoSerialization() throws IOException {
    for (MetadataTableType type : MetadataTableType.values()) {
        TableOperations ops = ((HasTableOperations) table).operations();
        Table metadataTable = MetadataTableUtils.createMetadataTableInstance(ops, table.name(), "meta", type);
        SerializableTable serializableMetadataTable = (SerializableTable) SerializableTable.copyOf(metadataTable);
        TestHelpers.assertSerializedAndLoadedMetadata(metadataTable, roundTripKryoSerialize(SerializableTable.class, serializableMetadataTable));
    }
}

Example 7 with MetadataTableType

use of org.apache.iceberg.MetadataTableType in project iceberg by apache.

the class TestCachingCatalog method testInvalidateMetadataTablesIfBaseTableIsDropped.

@Test
public void testInvalidateMetadataTablesIfBaseTableIsDropped() throws IOException {
    Catalog catalog = CachingCatalog.wrap(hadoopCatalog());
    // create the original table
    TableIdentifier tableIdent = TableIdentifier.of("db", "ns1", "ns2", "tbl");
    Table table = catalog.createTable(tableIdent, SCHEMA, SPEC, ImmutableMap.of("key2", "value2"));
    table.newAppend().appendFile(FILE_A).commit();
    // remember the original snapshot
    Snapshot oldSnapshot = table.currentSnapshot();
    // populate the cache with metadata tables
    for (MetadataTableType type : MetadataTableType.values()) {
        catalog.loadTable(TableIdentifier.parse(tableIdent + "." + type.name()));
        catalog.loadTable(TableIdentifier.parse(tableIdent + "." + type.name().toLowerCase(Locale.ROOT)));
    }
    // drop the original table
    catalog.dropTable(tableIdent);
    // create a new table with the same name
    table = catalog.createTable(tableIdent, SCHEMA, SPEC, ImmutableMap.of("key2", "value2"));
    table.newAppend().appendFile(FILE_B).commit();
    // remember the new snapshot
    Snapshot newSnapshot = table.currentSnapshot();
    Assert.assertNotEquals("Snapshots must be different", oldSnapshot, newSnapshot);
    // validate metadata tables were correctly invalidated
    for (MetadataTableType type : MetadataTableType.values()) {
        TableIdentifier metadataIdent1 = TableIdentifier.parse(tableIdent + "." + type.name());
        Table metadataTable1 = catalog.loadTable(metadataIdent1);
        Assert.assertEquals("Snapshot must be new", newSnapshot, metadataTable1.currentSnapshot());
        TableIdentifier metadataIdent2 = TableIdentifier.parse(tableIdent + "." + type.name().toLowerCase(Locale.ROOT));
        Table metadataTable2 = catalog.loadTable(metadataIdent2);
        Assert.assertEquals("Snapshot must be new", newSnapshot, metadataTable2.currentSnapshot());
    }
}

Example 8 with MetadataTableType

use of org.apache.iceberg.MetadataTableType in project iceberg by apache.

the class TestTableSerialization method testSerializableMetadataTable.

@Test
public void testSerializableMetadataTable() throws IOException, ClassNotFoundException {
    for (MetadataTableType type : MetadataTableType.values()) {
        Table metadataTable = getMetaDataTable(table, type);
        TestHelpers.assertSerializedAndLoadedMetadata(metadataTable, TestHelpers.roundTripSerialize(metadataTable));
    }
}

Example 9 with MetadataTableType

use of org.apache.iceberg.MetadataTableType in project iceberg by apache.

the class TestTableSerialization method testSerializableMetadataTablesPlanning.

@Test
public void testSerializableMetadataTablesPlanning() throws IOException {
    table.newAppend().appendFile(FILE_A).commit();
    Map<MetadataTableType, byte[]> serialized = Maps.newHashMap();
    Map<MetadataTableType, Set<CharSequence>> expected = Maps.newHashMap();
    for (MetadataTableType type : MetadataTableType.values()) {
        Table metaTable = getMetaDataTable(table, type);
        // Serialize the table
        serialized.put(type, serializeToBytes(metaTable));
        // Collect the expected result
        expected.put(type, getFiles(metaTable));
    }
    table.newAppend().appendFile(FILE_B).commit();
    for (MetadataTableType type : MetadataTableType.values()) {
        // Collect the deserialized data
        Set<CharSequence> deserializedFiles = getFiles(deserializeFromBytes(serialized.get(type)));
        // Checks that the deserialized data stays the same
        Assert.assertEquals(expected.get(type), deserializedFiles);
        // Collect the current data
        Set<CharSequence> newFiles = getFiles(getMetaDataTable(table, type));
        // Expect that the new data is changed in the meantime
        Assert.assertNotEquals(newFiles, deserializedFiles);
    }
}

Example 10 with MetadataTableType

use of org.apache.iceberg.MetadataTableType in project iceberg by apache.

the class TestMetadataTablesWithPartitionEvolution method testFilesMetadataTable.

@Test
public void testFilesMetadataTable() throws ParseException {
    sql("CREATE TABLE %s (id bigint NOT NULL, category string, data string) USING iceberg", tableName);
    initTable();
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables while the current spec is still unpartitioned
    for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
        Dataset<Row> df = loadMetadataTable(tableType);
        Assert.assertTrue("Partition must be skipped", df.schema().getFieldIndex("partition").isEmpty());
    }
    Table table = validationCatalog.loadTable(tableIdent);
    table.updateSpec().addField("data").commit();
    sql("REFRESH TABLE %s", tableName);
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables after adding the first partition column
    for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
        assertPartitions(ImmutableList.of(row(new Object[] { null }), row("b1")), "STRUCT<data:STRING>", tableType);
    }
    table.updateSpec().addField(Expressions.bucket("category", 8)).commit();
    sql("REFRESH TABLE %s", tableName);
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables after adding the second partition column
    for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
        assertPartitions(ImmutableList.of(row(null, null), row("b1", null), row("b1", 2)), "STRUCT<data:STRING,category_bucket_8:INT>", tableType);
    }
    table.updateSpec().removeField("data").commit();
    sql("REFRESH TABLE %s", tableName);
    sql("INSERT INTO TABLE %s VALUES (1, 'a1', 'b1')", tableName);
    // verify the metadata tables after dropping the first partition column
    for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
        assertPartitions(ImmutableList.of(row(null, null), row(null, 2), row("b1", null), row("b1", 2)), "STRUCT<data:STRING,category_bucket_8:INT>", tableType);
    }
    table.updateSpec().renameField("category_bucket_8", "category_bucket_8_another_name").commit();
    sql("REFRESH TABLE %s", tableName);
    // verify the metadata tables after renaming the second partition column
    for (MetadataTableType tableType : Arrays.asList(FILES, ALL_DATA_FILES)) {
        assertPartitions(ImmutableList.of(row(null, null), row(null, 2), row("b1", null), row("b1", 2)), "STRUCT<data:STRING,category_bucket_8_another_name:INT>", tableType);
    }
}