Examples with DeleteFile org.apache.iceberg.DeleteFile used on opensource projects

Example 11 with DeleteFile

use of org.apache.iceberg.DeleteFile in project hive by apache.

the class DeleteReadTests method testMultipleEqualityDeleteSchemas.

@Test
public void testMultipleEqualityDeleteSchemas() throws IOException {
    Schema dataSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(dataSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile dataEqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, dataSchema);
    Schema idSchema = table.schema().select("id");
    Record idDelete = GenericRecord.create(idSchema);
    List<Record> idDeletes = Lists.newArrayList(// id = 121
    idDelete.copy("id", 121), // id = 29
    idDelete.copy("id", 29));
    DeleteFile idEqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), idDeletes, idSchema);
    table.newRowDelta().addDeletes(dataEqDeletes).addDeletes(idEqDeletes).commit();
    StructLikeSet expected = rowSetWithoutIds(29, 89, 121, 122);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}

Example 12 with DeleteFile

use of org.apache.iceberg.DeleteFile in project hive by apache.

the class DeleteReadTests method testEqualityDeletes.

@Test
public void testEqualityDeletes() throws IOException {
    Schema deleteRowSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(deleteRowSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, deleteRowSchema);
    table.newRowDelta().addDeletes(eqDeletes).commit();
    StructLikeSet expected = rowSetWithoutIds(29, 89, 122);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}

Example 13 with DeleteFile

use of org.apache.iceberg.DeleteFile in project hive by apache.

the class DeleteReadTests method testEqualityDeletesWithRequiredEqColumn.

@Test
public void testEqualityDeletesWithRequiredEqColumn() throws IOException {
    Schema deleteRowSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(deleteRowSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, deleteRowSchema);
    table.newRowDelta().addDeletes(eqDeletes).commit();
    StructLikeSet expected = selectColumns(rowSetWithoutIds(29, 89, 122), "id");
    StructLikeSet actual = rowSet(tableName, table, "id");
    if (expectPruned()) {
        Assert.assertEquals("Table should contain expected rows", expected, actual);
    } else {
        // data is added by the reader to apply the eq deletes, use StructProjection to remove it from comparison
        Assert.assertEquals("Table should contain expected rows", expected, selectColumns(actual, "id"));
    }
}

Example 14 with DeleteFile

use of org.apache.iceberg.DeleteFile in project hive by apache.

the class DeleteReadTests method testEqualityDeleteByNull.

@Test
public void testEqualityDeleteByNull() throws IOException {
    // data is required in the test table; make it optional for this test
    table.updateSchema().makeColumnOptional("data").commit();
    // add a new data file with a record where data is null
    Record record = GenericRecord.create(table.schema());
    DataFile dataFileWithNull = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), Row.of(0), Lists.newArrayList(record.copy("id", 131, "data", null)));
    table.newAppend().appendFile(dataFileWithNull).commit();
    // delete where data is null
    Schema dataSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(dataSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 131
    dataDelete.copy("data", null));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, dataSchema);
    table.newRowDelta().addDeletes(eqDeletes).commit();
    StructLikeSet expected = rowSetWithoutIds(131);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}