Search in sources :

Example 6 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method testMultipleEqualityDeleteSchemas.

@Test
public void testMultipleEqualityDeleteSchemas() throws IOException {
    Schema dataSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(dataSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile dataEqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, dataSchema);
    Schema idSchema = table.schema().select("id");
    Record idDelete = GenericRecord.create(idSchema);
    List<Record> idDeletes = Lists.newArrayList(// id = 121
    idDelete.copy("id", 121), // id = 29
    idDelete.copy("id", 29));
    DeleteFile idEqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), idDeletes, idSchema);
    table.newRowDelta().addDeletes(dataEqDeletes).addDeletes(idEqDeletes).commit();
    StructLikeSet expected = rowSetWithoutIds(29, 89, 121, 122);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}
Also used : Schema(org.apache.iceberg.Schema) StructLikeSet(org.apache.iceberg.util.StructLikeSet) DeleteFile(org.apache.iceberg.DeleteFile) Test(org.junit.Test)

Example 7 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method rowSetWithoutIds.

private StructLikeSet rowSetWithoutIds(int... idsToRemove) {
    Set<Integer> deletedIds = Sets.newHashSet(ArrayUtil.toIntList(idsToRemove));
    StructLikeSet set = StructLikeSet.create(table.schema().asStruct());
    records.stream().filter(row -> !deletedIds.contains(row.getField("id"))).forEach(set::add);
    return set;
}
Also used : Types(org.apache.iceberg.types.Types) Table(org.apache.iceberg.Table) StructLikeSet(org.apache.iceberg.util.StructLikeSet) Set(java.util.Set) Pair(org.apache.iceberg.util.Pair) IOException(java.io.IOException) Test(org.junit.Test) Schema(org.apache.iceberg.Schema) Row(org.apache.iceberg.TestHelpers.Row) Sets(org.apache.iceberg.relocated.com.google.common.collect.Sets) List(java.util.List) Lists(org.apache.iceberg.relocated.com.google.common.collect.Lists) Rule(org.junit.Rule) ArrayUtil(org.apache.iceberg.util.ArrayUtil) After(org.junit.After) PartitionSpec(org.apache.iceberg.PartitionSpec) DeleteFile(org.apache.iceberg.DeleteFile) DataFile(org.apache.iceberg.DataFile) Assert(org.junit.Assert) StructProjection(org.apache.iceberg.util.StructProjection) TemporaryFolder(org.junit.rules.TemporaryFolder) Files(org.apache.iceberg.Files) Before(org.junit.Before) StructLikeSet(org.apache.iceberg.util.StructLikeSet)

Example 8 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method testEqualityDeletes.

@Test
public void testEqualityDeletes() throws IOException {
    Schema deleteRowSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(deleteRowSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, deleteRowSchema);
    table.newRowDelta().addDeletes(eqDeletes).commit();
    StructLikeSet expected = rowSetWithoutIds(29, 89, 122);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}
Also used : Schema(org.apache.iceberg.Schema) StructLikeSet(org.apache.iceberg.util.StructLikeSet) DeleteFile(org.apache.iceberg.DeleteFile) Test(org.junit.Test)

Example 9 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method testEqualityDeletesWithRequiredEqColumn.

@Test
public void testEqualityDeletesWithRequiredEqColumn() throws IOException {
    Schema deleteRowSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(deleteRowSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, deleteRowSchema);
    table.newRowDelta().addDeletes(eqDeletes).commit();
    StructLikeSet expected = selectColumns(rowSetWithoutIds(29, 89, 122), "id");
    StructLikeSet actual = rowSet(tableName, table, "id");
    if (expectPruned()) {
        Assert.assertEquals("Table should contain expected rows", expected, actual);
    } else {
        // data is added by the reader to apply the eq deletes, use StructProjection to remove it from comparison
        Assert.assertEquals("Table should contain expected rows", expected, selectColumns(actual, "id"));
    }
}
Also used : Schema(org.apache.iceberg.Schema) StructLikeSet(org.apache.iceberg.util.StructLikeSet) DeleteFile(org.apache.iceberg.DeleteFile) Test(org.junit.Test)

Example 10 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method testEqualityDeleteByNull.

@Test
public void testEqualityDeleteByNull() throws IOException {
    // data is required in the test table; make it optional for this test
    table.updateSchema().makeColumnOptional("data").commit();
    // add a new data file with a record where data is null
    Record record = GenericRecord.create(table.schema());
    DataFile dataFileWithNull = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), Row.of(0), Lists.newArrayList(record.copy("id", 131, "data", null)));
    table.newAppend().appendFile(dataFileWithNull).commit();
    // delete where data is null
    Schema dataSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(dataSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 131
    dataDelete.copy("data", null));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, dataSchema);
    table.newRowDelta().addDeletes(eqDeletes).commit();
    StructLikeSet expected = rowSetWithoutIds(131);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}
Also used : DataFile(org.apache.iceberg.DataFile) Schema(org.apache.iceberg.Schema) StructLikeSet(org.apache.iceberg.util.StructLikeSet) DeleteFile(org.apache.iceberg.DeleteFile) Test(org.junit.Test)

Aggregations

StructLikeSet (org.apache.iceberg.util.StructLikeSet)10 DeleteFile (org.apache.iceberg.DeleteFile)9 Schema (org.apache.iceberg.Schema)9 Test (org.junit.Test)9 Set (java.util.Set)4 Pair (org.apache.iceberg.util.Pair)4 IOException (java.io.IOException)3 List (java.util.List)3 DataFile (org.apache.iceberg.DataFile)3 PartitionSpec (org.apache.iceberg.PartitionSpec)3 Table (org.apache.iceberg.Table)3 Assert (org.junit.Assert)3 Before (org.junit.Before)3 Files (org.apache.iceberg.Files)2 Row (org.apache.iceberg.TestHelpers.Row)2 Lists (org.apache.iceberg.relocated.com.google.common.collect.Lists)2 Sets (org.apache.iceberg.relocated.com.google.common.collect.Sets)2 Types (org.apache.iceberg.types.Types)2 ArrayUtil (org.apache.iceberg.util.ArrayUtil)2 StructProjection (org.apache.iceberg.util.StructProjection)2