use of org.apache.iceberg.util.StructLikeSet in project hive by apache.
the class DeleteReadTests method testMultipleEqualityDeleteSchemas.
@Test
public void testMultipleEqualityDeleteSchemas() throws IOException {
Schema dataSchema = table.schema().select("data");
Record dataDelete = GenericRecord.create(dataSchema);
List<Record> dataDeletes = Lists.newArrayList(// id = 29
dataDelete.copy("data", "a"), // id = 89
dataDelete.copy("data", "d"), // id = 122
dataDelete.copy("data", "g"));
DeleteFile dataEqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, dataSchema);
Schema idSchema = table.schema().select("id");
Record idDelete = GenericRecord.create(idSchema);
List<Record> idDeletes = Lists.newArrayList(// id = 121
idDelete.copy("id", 121), // id = 29
idDelete.copy("id", 29));
DeleteFile idEqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), idDeletes, idSchema);
table.newRowDelta().addDeletes(dataEqDeletes).addDeletes(idEqDeletes).commit();
StructLikeSet expected = rowSetWithoutIds(29, 89, 121, 122);
StructLikeSet actual = rowSet(tableName, table, "*");
Assert.assertEquals("Table should contain expected rows", expected, actual);
}
use of org.apache.iceberg.util.StructLikeSet in project hive by apache.
the class DeleteReadTests method rowSetWithoutIds.
private StructLikeSet rowSetWithoutIds(int... idsToRemove) {
Set<Integer> deletedIds = Sets.newHashSet(ArrayUtil.toIntList(idsToRemove));
StructLikeSet set = StructLikeSet.create(table.schema().asStruct());
records.stream().filter(row -> !deletedIds.contains(row.getField("id"))).forEach(set::add);
return set;
}
use of org.apache.iceberg.util.StructLikeSet in project hive by apache.
the class DeleteReadTests method testEqualityDeletes.
@Test
public void testEqualityDeletes() throws IOException {
Schema deleteRowSchema = table.schema().select("data");
Record dataDelete = GenericRecord.create(deleteRowSchema);
List<Record> dataDeletes = Lists.newArrayList(// id = 29
dataDelete.copy("data", "a"), // id = 89
dataDelete.copy("data", "d"), // id = 122
dataDelete.copy("data", "g"));
DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, deleteRowSchema);
table.newRowDelta().addDeletes(eqDeletes).commit();
StructLikeSet expected = rowSetWithoutIds(29, 89, 122);
StructLikeSet actual = rowSet(tableName, table, "*");
Assert.assertEquals("Table should contain expected rows", expected, actual);
}
use of org.apache.iceberg.util.StructLikeSet in project hive by apache.
the class DeleteReadTests method testEqualityDeletesWithRequiredEqColumn.
@Test
public void testEqualityDeletesWithRequiredEqColumn() throws IOException {
Schema deleteRowSchema = table.schema().select("data");
Record dataDelete = GenericRecord.create(deleteRowSchema);
List<Record> dataDeletes = Lists.newArrayList(// id = 29
dataDelete.copy("data", "a"), // id = 89
dataDelete.copy("data", "d"), // id = 122
dataDelete.copy("data", "g"));
DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, deleteRowSchema);
table.newRowDelta().addDeletes(eqDeletes).commit();
StructLikeSet expected = selectColumns(rowSetWithoutIds(29, 89, 122), "id");
StructLikeSet actual = rowSet(tableName, table, "id");
if (expectPruned()) {
Assert.assertEquals("Table should contain expected rows", expected, actual);
} else {
// data is added by the reader to apply the eq deletes, use StructProjection to remove it from comparison
Assert.assertEquals("Table should contain expected rows", expected, selectColumns(actual, "id"));
}
}
use of org.apache.iceberg.util.StructLikeSet in project hive by apache.
the class DeleteReadTests method testEqualityDeleteByNull.
@Test
public void testEqualityDeleteByNull() throws IOException {
// data is required in the test table; make it optional for this test
table.updateSchema().makeColumnOptional("data").commit();
// add a new data file with a record where data is null
Record record = GenericRecord.create(table.schema());
DataFile dataFileWithNull = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), Row.of(0), Lists.newArrayList(record.copy("id", 131, "data", null)));
table.newAppend().appendFile(dataFileWithNull).commit();
// delete where data is null
Schema dataSchema = table.schema().select("data");
Record dataDelete = GenericRecord.create(dataSchema);
List<Record> dataDeletes = Lists.newArrayList(// id = 131
dataDelete.copy("data", null));
DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, dataSchema);
table.newRowDelta().addDeletes(eqDeletes).commit();
StructLikeSet expected = rowSetWithoutIds(131);
StructLikeSet actual = rowSet(tableName, table, "*");
Assert.assertEquals("Table should contain expected rows", expected, actual);
}
Aggregations