Examples with StructLikeSet org.apache.iceberg.util.StructLikeSet used on opensource projects

Example 1 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class TestInputFormatReaderDeletes method rowSet.

@Override
public StructLikeSet rowSet(String name, Table table, String... columns) {
    InputFormatConfig.ConfigBuilder builder = new InputFormatConfig.ConfigBuilder(conf).readFrom(table.location());
    Schema projected = table.schema().select(columns);
    StructLikeSet set = StructLikeSet.create(projected.asStruct());
    set.addAll(TestIcebergInputFormats.TESTED_INPUT_FORMATS.stream().filter(recordFactory -> recordFactory.name().equals(inputFormat)).map(recordFactory -> recordFactory.create(builder.project(projected).conf()).getRecords()).flatMap(List::stream).map(record -> new InternalRecordWrapper(projected.asStruct()).wrap(record)).collect(Collectors.toList()));
    return set;
}

Example 2 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method testPositionDeletes.

@Test
public void testPositionDeletes() throws IOException {
    List<Pair<CharSequence, Long>> deletes = Lists.newArrayList(// id = 29
    Pair.of(dataFile.path(), 0L), // id = 89
    Pair.of(dataFile.path(), 3L), // id = 122
    Pair.of(dataFile.path(), 6L));
    Pair<DeleteFile, Set<CharSequence>> posDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), deletes);
    table.newRowDelta().addDeletes(posDeletes.first()).validateDataFilesExist(posDeletes.second()).commit();
    StructLikeSet expected = rowSetWithoutIds(29, 89, 122);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}

Example 3 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method selectColumns.

private StructLikeSet selectColumns(StructLikeSet rows, String... columns) {
    Schema projection = table.schema().select(columns);
    StructLikeSet set = StructLikeSet.create(projection.asStruct());
    rows.stream().map(row -> StructProjection.create(table.schema(), projection).wrap(row)).forEach(set::add);
    return set;
}

Example 4 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method testMixedPositionAndEqualityDeletes.

@Test
public void testMixedPositionAndEqualityDeletes() throws IOException {
    Schema dataSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(dataSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, dataSchema);
    List<Pair<CharSequence, Long>> deletes = Lists.newArrayList(// id = 89
    Pair.of(dataFile.path(), 3L), // id = 121
    Pair.of(dataFile.path(), 5L));
    Pair<DeleteFile, Set<CharSequence>> posDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), deletes);
    table.newRowDelta().addDeletes(eqDeletes).addDeletes(posDeletes.first()).validateDataFilesExist(posDeletes.second()).commit();
    StructLikeSet expected = rowSetWithoutIds(29, 89, 121, 122);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}

Example 5 with StructLikeSet

use of org.apache.iceberg.util.StructLikeSet in project hive by apache.

the class DeleteReadTests method testEqualityDeletesSpanningMultipleDataFiles.

@Test
public void testEqualityDeletesSpanningMultipleDataFiles() throws IOException {
    // Add another DataFile with common values
    GenericRecord record = GenericRecord.create(table.schema());
    records.add(record.copy("id", 144, "data", "a"));
    this.dataFile = FileHelpers.writeDataFile(table, Files.localOutput(temp.newFile()), Row.of(0), records);
    table.newAppend().appendFile(dataFile).commit();
    Schema deleteRowSchema = table.schema().select("data");
    Record dataDelete = GenericRecord.create(deleteRowSchema);
    List<Record> dataDeletes = Lists.newArrayList(// id = 29, 144
    dataDelete.copy("data", "a"), // id = 89
    dataDelete.copy("data", "d"), // id = 122
    dataDelete.copy("data", "g"));
    DeleteFile eqDeletes = FileHelpers.writeDeleteFile(table, Files.localOutput(temp.newFile()), Row.of(0), dataDeletes, deleteRowSchema);
    table.newRowDelta().addDeletes(eqDeletes).commit();
    StructLikeSet expected = rowSetWithoutIds(29, 89, 122, 144);
    StructLikeSet actual = rowSet(tableName, table, "*");
    Assert.assertEquals("Table should contain expected rows", expected, actual);
}