use of org.apache.iceberg.data.GenericAppenderFactory in project hive by apache.
the class HiveIcebergTestUtils method createEqualityDeleteFile.
/**
* @param table The table to create the delete file for
* @param deleteFilePath The path where the delete file should be created, relative to the table location root
* @param equalityFields List of field names that should play a role in the equality check
* @param fileFormat The file format that should be used for writing out the delete file
* @param rowsToDelete The rows that should be deleted. It's enough to fill out the fields that are relevant for the
* equality check, as listed in equalityFields, the rest of the fields are ignored
* @return The DeleteFile created
* @throws IOException If there is an error during DeleteFile write
*/
public static DeleteFile createEqualityDeleteFile(Table table, String deleteFilePath, List<String> equalityFields, FileFormat fileFormat, List<Record> rowsToDelete) throws IOException {
List<Integer> equalityFieldIds = equalityFields.stream().map(id -> table.schema().findField(id).fieldId()).collect(Collectors.toList());
Schema eqDeleteRowSchema = table.schema().select(equalityFields.toArray(new String[] {}));
FileAppenderFactory<Record> appenderFactory = new GenericAppenderFactory(table.schema(), table.spec(), ArrayUtil.toIntArray(equalityFieldIds), eqDeleteRowSchema, null);
EncryptedOutputFile outputFile = table.encryption().encrypt(HadoopOutputFile.fromPath(new org.apache.hadoop.fs.Path(table.location(), deleteFilePath), new Configuration()));
PartitionKey part = new PartitionKey(table.spec(), eqDeleteRowSchema);
part.partition(rowsToDelete.get(0));
EqualityDeleteWriter<Record> eqWriter = appenderFactory.newEqDeleteWriter(outputFile, fileFormat, part);
try (EqualityDeleteWriter<Record> writer = eqWriter) {
writer.deleteAll(rowsToDelete);
}
return eqWriter.toDeleteFile();
}
use of org.apache.iceberg.data.GenericAppenderFactory in project hive by apache.
the class HiveIcebergTestUtils method createPositionalDeleteFile.
/**
* @param table The table to create the delete file for
* @param deleteFilePath The path where the delete file should be created, relative to the table location root
* @param fileFormat The file format that should be used for writing out the delete file
* @param partitionValues A map of partition values (partitionKey=partitionVal, ...) to be used for the delete file
* @param deletes The list of position deletes, each containing the data file path, the position of the row in the
* data file and the row itself that should be deleted
* @return The DeleteFile created
* @throws IOException If there is an error during DeleteFile write
*/
public static DeleteFile createPositionalDeleteFile(Table table, String deleteFilePath, FileFormat fileFormat, Map<String, Object> partitionValues, List<PositionDelete<Record>> deletes) throws IOException {
Schema posDeleteRowSchema = deletes.get(0).row() == null ? null : table.schema();
FileAppenderFactory<Record> appenderFactory = new GenericAppenderFactory(table.schema(), table.spec(), null, null, posDeleteRowSchema);
EncryptedOutputFile outputFile = table.encryption().encrypt(HadoopOutputFile.fromPath(new org.apache.hadoop.fs.Path(table.location(), deleteFilePath), new Configuration()));
PartitionKey partitionKey = null;
if (partitionValues != null) {
Record record = GenericRecord.create(table.schema()).copy(partitionValues);
partitionKey = new PartitionKey(table.spec(), table.schema());
partitionKey.partition(record);
}
PositionDeleteWriter<Record> posWriter = appenderFactory.newPosDeleteWriter(outputFile, fileFormat, partitionKey);
try (PositionDeleteWriter<Record> writer = posWriter) {
deletes.forEach(del -> writer.delete(del.path(), del.pos(), del.row()));
}
return posWriter.toDeleteFile();
}
use of org.apache.iceberg.data.GenericAppenderFactory in project drill by apache.
the class IcebergQueriesTest method writeAndCommitDataFile.
private static void writeAndCommitDataFile(Table table, String name, FileFormat fileFormat, Iterable<Record> records) throws IOException {
OutputFile outputFile = table.io().newOutputFile(new Path(table.location(), fileFormat.addExtension(name)).toUri().getPath());
FileAppender<Record> fileAppender = new GenericAppenderFactory(table.schema()).newAppender(outputFile, fileFormat);
fileAppender.addAll(records);
fileAppender.close();
DataFile dataFile = DataFiles.builder(table.spec()).withInputFile(outputFile.toInputFile()).withMetrics(fileAppender.metrics()).build();
Transaction transaction = table.newTransaction();
transaction.newAppend().appendFile(dataFile).commit();
transaction.commitTransaction();
}
Aggregations