Examples with HoodieIndex org.apache.hudi.index.HoodieIndex used on opensource projects

Example 6 with HoodieIndex

use of org.apache.hudi.index.HoodieIndex in project hudi by apache.

the class TestCleaner method insertFirstBigBatchForClientCleanerTest.

/**
 * Helper method to do first batch of insert for clean by versions/commits tests.
 *
 * @param cfg Hoodie Write Config
 * @param client Hoodie Client
 * @param recordGenFunction Function to generate records for insertion
 * @param insertFn Insertion API for testing
 * @throws Exception in case of error
 */
private Pair<String, JavaRDD<WriteStatus>> insertFirstBigBatchForClientCleanerTest(HoodieWriteConfig cfg, SparkRDDWriteClient client, Function2<List<HoodieRecord>, String, Integer> recordGenFunction, Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> insertFn, HoodieCleaningPolicy cleaningPolicy) throws Exception {
    /*
     * do a big insert (this is basically same as insert part of upsert, just adding it here so we can catch breakages
     * in insert(), if the implementation diverges.)
     */
    String newCommitTime = client.startCommit();
    List<HoodieRecord> records = recordGenFunction.apply(newCommitTime, BIG_BATCH_INSERT_SIZE);
    JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 5);
    JavaRDD<WriteStatus> statuses = insertFn.apply(client, writeRecords, newCommitTime);
    // Verify there are no errors
    assertNoWriteErrors(statuses.collect());
    // verify that there is a commit
    metaClient = HoodieTableMetaClient.reload(metaClient);
    HoodieTimeline timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline();
    assertEquals(1, timeline.findInstantsAfter("000", Integer.MAX_VALUE).countInstants(), "Expecting a single commit.");
    // Should have 100 records in table (check using Index), all in locations marked at commit
    HoodieTable table = HoodieSparkTable.create(client.getConfig(), context, metaClient);
    if (client.getConfig().shouldAutoCommit()) {
        assertFalse(table.getCompletedCommitsTimeline().empty());
    }
    // We no longer write empty cleaner plans when there is nothing to be cleaned.
    assertTrue(table.getCompletedCleanTimeline().empty());
    if (client.getConfig().shouldAutoCommit()) {
        HoodieIndex index = SparkHoodieIndexFactory.createIndex(cfg);
        List<HoodieRecord> taggedRecords = tagLocation(index, jsc.parallelize(records, 1), table).collect();
        checkTaggedRecords(taggedRecords, newCommitTime);
    }
    return Pair.of(newCommitTime, statuses);
}

Example 7 with HoodieIndex

use of org.apache.hudi.index.HoodieIndex in project hudi by apache.

the class FlinkWriteHelper method deduplicateRecords.

@Override
public List<HoodieRecord<T>> deduplicateRecords(List<HoodieRecord<T>> records, HoodieIndex<?, ?> index, int parallelism) {
    Map<Object, List<Pair<Object, HoodieRecord<T>>>> keyedRecords = records.stream().map(record -> {
        // If index used is global, then records are expected to differ in their partitionPath
        final Object key = record.getKey().getRecordKey();
        return Pair.of(key, record);
    }).collect(Collectors.groupingBy(Pair::getLeft));
    return keyedRecords.values().stream().map(x -> x.stream().map(Pair::getRight).reduce((rec1, rec2) -> {
        final T data1 = rec1.getData();
        final T data2 = rec2.getData();
        @SuppressWarnings("unchecked") final T reducedData = (T) data2.preCombine(data1);
        // we cannot allow the user to change the key or partitionPath, since that will affect
        // everything
        // so pick it from one of the records.
        boolean choosePrev = data1.equals(reducedData);
        HoodieKey reducedKey = choosePrev ? rec1.getKey() : rec2.getKey();
        HoodieOperation operation = choosePrev ? rec1.getOperation() : rec2.getOperation();
        HoodieRecord<T> hoodieRecord = new HoodieAvroRecord<>(reducedKey, reducedData, operation);
        // reuse the location from the first record.
        hoodieRecord.setCurrentLocation(rec1.getCurrentLocation());
        return hoodieRecord;
    }).orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
}