Examples with WorkloadStat org.apache.hudi.table.WorkloadStat used on opensource projects

Example 11 with WorkloadStat

use of org.apache.hudi.table.WorkloadStat in project hudi by apache.

the class JavaUpsertPartitioner method assignInserts.

private void assignInserts(WorkloadProfile profile, HoodieEngineContext context) {
    // for new inserts, compute buckets depending on how many records we have for each partition
    Set<String> partitionPaths = profile.getPartitionPaths();
    long averageRecordSize = averageBytesPerRecord(table.getMetaClient().getActiveTimeline().getCommitTimeline().filterCompletedInstants(), config);
    LOG.info("AvgRecordSize => " + averageRecordSize);
    Map<String, List<SmallFile>> partitionSmallFilesMap = getSmallFilesForPartitions(new ArrayList<String>(partitionPaths), context);
    for (String partitionPath : partitionPaths) {
        WorkloadStat pStat = profile.getWorkloadStat(partitionPath);
        WorkloadStat outputWorkloadStats = profile.getOutputPartitionPathStatMap().getOrDefault(partitionPath, new WorkloadStat());
        if (pStat.getNumInserts() > 0) {
            List<SmallFile> smallFiles = partitionSmallFilesMap.getOrDefault(partitionPath, new ArrayList<>());
            this.smallFiles.addAll(smallFiles);
            LOG.info("For partitionPath : " + partitionPath + " Small Files => " + smallFiles);
            long totalUnassignedInserts = pStat.getNumInserts();
            List<Integer> bucketNumbers = new ArrayList<>();
            List<Long> recordsPerBucket = new ArrayList<>();
            // first try packing this into one of the smallFiles
            for (SmallFile smallFile : smallFiles) {
                long recordsToAppend = Math.min((config.getParquetMaxFileSize() - smallFile.sizeBytes) / averageRecordSize, totalUnassignedInserts);
                if (recordsToAppend > 0) {
                    // create a new bucket or re-use an existing bucket
                    int bucket;
                    if (updateLocationToBucket.containsKey(smallFile.location.getFileId())) {
                        bucket = updateLocationToBucket.get(smallFile.location.getFileId());
                        LOG.info("Assigning " + recordsToAppend + " inserts to existing update bucket " + bucket);
                    } else {
                        bucket = addUpdateBucket(partitionPath, smallFile.location.getFileId());
                        LOG.info("Assigning " + recordsToAppend + " inserts to new update bucket " + bucket);
                    }
                    if (profile.hasOutputWorkLoadStats()) {
                        outputWorkloadStats.addInserts(smallFile.location, recordsToAppend);
                    }
                    bucketNumbers.add(bucket);
                    recordsPerBucket.add(recordsToAppend);
                    totalUnassignedInserts -= recordsToAppend;
                }
            }
            // if we have anything more, create new insert buckets, like normal
            if (totalUnassignedInserts > 0) {
                long insertRecordsPerBucket = config.getCopyOnWriteInsertSplitSize();
                if (config.shouldAutoTuneInsertSplits()) {
                    insertRecordsPerBucket = config.getParquetMaxFileSize() / averageRecordSize;
                }
                int insertBuckets = (int) Math.ceil((1.0 * totalUnassignedInserts) / insertRecordsPerBucket);
                LOG.info("After small file assignment: unassignedInserts => " + totalUnassignedInserts + ", totalInsertBuckets => " + insertBuckets + ", recordsPerBucket => " + insertRecordsPerBucket);
                for (int b = 0; b < insertBuckets; b++) {
                    bucketNumbers.add(totalBuckets);
                    if (b < insertBuckets - 1) {
                        recordsPerBucket.add(insertRecordsPerBucket);
                    } else {
                        recordsPerBucket.add(totalUnassignedInserts - (insertBuckets - 1) * insertRecordsPerBucket);
                    }
                    BucketInfo bucketInfo = new BucketInfo(BucketType.INSERT, FSUtils.createNewFileIdPfx(), partitionPath);
                    bucketInfoMap.put(totalBuckets, bucketInfo);
                    if (profile.hasOutputWorkLoadStats()) {
                        outputWorkloadStats.addInserts(new HoodieRecordLocation(HoodieWriteStat.NULL_COMMIT, bucketInfo.getFileIdPrefix()), recordsPerBucket.get(recordsPerBucket.size() - 1));
                    }
                    totalBuckets++;
                }
            }
            // Go over all such buckets, and assign weights as per amount of incoming inserts.
            List<InsertBucketCumulativeWeightPair> insertBuckets = new ArrayList<>();
            double currentCumulativeWeight = 0;
            for (int i = 0; i < bucketNumbers.size(); i++) {
                InsertBucket bkt = new InsertBucket();
                bkt.bucketNumber = bucketNumbers.get(i);
                bkt.weight = (1.0 * recordsPerBucket.get(i)) / pStat.getNumInserts();
                currentCumulativeWeight += bkt.weight;
                insertBuckets.add(new InsertBucketCumulativeWeightPair(bkt, currentCumulativeWeight));
            }
            LOG.info("Total insert buckets for partition path " + partitionPath + " => " + insertBuckets);
            partitionPathToInsertBucketInfos.put(partitionPath, insertBuckets);
        }
        if (profile.hasOutputWorkLoadStats()) {
            profile.updateOutputPartitionPathStatMap(partitionPath, outputWorkloadStats);
        }
    }
}

Example 12 with WorkloadStat

use of org.apache.hudi.table.WorkloadStat in project hudi by apache.

the class BaseSparkCommitActionExecutor method buildProfile.

private Pair<HashMap<String, WorkloadStat>, WorkloadStat> buildProfile(HoodieData<HoodieRecord<T>> inputRecords) {
    HashMap<String, WorkloadStat> partitionPathStatMap = new HashMap<>();
    WorkloadStat globalStat = new WorkloadStat();
    // group the records by partitionPath + currentLocation combination, count the number of
    // records in each partition
    Map<Tuple2<String, Option<HoodieRecordLocation>>, Long> partitionLocationCounts = inputRecords.mapToPair(record -> Pair.of(new Tuple2<>(record.getPartitionPath(), Option.ofNullable(record.getCurrentLocation())), record)).countByKey();
    // count the number of both inserts and updates in each partition, update the counts to workLoadStats
    for (Map.Entry<Tuple2<String, Option<HoodieRecordLocation>>, Long> e : partitionLocationCounts.entrySet()) {
        String partitionPath = e.getKey()._1();
        Long count = e.getValue();
        Option<HoodieRecordLocation> locOption = e.getKey()._2();
        if (!partitionPathStatMap.containsKey(partitionPath)) {
            partitionPathStatMap.put(partitionPath, new WorkloadStat());
        }
        if (locOption.isPresent()) {
            // update
            partitionPathStatMap.get(partitionPath).addUpdates(locOption.get(), count);
            globalStat.addUpdates(locOption.get(), count);
        } else {
            // insert
            partitionPathStatMap.get(partitionPath).addInserts(count);
            globalStat.addInserts(count);
        }
    }
    return Pair.of(partitionPathStatMap, globalStat);
}