use of org.apache.spark.sql.connector.distributions.ClusteredDistribution in project iceberg by apache.
the class SparkDistributionAndOrderingUtil method buildPositionMergeDistribution.
private static Distribution buildPositionMergeDistribution(Table table, DistributionMode distributionMode) {
switch(distributionMode) {
case NONE:
return Distributions.unspecified();
case HASH:
if (table.spec().isUnpartitioned()) {
Expression[] clustering = new Expression[] { SPEC_ID, PARTITION, FILE_PATH };
return Distributions.clustered(clustering);
} else {
Distribution dataDistribution = buildRequiredDistribution(table, distributionMode);
Expression[] dataClustering = ((ClusteredDistribution) dataDistribution).clustering();
Expression[] deleteClustering = new Expression[] { SPEC_ID, PARTITION };
Expression[] clustering = ObjectArrays.concat(deleteClustering, dataClustering, Expression.class);
return Distributions.clustered(clustering);
}
case RANGE:
Distribution dataDistribution = buildRequiredDistribution(table, distributionMode);
SortOrder[] dataOrdering = ((OrderedDistribution) dataDistribution).ordering();
SortOrder[] deleteOrdering = new SortOrder[] { SPEC_ID_ORDER, PARTITION_ORDER, FILE_PATH_ORDER };
SortOrder[] ordering = ObjectArrays.concat(deleteOrdering, dataOrdering, SortOrder.class);
return Distributions.ordered(ordering);
default:
throw new IllegalArgumentException("Unexpected distribution mode: " + distributionMode);
}
}
Aggregations