use of org.apache.flink.api.common.operators.Ordering in project flink by apache.
the class GlobalPropertiesFilteringTest method testRangePartitioningErased.
@Test
public void testRangePartitioningErased() {
SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "1;5" }, null, null, tupleInfo, tupleInfo);
Ordering o = new Ordering();
o.appendOrdering(1, IntValue.class, Order.ASCENDING);
o.appendOrdering(5, LongValue.class, Order.DESCENDING);
o.appendOrdering(2, StringValue.class, Order.ASCENDING);
GlobalProperties gprops = new GlobalProperties();
gprops.setRangePartitioned(o);
GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
assertNull(result.getPartitioningOrdering());
assertNull(result.getPartitioningFields());
}
use of org.apache.flink.api.common.operators.Ordering in project flink by apache.
the class GlobalPropertiesFilteringTest method testRangePartitioningPreserved1.
@Test
public void testRangePartitioningPreserved1() {
SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[] { "1;2;5" }, null, null, tupleInfo, tupleInfo);
Ordering o = new Ordering();
o.appendOrdering(1, IntValue.class, Order.ASCENDING);
o.appendOrdering(5, LongValue.class, Order.DESCENDING);
o.appendOrdering(2, StringValue.class, Order.ASCENDING);
GlobalProperties gprops = new GlobalProperties();
gprops.setRangePartitioned(o);
GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
FieldList pFields = result.getPartitioningFields();
assertEquals(3, pFields.size());
assertEquals(1, pFields.get(0).intValue());
assertEquals(5, pFields.get(1).intValue());
assertEquals(2, pFields.get(2).intValue());
Ordering pOrder = result.getPartitioningOrdering();
assertEquals(3, pOrder.getNumberOfFields());
assertEquals(1, pOrder.getFieldNumber(0).intValue());
assertEquals(5, pOrder.getFieldNumber(1).intValue());
assertEquals(2, pOrder.getFieldNumber(2).intValue());
assertEquals(Order.ASCENDING, pOrder.getOrder(0));
assertEquals(Order.DESCENDING, pOrder.getOrder(1));
assertEquals(Order.ASCENDING, pOrder.getOrder(2));
assertEquals(IntValue.class, pOrder.getType(0));
assertEquals(LongValue.class, pOrder.getType(1));
assertEquals(StringValue.class, pOrder.getType(2));
}
use of org.apache.flink.api.common.operators.Ordering in project flink by apache.
the class CoGroupGlobalPropertiesCompatibilityTest method checkCompatiblePartitionings.
@Test
public void checkCompatiblePartitionings() {
try {
final FieldList keysLeft = new FieldList(1, 4);
final FieldList keysRight = new FieldList(3, 1);
CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
// test compatible hash partitioning
{
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setHashPartitioned(keysLeft);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setHashPartitioned(keysRight);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setHashPartitioned(keysLeft);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setHashPartitioned(keysRight);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
// test compatible custom partitioning
{
Partitioner<Object> part = new Partitioner<Object>() {
@Override
public int partition(Object key, int numPartitions) {
return 0;
}
};
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setCustomPartitioned(keysLeft, part);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setCustomPartitioned(keysRight, part);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setCustomPartitioned(keysLeft, part);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setCustomPartitioned(keysRight, part);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
// test custom partitioning matching any partitioning
{
Partitioner<Object> part = new Partitioner<Object>() {
@Override
public int partition(Object key, int numPartitions) {
return 0;
}
};
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setAnyPartitioning(keysLeft);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setAnyPartitioning(keysRight);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setCustomPartitioned(keysLeft, part);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setCustomPartitioned(keysRight, part);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
TestDistribution dist1 = new TestDistribution(1);
TestDistribution dist2 = new TestDistribution(1);
// test compatible range partitioning with one ordering
{
Ordering ordering1 = new Ordering();
for (int field : keysLeft) {
ordering1.appendOrdering(field, null, Order.ASCENDING);
}
Ordering ordering2 = new Ordering();
for (int field : keysRight) {
ordering2.appendOrdering(field, null, Order.ASCENDING);
}
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setRangePartitioned(ordering1, dist1);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setRangePartitioned(ordering2, dist2);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setRangePartitioned(ordering1, dist1);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setRangePartitioned(ordering2, dist2);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
// test compatible range partitioning with two orderings
{
Ordering ordering1 = new Ordering();
ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
Ordering ordering2 = new Ordering();
ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setRangePartitioned(ordering1, dist1);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setRangePartitioned(ordering2, dist2);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setRangePartitioned(ordering1, dist1);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setRangePartitioned(ordering2, dist2);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.api.common.operators.Ordering in project flink by apache.
the class JoinGlobalPropertiesCompatibilityTest method checkCompatiblePartitionings.
@Test
public void checkCompatiblePartitionings() {
try {
final FieldList keysLeft = new FieldList(1, 4);
final FieldList keysRight = new FieldList(3, 1);
SortMergeInnerJoinDescriptor descr = new SortMergeInnerJoinDescriptor(keysLeft, keysRight);
// test compatible hash partitioning
{
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setHashPartitioned(keysLeft);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setHashPartitioned(keysRight);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setHashPartitioned(keysLeft);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setHashPartitioned(keysRight);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
// test compatible custom partitioning
{
Partitioner<Object> part = new Partitioner<Object>() {
@Override
public int partition(Object key, int numPartitions) {
return 0;
}
};
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setCustomPartitioned(keysLeft, part);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setCustomPartitioned(keysRight, part);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setCustomPartitioned(keysLeft, part);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setCustomPartitioned(keysRight, part);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
// test custom partitioning matching any partitioning
{
Partitioner<Object> part = new Partitioner<Object>() {
@Override
public int partition(Object key, int numPartitions) {
return 0;
}
};
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setAnyPartitioning(keysLeft);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setAnyPartitioning(keysRight);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setCustomPartitioned(keysLeft, part);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setCustomPartitioned(keysRight, part);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
TestDistribution dist1 = new TestDistribution(1);
TestDistribution dist2 = new TestDistribution(1);
// test compatible range partitioning with one ordering
{
Ordering ordering1 = new Ordering();
for (int field : keysLeft) {
ordering1.appendOrdering(field, null, Order.ASCENDING);
}
Ordering ordering2 = new Ordering();
for (int field : keysRight) {
ordering2.appendOrdering(field, null, Order.ASCENDING);
}
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setRangePartitioned(ordering1, dist1);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setRangePartitioned(ordering2, dist2);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setRangePartitioned(ordering1, dist1);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setRangePartitioned(ordering2, dist2);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
// test compatible range partitioning with two orderings
{
Ordering ordering1 = new Ordering();
ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
Ordering ordering2 = new Ordering();
ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
reqLeft.setRangePartitioned(ordering1, dist1);
RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
reqRight.setRangePartitioned(ordering2, dist2);
GlobalProperties propsLeft = new GlobalProperties();
propsLeft.setRangePartitioned(ordering1, dist1);
GlobalProperties propsRight = new GlobalProperties();
propsRight.setRangePartitioned(ordering2, dist2);
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
use of org.apache.flink.api.common.operators.Ordering in project flink by apache.
the class RequestedLocalPropertiesFilteringTest method testOrderErased.
@Test
public void testOrderErased() {
SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[] { "1; 4" }, null, null, tupleInfo, tupleInfo);
Ordering o = new Ordering();
o.appendOrdering(4, LongValue.class, Order.DESCENDING);
o.appendOrdering(1, IntValue.class, Order.ASCENDING);
o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
RequestedLocalProperties rlProp = new RequestedLocalProperties();
rlProp.setOrdering(o);
RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);
assertNull(filtered);
}
Aggregations