Examples with Ordering org.apache.flink.api.common.operators.Ordering used on opensource projects

Example 36 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class CoGroupGlobalPropertiesCompatibilityTest method checkInompatiblePartitionings.

@Test
public void checkInompatiblePartitionings() {
    try {
        final FieldList keysLeft = new FieldList(1);
        final FieldList keysRight = new FieldList(3);
        final Partitioner<Object> part = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        final Partitioner<Object> part2 = new Partitioner<Object>() {

            @Override
            public int partition(Object key, int numPartitions) {
                return 0;
            }
        };
        CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
        // test incompatible hash with custom partitioning
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setHashPartitioned(keysLeft);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible custom partitionings
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist1 = new TestDistribution(1);
        TestDistribution dist2 = new TestDistribution(1);
        // test incompatible range partitioning with different key size
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test incompatible range partitioning with different ordering
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.DESCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist3 = new TestDistribution(1);
        TestDistribution dist4 = new TestDistribution(2);
        // test incompatible range partitioning with different distribution
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist3);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist4);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist3);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist4);
            assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 37 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class LocalPropertiesFilteringTest method testSortingPreserved1.

@Test
public void testSortingPreserved1() {
    SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[] { "0;2;5" }, null, null, tupleInfo, tupleInfo);
    Ordering o = new Ordering();
    o.appendOrdering(2, IntValue.class, Order.ASCENDING);
    o.appendOrdering(0, StringValue.class, Order.DESCENDING);
    o.appendOrdering(5, LongValue.class, Order.DESCENDING);
    LocalProperties lProps = LocalProperties.forOrdering(o);
    LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
    FieldList gFields = filtered.getGroupedFields();
    Ordering order = filtered.getOrdering();
    assertNotNull(gFields);
    assertEquals(3, gFields.size());
    assertTrue(gFields.contains(0));
    assertTrue(gFields.contains(2));
    assertTrue(gFields.contains(5));
    assertNotNull(order);
    assertEquals(3, order.getNumberOfFields());
    assertEquals(2, order.getFieldNumber(0).intValue());
    assertEquals(0, order.getFieldNumber(1).intValue());
    assertEquals(5, order.getFieldNumber(2).intValue());
    assertEquals(Order.ASCENDING, order.getOrder(0));
    assertEquals(Order.DESCENDING, order.getOrder(1));
    assertEquals(Order.DESCENDING, order.getOrder(2));
    assertEquals(IntValue.class, order.getType(0));
    assertEquals(StringValue.class, order.getType(1));
    assertEquals(LongValue.class, order.getType(2));
    assertNull(filtered.getUniqueFields());
}

Example 38 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class LocalPropertiesFilteringTest method testSortingPreserved4.

@Test
public void testSortingPreserved4() {
    SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[] { "2->7;5" }, null, null, tupleInfo, tupleInfo);
    Ordering o = new Ordering();
    o.appendOrdering(2, IntValue.class, Order.ASCENDING);
    o.appendOrdering(0, StringValue.class, Order.DESCENDING);
    o.appendOrdering(5, LongValue.class, Order.DESCENDING);
    LocalProperties lProps = LocalProperties.forOrdering(o);
    LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
    FieldList gFields = filtered.getGroupedFields();
    Ordering order = filtered.getOrdering();
    assertNotNull(gFields);
    assertEquals(1, gFields.size());
    assertTrue(gFields.contains(7));
    assertNotNull(order);
    assertEquals(1, order.getNumberOfFields());
    assertEquals(7, order.getFieldNumber(0).intValue());
    assertEquals(Order.ASCENDING, order.getOrder(0));
    assertEquals(IntValue.class, order.getType(0));
    assertNull(filtered.getUniqueFields());
}

Example 39 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class RequestedGlobalPropertiesFilteringTest method testRangePartitioningPreserved1.

@Test
public void testRangePartitioningPreserved1() {
    SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[] { "1;3;6" }, null, null, tupleInfo, tupleInfo);
    Ordering o = new Ordering();
    o.appendOrdering(3, LongValue.class, Order.DESCENDING);
    o.appendOrdering(1, IntValue.class, Order.ASCENDING);
    o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
    RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
    rgProps.setRangePartitioned(o);
    RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
    assertNotNull(filtered);
    assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
    assertNotNull(filtered.getOrdering());
    assertEquals(3, filtered.getOrdering().getNumberOfFields());
    assertEquals(3, filtered.getOrdering().getFieldNumber(0).intValue());
    assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
    assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
    assertEquals(LongValue.class, filtered.getOrdering().getType(0));
    assertEquals(IntValue.class, filtered.getOrdering().getType(1));
    assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
    assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
    assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
    assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
    assertNull(filtered.getPartitionedFields());
    assertNull(filtered.getDataDistribution());
    assertNull(filtered.getCustomPartitioner());
}

Example 40 with Ordering

use of org.apache.flink.api.common.operators.Ordering in project flink by apache.

the class RequestedGlobalPropertiesFilteringTest method testRangePartitioningPreserved2.

@Test
public void testRangePartitioningPreserved2() {
    SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
    SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[] { "7->3;1->1;2->6" }, null, null, tupleInfo, tupleInfo);
    Ordering o = new Ordering();
    o.appendOrdering(3, LongValue.class, Order.DESCENDING);
    o.appendOrdering(1, IntValue.class, Order.ASCENDING);
    o.appendOrdering(6, ByteValue.class, Order.DESCENDING);
    RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
    rgProps.setRangePartitioned(o);
    RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);
    assertNotNull(filtered);
    assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
    assertNotNull(filtered.getOrdering());
    assertEquals(3, filtered.getOrdering().getNumberOfFields());
    assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
    assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
    assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
    assertEquals(LongValue.class, filtered.getOrdering().getType(0));
    assertEquals(IntValue.class, filtered.getOrdering().getType(1));
    assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
    assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
    assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
    assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
    assertNull(filtered.getPartitionedFields());
    assertNull(filtered.getDataDistribution());
    assertNull(filtered.getCustomPartitioner());
}