Examples with OutputEmitter org.apache.flink.runtime.operators.shipping.OutputEmitter used on opensource projects

Example 6 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testNullKey.

@Test
public void testNullKey() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    Record rec = new Record(2);
    rec.setField(1, new IntValue(1));
    delegate.setInstance(rec);
    try {
        oe1.selectChannels(delegate, 100);
    } catch (NullKeyFieldException re) {
        Assert.assertEquals(0, re.getFieldNumber());
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}

Example 7 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testMultiKeys.

@Test
public void testMultiKeys() {
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> multiComp = new RecordComparatorFactory(new int[] { 0, 1, 3 }, new Class[] { IntValue.class, StringValue.class, DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, multiComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChannels = 100;
    int numRecords = 5000;
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        Record rec = new Record(4);
        rec.setField(0, new IntValue(i));
        rec.setField(1, new StringValue("AB" + i + "CD" + i));
        rec.setField(3, new DoubleValue(i * 3.141d));
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    int cnt = 0;
    for (int aHit : hit) {
        assertTrue(aHit > 0);
        cnt += aHit;
    }
    assertTrue(cnt == numRecords);
}

Example 8 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testBroadcast.

@Test
public void testBroadcast() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChannels = 100;
    int numRecords = 50000;
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    for (int aHit : hit) {
        assertTrue(aHit + "", aHit == numRecords);
    }
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, stringComp);
    numChannels = 100;
    numRecords = 5000;
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    for (int aHit : hit) {
        assertTrue(aHit + "", aHit == numRecords);
    }
}