Examples with OutputEmitter org.apache.flink.runtime.operators.shipping.OutputEmitter used on opensource projects

Example 1 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testForcedRebalance.

@Test
public void testForcedRebalance() {
    // Test for IntValue
    int numChannels = 100;
    int toTaskIndex = numChannels * 6 / 7;
    int fromTaskIndex = toTaskIndex + numChannels;
    int extraRecords = numChannels / 3;
    int numRecords = 50000 + extraRecords;
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
        if (toTaskIndex <= i || i < toTaskIndex + extraRecords - numChannels) {
            assertTrue(hit[i] == (numRecords / numChannels) + 1);
        } else {
            assertTrue(hit[i] == numRecords / numChannels);
        }
        cnt += hit[i];
    }
    assertTrue(cnt == numRecords);
    // Test for StringValue
    numChannels = 100;
    toTaskIndex = numChannels / 5;
    fromTaskIndex = toTaskIndex + 2 * numChannels;
    extraRecords = numChannels * 2 / 9;
    numRecords = 10000 + extraRecords;
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    cnt = 0;
    for (int i = 0; i < hit.length; i++) {
        if (toTaskIndex <= i && i < toTaskIndex + extraRecords) {
            assertTrue(hit[i] == (numRecords / numChannels) + 1);
        } else {
            assertTrue(hit[i] == numRecords / numChannels);
        }
        cnt += hit[i];
    }
    assertTrue(cnt == numRecords);
}

Example 2 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testMissingKey.

@Test
public void testMissingKey() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 1 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    Record rec = new Record(0);
    rec.setField(0, new IntValue(1));
    delegate.setInstance(rec);
    try {
        oe1.selectChannels(delegate, 100);
    } catch (KeyFieldOutOfBoundsException re) {
        Assert.assertEquals(1, re.getFieldNumber());
        return;
    }
    Assert.fail("Expected a KeyFieldOutOfBoundsException.");
}

Example 3 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testForward.

@Test
public void testForward() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChannels = 100;
    int numRecords = 50000 + numChannels / 2;
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);
    numChannels = 100;
    numRecords = 10000 + numChannels / 2;
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
}

Example 4 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testPartitionHash.

@Test
public void testPartitionHash() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChans = 100;
    int numRecs = 50000;
    int[] hit = new int[numChans];
    for (int i = 0; i < numRecs; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    int cnt = 0;
    for (int aHit : hit) {
        assertTrue(aHit > 0);
        cnt += aHit;
    }
    assertTrue(cnt == numRecs);
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, stringComp);
    numChans = 100;
    numRecs = 10000;
    hit = new int[numChans];
    for (int i = 0; i < numRecs; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    cnt = 0;
    for (int aHit : hit) {
        assertTrue(aHit > 0);
        cnt += aHit;
    }
    assertTrue(cnt == numRecs);
    // test hash corner cases
    final TestIntComparator testIntComp = new TestIntComparator();
    final ChannelSelector<SerializationDelegate<Integer>> oe3 = new OutputEmitter<Integer>(ShipStrategyType.PARTITION_HASH, testIntComp);
    final SerializationDelegate<Integer> intDel = new SerializationDelegate<Integer>(new IntSerializer());
    numChans = 100;
    // MinVal hash
    intDel.setInstance(Integer.MIN_VALUE);
    int[] chans = oe3.selectChannels(intDel, numChans);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // -1 hash
    intDel.setInstance(-1);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // 0 hash
    intDel.setInstance(0);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // 1 hash
    intDel.setInstance(1);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // MaxVal hash
    intDel.setInstance(Integer.MAX_VALUE);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
}

Example 5 with OutputEmitter

use of org.apache.flink.runtime.operators.shipping.OutputEmitter in project flink by apache.

the class OutputEmitterTest method testWrongKeyClass.

@Test
public void testWrongKeyClass() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> doubleComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, doubleComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    ;
    Record rec = null;
    try {
        PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
        DataInputView in = new DataInputViewStreamWrapper(pipedInput);
        DataOutputView out = new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
        rec = new Record(1);
        rec.setField(0, new IntValue());
        rec.write(out);
        rec = new Record();
        rec.read(in);
    } catch (IOException e) {
        fail("Test erroneous");
    }
    try {
        delegate.setInstance(rec);
        oe1.selectChannels(delegate, 100);
    } catch (DeserializationException re) {
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}