Examples with RecordSerializerFactory org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory used on opensource projects

Example 1 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method testForcedRebalance.

@Test
public void testForcedRebalance() {
    // Test for IntValue
    int numChannels = 100;
    int toTaskIndex = numChannels * 6 / 7;
    int fromTaskIndex = toTaskIndex + numChannels;
    int extraRecords = numChannels / 3;
    int numRecords = 50000 + extraRecords;
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
        if (toTaskIndex <= i || i < toTaskIndex + extraRecords - numChannels) {
            assertTrue(hit[i] == (numRecords / numChannels) + 1);
        } else {
            assertTrue(hit[i] == numRecords / numChannels);
        }
        cnt += hit[i];
    }
    assertTrue(cnt == numRecords);
    // Test for StringValue
    numChannels = 100;
    toTaskIndex = numChannels / 5;
    fromTaskIndex = toTaskIndex + 2 * numChannels;
    extraRecords = numChannels * 2 / 9;
    numRecords = 10000 + extraRecords;
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_FORCED_REBALANCE, fromTaskIndex);
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    cnt = 0;
    for (int i = 0; i < hit.length; i++) {
        if (toTaskIndex <= i && i < toTaskIndex + extraRecords) {
            assertTrue(hit[i] == (numRecords / numChannels) + 1);
        } else {
            assertTrue(hit[i] == numRecords / numChannels);
        }
        cnt += hit[i];
    }
    assertTrue(cnt == numRecords);
}

Example 2 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method testMissingKey.

@Test
public void testMissingKey() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 1 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    Record rec = new Record(0);
    rec.setField(0, new IntValue(1));
    delegate.setInstance(rec);
    try {
        oe1.selectChannels(delegate, 100);
    } catch (KeyFieldOutOfBoundsException re) {
        Assert.assertEquals(1, re.getFieldNumber());
        return;
    }
    Assert.fail("Expected a KeyFieldOutOfBoundsException.");
}

Example 3 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method testForward.

@Test
public void testForward() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChannels = 100;
    int numRecords = 50000 + numChannels / 2;
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);
    numChannels = 100;
    numRecords = 10000 + numChannels / 2;
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
}

Example 4 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method testWrongKeyClass.

@Test
public void testWrongKeyClass() throws Exception {
    // Test for IntValue
    final TypeComparator<Record> doubleComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(ShipStrategyType.PARTITION_HASH, doubleComp, 100);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());
    PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
    DataInputView in = new DataInputViewStreamWrapper(pipedInput);
    DataOutputView out = new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
    Record record = new Record(1);
    record.setField(0, new IntValue());
    record.write(out);
    record = new Record();
    record.read(in);
    try {
        delegate.setInstance(record);
        selector.selectChannel(delegate);
    } catch (DeserializationException re) {
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}

Example 5 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method testNullKey.

@Test
public void testNullKey() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    Record rec = new Record(2);
    rec.setField(1, new IntValue(1));
    delegate.setInstance(rec);
    try {
        oe1.selectChannels(delegate, 100);
    } catch (NullKeyFieldException re) {
        Assert.assertEquals(0, re.getFieldNumber());
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}