Examples with RecordSerializerFactory org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory used on opensource projects

Example 6 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method testBroadcast.

@Test
public void testBroadcast() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChannels = 100;
    int numRecords = 50000;
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    for (int aHit : hit) {
        assertTrue(aHit + "", aHit == numRecords);
    }
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, stringComp);
    numChannels = 100;
    numRecords = 5000;
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    for (int aHit : hit) {
        assertTrue(aHit + "", aHit == numRecords);
    }
}

Example 7 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method testMultiKeys.

@Test
public void testMultiKeys() {
    final int numberOfChannels = 100;
    final int numRecords = 5000;
    final TypeComparator<Record> multiComp = new RecordComparatorFactory(new int[] { 0, 1, 3 }, new Class[] { IntValue.class, StringValue.class, DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(ShipStrategyType.PARTITION_HASH, multiComp, numberOfChannels);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());
    int[] hits = new int[numberOfChannels];
    for (int i = 0; i < numRecords; i++) {
        Record record = new Record(4);
        record.setField(0, new IntValue(i));
        record.setField(1, new StringValue("AB" + i + "CD" + i));
        record.setField(3, new DoubleValue(i * 3.141d));
        delegate.setInstance(record);
        int channel = selector.selectChannel(delegate);
        hits[channel]++;
    }
    int totalHitCount = 0;
    for (int hit : hits) {
        assertTrue(hit > 0);
        totalHitCount += hit;
    }
    assertTrue(totalHitCount == numRecords);
}

Example 8 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method verifyWrongPartitionHashKey.

private boolean verifyWrongPartitionHashKey(int position, int fieldNum) {
    final TypeComparator<Record> comparator = new RecordComparatorFactory(new int[] { position }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(ShipStrategyType.PARTITION_HASH, comparator, 100);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());
    Record record = new Record(2);
    record.setField(fieldNum, new IntValue(1));
    delegate.setInstance(record);
    try {
        selector.selectChannel(delegate);
    } catch (NullKeyFieldException re) {
        Assert.assertEquals(position, re.getFieldNumber());
        return true;
    }
    return false;
}

Example 9 with RecordSerializerFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordSerializerFactory in project flink by apache.

the class OutputEmitterTest method getSelectedChannelsHitCount.

private int[] getSelectedChannelsHitCount(ShipStrategyType shipStrategyType, int numRecords, int numberOfChannels, Enum recordType) {
    final TypeComparator<Record> comparator = new RecordComparatorFactory(new int[] { 0 }, new Class[] { recordType == RecordType.INTEGER ? IntValue.class : StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(shipStrategyType, comparator, numberOfChannels);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());
    return getSelectedChannelsHitCount(selector, delegate, recordType, numRecords, numberOfChannels);
}