Examples with RecordComparatorFactory org.apache.flink.runtime.testutils.recordutils.RecordComparatorFactory used on opensource projects

Example 1 with RecordComparatorFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordComparatorFactory in project flink by apache.

the class OutputEmitterTest method testMissingKey.

@Test
public void testMissingKey() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 1 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    Record rec = new Record(0);
    rec.setField(0, new IntValue(1));
    delegate.setInstance(rec);
    try {
        oe1.selectChannels(delegate, 100);
    } catch (KeyFieldOutOfBoundsException re) {
        Assert.assertEquals(1, re.getFieldNumber());
        return;
    }
    Assert.fail("Expected a KeyFieldOutOfBoundsException.");
}

Example 2 with RecordComparatorFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordComparatorFactory in project flink by apache.

the class OutputEmitterTest method testForward.

@Test
public void testForward() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChannels = 100;
    int numRecords = 50000 + numChannels / 2;
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);
    numChannels = 100;
    numRecords = 10000 + numChannels / 2;
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
}

Example 3 with RecordComparatorFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordComparatorFactory in project flink by apache.

the class OutputEmitterTest method testWrongKeyClass.

@Test
public void testWrongKeyClass() throws Exception {
    // Test for IntValue
    final TypeComparator<Record> doubleComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(ShipStrategyType.PARTITION_HASH, doubleComp, 100);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());
    PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
    DataInputView in = new DataInputViewStreamWrapper(pipedInput);
    DataOutputView out = new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
    Record record = new Record(1);
    record.setField(0, new IntValue());
    record.write(out);
    record = new Record();
    record.read(in);
    try {
        delegate.setInstance(record);
        selector.selectChannel(delegate);
    } catch (DeserializationException re) {
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}

Example 4 with RecordComparatorFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordComparatorFactory in project flink by apache.

the class DataSinkTaskTest method testCancelSortingDataSinkTask.

@Test
@SuppressWarnings("unchecked")
public void testCancelSortingDataSinkTask() {
    double memoryFraction = 1.0;
    super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    super.addInput(new InfiniteInputIterator(), 0);
    final DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
    Configuration stubParams = new Configuration();
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(new RecordComparatorFactory(new int[] { 1 }, (new Class[] { IntValue.class })), 0);
    super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
    File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
    super.registerFileOutputTask(MockOutputFormat.class, tempTestFile.toURI().toString(), stubParams);
    Thread taskRunner = new Thread() {

        @Override
        public void run() {
            try {
                testTask.invoke();
            } catch (Exception ie) {
                ie.printStackTrace();
                Assert.fail("Task threw exception although it was properly canceled");
            }
        }
    };
    taskRunner.start();
    TaskCancelThread tct = new TaskCancelThread(2, taskRunner, testTask);
    tct.start();
    try {
        tct.join();
        taskRunner.join();
    } catch (InterruptedException ie) {
        Assert.fail("Joining threads failed");
    }
}

Example 5 with RecordComparatorFactory

use of org.apache.flink.runtime.testutils.recordutils.RecordComparatorFactory in project flink by apache.

the class DataSinkTaskTest method testSortingDataSinkTask.

@Test
@SuppressWarnings("unchecked")
public void testSortingDataSinkTask() {
    int keyCnt = 100;
    int valCnt = 20;
    double memoryFraction = 1.0;
    super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    super.addInput(new UniformRecordGenerator(keyCnt, valCnt, true), 0);
    DataSinkTask<Record> testTask = new DataSinkTask<>(this.mockEnv);
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(new RecordComparatorFactory(new int[] { 1 }, (new Class[] { IntValue.class })), 0);
    super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
    File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
    super.registerFileOutputTask(MockOutputFormat.class, tempTestFile.toURI().toString(), new Configuration());
    try {
        testTask.invoke();
    } catch (Exception e) {
        LOG.debug("Exception while invoking the test task.", e);
        Assert.fail("Invoke method caused exception.");
    }
    Assert.assertTrue("Temp output file does not exist", tempTestFile.exists());
    FileReader fr = null;
    BufferedReader br = null;
    try {
        fr = new FileReader(tempTestFile);
        br = new BufferedReader(fr);
        Set<Integer> keys = new HashSet<>();
        int curVal = -1;
        while (br.ready()) {
            String line = br.readLine();
            Integer key = Integer.parseInt(line.substring(0, line.indexOf("_")));
            Integer val = Integer.parseInt(line.substring(line.indexOf("_") + 1, line.length()));
            // check that values are in correct order
            Assert.assertTrue("Values not in ascending order", val >= curVal);
            // next value hit
            if (val > curVal) {
                if (curVal != -1) {
                    // check that we saw 100 distinct keys for this values
                    Assert.assertTrue("Keys missing for value", keys.size() == 100);
                }
                // empty keys set
                keys.clear();
                // update current value
                curVal = val;
            }
            Assert.assertTrue("Duplicate key for value", keys.add(key));
        }
    } catch (FileNotFoundException e) {
        Assert.fail("Out file got lost...");
    } catch (IOException ioe) {
        Assert.fail("Caught IOE while reading out file");
    } finally {
        if (br != null) {
            try {
                br.close();
            } catch (Throwable t) {
            }
        }
        if (fr != null) {
            try {
                fr.close();
            } catch (Throwable t) {
            }
        }
    }
}