Examples with IntValue org.apache.flink.types.IntValue used on opensource projects

Example 26 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class OutputEmitterTest method testForward.

@Test
public void testForward() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChannels = 100;
    int numRecords = 50000 + numChannels / 2;
    int[] hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);
    numChannels = 100;
    numRecords = 10000 + numChannels / 2;
    hit = new int[numChannels];
    for (int i = 0; i < numRecords; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    assertTrue(hit[0] == numRecords);
    for (int i = 1; i < hit.length; i++) {
        assertTrue(hit[i] == 0);
    }
}

Example 27 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class OutputEmitterTest method testPartitionHash.

@Test
public void testPartitionHash() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { IntValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    int numChans = 100;
    int numRecs = 50000;
    int[] hit = new int[numChans];
    for (int i = 0; i < numRecs; i++) {
        IntValue k = new IntValue(i);
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe1.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    int cnt = 0;
    for (int aHit : hit) {
        assertTrue(aHit > 0);
        cnt += aHit;
    }
    assertTrue(cnt == numRecs);
    // Test for StringValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { StringValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, stringComp);
    numChans = 100;
    numRecs = 10000;
    hit = new int[numChans];
    for (int i = 0; i < numRecs; i++) {
        StringValue k = new StringValue(i + "");
        Record rec = new Record(k);
        delegate.setInstance(rec);
        int[] chans = oe2.selectChannels(delegate, hit.length);
        for (int chan : chans) {
            hit[chan]++;
        }
    }
    cnt = 0;
    for (int aHit : hit) {
        assertTrue(aHit > 0);
        cnt += aHit;
    }
    assertTrue(cnt == numRecs);
    // test hash corner cases
    final TestIntComparator testIntComp = new TestIntComparator();
    final ChannelSelector<SerializationDelegate<Integer>> oe3 = new OutputEmitter<Integer>(ShipStrategyType.PARTITION_HASH, testIntComp);
    final SerializationDelegate<Integer> intDel = new SerializationDelegate<Integer>(new IntSerializer());
    numChans = 100;
    // MinVal hash
    intDel.setInstance(Integer.MIN_VALUE);
    int[] chans = oe3.selectChannels(intDel, numChans);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // -1 hash
    intDel.setInstance(-1);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // 0 hash
    intDel.setInstance(0);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // 1 hash
    intDel.setInstance(1);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
    // MaxVal hash
    intDel.setInstance(Integer.MAX_VALUE);
    chans = oe3.selectChannels(intDel, hit.length);
    assertTrue(chans.length == 1);
    assertTrue(chans[0] >= 0 && chans[0] <= numChans - 1);
}

Example 28 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class OutputEmitterTest method testWrongKeyClass.

@Test
public void testWrongKeyClass() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> doubleComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, doubleComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    ;
    Record rec = null;
    try {
        PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
        DataInputView in = new DataInputViewStreamWrapper(pipedInput);
        DataOutputView out = new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
        rec = new Record(1);
        rec.setField(0, new IntValue());
        rec.write(out);
        rec = new Record();
        rec.read(in);
    } catch (IOException e) {
        fail("Test erroneous");
    }
    try {
        delegate.setInstance(rec);
        oe1.selectChannels(delegate, 100);
    } catch (DeserializationException re) {
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}

Example 29 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class GenericCsvInputFormatTest method readWithHeaderLineAndInvalidIntermediate.

@Test
public void readWithHeaderLineAndInvalidIntermediate() {
    try {
        final String fileContent = "colname-1|colname-2|some name 3|column four|\n" + "123|abc|456|def|\n" + // repeated header in the middle
        "colname-1|colname-2|some name 3|column four|\n" + "987|xyz|654|pqr|\n";
        final FileInputSplit split = createTempFile(fileContent);
        final Configuration parameters = new Configuration();
        format.setFieldDelimiter("|");
        format.setFieldTypesGeneric(IntValue.class, StringValue.class, IntValue.class, StringValue.class);
        format.setSkipFirstLineAsHeader(true);
        format.configure(parameters);
        format.open(split);
        Value[] values = new Value[] { new IntValue(), new StringValue(), new IntValue(), new StringValue() };
        // first line is skipped as header
        //  first row (= second line)
        assertNotNull(format.nextRecord(values));
        try {
            format.nextRecord(values);
            fail("Format accepted invalid line.");
        } catch (ParseException e) {
        // as we expected
        }
    } catch (Exception ex) {
        fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
    }
}

Example 30 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class GenericCsvInputFormatTest method testReadInvalidContentsLenientWithSkipping.

@Test
public void testReadInvalidContentsLenientWithSkipping() {
    try {
        final String fileContent = // good line
        "abc|dfgsdf|777|444\n" + // wrong data type in field
        "kkz|777|foobar|hhg\n" + // too short, a skipped field never ends
        "kkz|777foobarhhg  \n" + // another good line
        "xyx|ignored|42|\n";
        final FileInputSplit split = createTempFile(fileContent);
        final Configuration parameters = new Configuration();
        format.setFieldDelimiter("|");
        format.setFieldTypesGeneric(StringValue.class, null, IntValue.class);
        format.setLenient(true);
        format.configure(parameters);
        format.open(split);
        Value[] values = new Value[] { new StringValue(), new IntValue() };
        assertNotNull(format.nextRecord(values));
        assertNull(format.nextRecord(values));
        assertNull(format.nextRecord(values));
        assertNotNull(format.nextRecord(values));
    } catch (Exception ex) {
        fail("Test failed due to a " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
    }
}