Examples with IntValue org.apache.flink.types.IntValue used on opensource projects

Example 1 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class RecordWriterTest method testClearBuffersAfterExceptionInPartitionWriter.

@Test
public void testClearBuffersAfterExceptionInPartitionWriter() throws Exception {
    NetworkBufferPool buffers = null;
    BufferPool bufferPool = null;
    try {
        buffers = new NetworkBufferPool(1, 1024, MemoryType.HEAP);
        bufferPool = spy(buffers.createBufferPool(1, Integer.MAX_VALUE));
        ResultPartitionWriter partitionWriter = mock(ResultPartitionWriter.class);
        when(partitionWriter.getBufferProvider()).thenReturn(checkNotNull(bufferPool));
        when(partitionWriter.getNumberOfOutputChannels()).thenReturn(1);
        // Recycle buffer and throw Exception
        doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                Buffer buffer = (Buffer) invocation.getArguments()[0];
                buffer.recycle();
                throw new RuntimeException("Expected test Exception");
            }
        }).when(partitionWriter).writeBuffer(any(Buffer.class), anyInt());
        RecordWriter<IntValue> recordWriter = new RecordWriter<>(partitionWriter);
        try {
            // manual flush here doesn't test this case (see next).
            for (; ; ) {
                recordWriter.emit(new IntValue(0));
            }
        } catch (Exception e) {
            // Verify that the buffer is not part of the record writer state after a failure
            // to flush it out. If the buffer is still part of the record writer state, this
            // will fail, because the buffer has already been recycled. NOTE: The mock
            // partition writer needs to recycle the buffer to correctly test this.
            recordWriter.clearBuffers();
        }
        // Verify expected methods have been called
        verify(partitionWriter, times(1)).writeBuffer(any(Buffer.class), anyInt());
        verify(bufferPool, times(1)).requestBufferBlocking();
        try {
            // Verify that manual flushing correctly clears the buffer.
            recordWriter.emit(new IntValue(0));
            recordWriter.flush();
            Assert.fail("Did not throw expected test Exception");
        } catch (Exception e) {
            recordWriter.clearBuffers();
        }
        // Verify expected methods have been called
        verify(partitionWriter, times(2)).writeBuffer(any(Buffer.class), anyInt());
        verify(bufferPool, times(2)).requestBufferBlocking();
        try {
            // Verify that broadcast emit correctly clears the buffer.
            for (; ; ) {
                recordWriter.broadcastEmit(new IntValue(0));
            }
        } catch (Exception e) {
            recordWriter.clearBuffers();
        }
        // Verify expected methods have been called
        verify(partitionWriter, times(3)).writeBuffer(any(Buffer.class), anyInt());
        verify(bufferPool, times(3)).requestBufferBlocking();
        try {
            // Verify that end of super step correctly clears the buffer.
            recordWriter.emit(new IntValue(0));
            recordWriter.broadcastEvent(EndOfSuperstepEvent.INSTANCE);
            Assert.fail("Did not throw expected test Exception");
        } catch (Exception e) {
            recordWriter.clearBuffers();
        }
        // Verify expected methods have been called
        verify(partitionWriter, times(4)).writeBuffer(any(Buffer.class), anyInt());
        verify(bufferPool, times(4)).requestBufferBlocking();
        try {
            // Verify that broadcasting and event correctly clears the buffer.
            recordWriter.emit(new IntValue(0));
            recordWriter.broadcastEvent(new TestTaskEvent());
            Assert.fail("Did not throw expected test Exception");
        } catch (Exception e) {
            recordWriter.clearBuffers();
        }
        // Verify expected methods have been called
        verify(partitionWriter, times(5)).writeBuffer(any(Buffer.class), anyInt());
        verify(bufferPool, times(5)).requestBufferBlocking();
    } finally {
        if (bufferPool != null) {
            assertEquals(1, bufferPool.getNumberOfAvailableMemorySegments());
            bufferPool.lazyDestroy();
        }
        if (buffers != null) {
            assertEquals(1, buffers.getNumberOfAvailableMemorySegments());
            buffers.destroy();
        }
    }
}

Example 2 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class RecordWriterTest method testSerializerClearedAfterClearBuffers.

@Test
public void testSerializerClearedAfterClearBuffers() throws Exception {
    final Buffer buffer = TestBufferFactory.createBuffer(16);
    ResultPartitionWriter partitionWriter = createResultPartitionWriter(createBufferProvider(buffer));
    RecordWriter<IntValue> recordWriter = new RecordWriter<IntValue>(partitionWriter);
    // Fill a buffer, but don't write it out.
    recordWriter.emit(new IntValue(0));
    verify(partitionWriter, never()).writeBuffer(any(Buffer.class), anyInt());
    // Clear all buffers.
    recordWriter.clearBuffers();
    // This should not throw an Exception iff the serializer state
    // has been cleared as expected.
    recordWriter.flush();
}

Example 3 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class StateBackendTestBase method testRequireNonNullNamespace.

/**
	 * Previously, it was possible to create partitioned state with
	 * <code>null</code> namespace. This test makes sure that this is
	 * prohibited now.
	 */
@Test
public void testRequireNonNullNamespace() throws Exception {
    AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
    ValueStateDescriptor<IntValue> kvId = new ValueStateDescriptor<>("id", IntValue.class, new IntValue(-1));
    kvId.initializeSerializerUnlessSet(new ExecutionConfig());
    try {
        backend.getPartitionedState(null, VoidNamespaceSerializer.INSTANCE, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    try {
        backend.getPartitionedState(VoidNamespace.INSTANCE, null, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    try {
        backend.getPartitionedState(null, null, kvId);
        fail("Did not throw expected NullPointerException");
    } catch (NullPointerException ignored) {
    }
    backend.dispose();
}

Example 4 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method setup.

@Before
public void setup() {
    final ArrayList<IntStringPair> source = new ArrayList<IntStringPair>();
    // add elements to the source
    source.add(new IntStringPair(new IntValue(1), new StringValue("A")));
    source.add(new IntStringPair(new IntValue(2), new StringValue("B")));
    source.add(new IntStringPair(new IntValue(3), new StringValue("C")));
    source.add(new IntStringPair(new IntValue(3), new StringValue("D")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("E")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("F")));
    source.add(new IntStringPair(new IntValue(4), new StringValue("G")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("H")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("I")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("J")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("K")));
    source.add(new IntStringPair(new IntValue(5), new StringValue("L")));
    this.sourceIter = new MutableObjectIterator<Record>() {

        final Iterator<IntStringPair> it = source.iterator();

        @Override
        public Record next(Record reuse) throws IOException {
            if (it.hasNext()) {
                IntStringPair pair = it.next();
                reuse.setField(0, pair.getInteger());
                reuse.setField(1, pair.getString());
                return reuse;
            } else {
                return null;
            }
        }

        @Override
        public Record next() throws IOException {
            if (it.hasNext()) {
                IntStringPair pair = it.next();
                Record result = new Record(2);
                result.setField(0, pair.getInteger());
                result.setField(1, pair.getString());
                return result;
            } else {
                return null;
            }
        }
    };
    @SuppressWarnings("unchecked") final RecordComparator comparator = new RecordComparator(new int[] { 0 }, new Class[] { IntValue.class });
    this.psi = new NonReusingKeyGroupedIterator<Record>(this.sourceIter, comparator);
}

Example 5 with IntValue

use of org.apache.flink.types.IntValue in project flink by apache.

the class NonReusingKeyGroupedIteratorTest method testFullIterationThroughAllValues.

@Test
public void testFullIterationThroughAllValues() throws IOException {
    try {
        // Key 1, Value A
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(1))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("A"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 1, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 2, Value B
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(2))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("B"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 2, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 3, Values C, D
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("C"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("D"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(3))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 3, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 4, Values E, F, G
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("E"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("F"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("G"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(4))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 4, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        // Key 5, Values H, I, J, K, L
        Assert.assertTrue("KeyGroupedIterator must have another key.", this.psi.nextKey());
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("H"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("I"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("J"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("K"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator must have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        Assert.assertEquals("KeyGroupedIterator returned a wrong value.", new StringValue("L"), this.psi.getValues().next().getField(1, StringValue.class));
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another value.", this.psi.getValues().hasNext());
        Assert.assertTrue("KeyGroupedIterator returned a wrong key.", this.psi.getComparatorWithCurrentReference().equalToReference(new Record(new IntValue(5))));
        Assert.assertEquals("KeyGroupedIterator returned a wrong key.", 5, this.psi.getCurrent().getField(0, IntValue.class).getValue());
        try {
            this.psi.getValues().next();
            Assert.fail("A new KeyGroupedIterator must not have any value available and hence throw an exception on next().");
        } catch (NoSuchElementException nseex) {
        }
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
        Assert.assertFalse("KeyGroupedIterator must not have another key.", this.psi.nextKey());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("The test encountered an unexpected exception.");
    }
}