Examples with IntSerializer org.apache.flink.api.common.typeutils.base.IntSerializer used on opensource projects

Example 1 with IntSerializer

use of org.apache.flink.api.common.typeutils.base.IntSerializer in project flink by apache.

the class LockableTypeSerializerTest method testDuplicate.

/**
 * This tests that {@link Lockable.LockableTypeSerializer#duplicate()} works as expected.
 */
@Test
public void testDuplicate() {
    IntSerializer nonDuplicatingInnerSerializer = IntSerializer.INSTANCE;
    Assert.assertSame(nonDuplicatingInnerSerializer, nonDuplicatingInnerSerializer.duplicate());
    Lockable.LockableTypeSerializer<Integer> candidateTestShallowDuplicate = new Lockable.LockableTypeSerializer<>(nonDuplicatingInnerSerializer);
    Assert.assertSame(candidateTestShallowDuplicate, candidateTestShallowDuplicate.duplicate());
    TestDuplicateSerializer duplicatingInnerSerializer = new TestDuplicateSerializer();
    Assert.assertNotSame(duplicatingInnerSerializer, duplicatingInnerSerializer.duplicate());
    Lockable.LockableTypeSerializer<Integer> candidateTestDeepDuplicate = new Lockable.LockableTypeSerializer<>(duplicatingInnerSerializer);
    Lockable.LockableTypeSerializer<Integer> deepDuplicate = candidateTestDeepDuplicate.duplicate();
    Assert.assertNotSame(candidateTestDeepDuplicate, deepDuplicate);
    Assert.assertNotSame(candidateTestDeepDuplicate.getElementSerializer(), deepDuplicate.getElementSerializer());
}

Example 2 with IntSerializer

use of org.apache.flink.api.common.typeutils.base.IntSerializer in project flink by apache.

the class OutputEmitterTest method testPartitionHash.

@Test
public void testPartitionHash() {
    // Test for IntValue
    verifyPartitionHashSelectedChannels(50000, 100, RecordType.INTEGER);
    // Test for StringValue
    verifyPartitionHashSelectedChannels(10000, 100, RecordType.STRING);
    // Test hash corner cases
    final TestIntComparator testIntComp = new TestIntComparator();
    final ChannelSelector<SerializationDelegate<Integer>> selector = createChannelSelector(ShipStrategyType.PARTITION_HASH, testIntComp, 100);
    final SerializationDelegate<Integer> serializationDelegate = new SerializationDelegate<>(new IntSerializer());
    assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MIN_VALUE, 100);
    assertPartitionHashSelectedChannels(selector, serializationDelegate, -1, 100);
    assertPartitionHashSelectedChannels(selector, serializationDelegate, 0, 100);
    assertPartitionHashSelectedChannels(selector, serializationDelegate, 1, 100);
    assertPartitionHashSelectedChannels(selector, serializationDelegate, Integer.MAX_VALUE, 100);
}

Example 3 with IntSerializer

use of org.apache.flink.api.common.typeutils.base.IntSerializer in project flink by apache.

the class MultiInputSortingDataInputsTest method twoInputOrderTest.

@SuppressWarnings("unchecked")
public void twoInputOrderTest(int preferredIndex, int sortedIndex) throws Exception {
    CollectingDataOutput<Object> collectingDataOutput = new CollectingDataOutput<>();
    List<StreamElement> sortedInputElements = Arrays.asList(new StreamRecord<>(1, 3), new StreamRecord<>(1, 1), new StreamRecord<>(2, 1), new StreamRecord<>(2, 3), new StreamRecord<>(1, 2), new StreamRecord<>(2, 2), Watermark.MAX_WATERMARK);
    CollectionDataInput<Integer> sortedInput = new CollectionDataInput<>(sortedInputElements, sortedIndex);
    List<StreamElement> preferredInputElements = Arrays.asList(new StreamRecord<>(99, 3), new StreamRecord<>(99, 1), new Watermark(99L));
    CollectionDataInput<Integer> preferredInput = new CollectionDataInput<>(preferredInputElements, preferredIndex);
    KeySelector<Integer, Integer> keySelector = value -> value;
    try (MockEnvironment environment = MockEnvironment.builder().build()) {
        SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(new DummyInvokable(), new StreamTaskInput[] { sortedInput }, new KeySelector[] { keySelector }, new TypeSerializer[] { new IntSerializer() }, new IntSerializer(), new StreamTaskInput[] { preferredInput }, environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new ExecutionConfig());
        StreamTaskInput<?>[] sortingDataInputs = selectableSortingInputs.getSortedInputs();
        StreamTaskInput<?>[] preferredDataInputs = selectableSortingInputs.getPassThroughInputs();
        try (StreamTaskInput<Object> preferredTaskInput = (StreamTaskInput<Object>) preferredDataInputs[0];
            StreamTaskInput<Object> sortedTaskInput = (StreamTaskInput<Object>) sortingDataInputs[0]) {
            MultipleInputSelectionHandler selectionHandler = new MultipleInputSelectionHandler(selectableSortingInputs.getInputSelectable(), 2);
            @SuppressWarnings("rawtypes") StreamOneInputProcessor[] inputProcessors = new StreamOneInputProcessor[2];
            inputProcessors[preferredIndex] = new StreamOneInputProcessor<>(preferredTaskInput, collectingDataOutput, new DummyOperatorChain());
            inputProcessors[sortedIndex] = new StreamOneInputProcessor<>(sortedTaskInput, collectingDataOutput, new DummyOperatorChain());
            StreamMultipleInputProcessor processor = new StreamMultipleInputProcessor(selectionHandler, inputProcessors);
            DataInputStatus inputStatus;
            do {
                inputStatus = processor.processInput();
            } while (inputStatus != DataInputStatus.END_OF_INPUT);
        }
    }
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(99, 3), new StreamRecord<>(99, 1), // max watermark from the preferred input
    new Watermark(99L), new StreamRecord<>(1, 1), new StreamRecord<>(1, 2), new StreamRecord<>(1, 3), new StreamRecord<>(2, 1), new StreamRecord<>(2, 2), new StreamRecord<>(2, 3), // max watermark from the sorted input
    Watermark.MAX_WATERMARK)));
}

Example 4 with IntSerializer

use of org.apache.flink.api.common.typeutils.base.IntSerializer in project flink by apache.

the class SortingDataInputTest method simpleVariableLengthKeySorting.

@Test
public void simpleVariableLengthKeySorting() throws Exception {
    CollectingDataOutput<Integer> collectingDataOutput = new CollectingDataOutput<>();
    CollectionDataInput<Integer> input = new CollectionDataInput<>(Arrays.asList(new StreamRecord<>(1, 3), new StreamRecord<>(1, 1), new StreamRecord<>(2, 1), new StreamRecord<>(2, 3), new StreamRecord<>(1, 2), new StreamRecord<>(2, 2)));
    MockEnvironment environment = MockEnvironment.builder().build();
    SortingDataInput<Integer, String> sortingDataInput = new SortingDataInput<>(input, new IntSerializer(), new StringSerializer(), (KeySelector<Integer, String>) value -> "" + value, environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new DummyInvokable(), new ExecutionConfig());
    DataInputStatus inputStatus;
    do {
        inputStatus = sortingDataInput.emitNext(collectingDataOutput);
    } while (inputStatus != DataInputStatus.END_OF_INPUT);
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(1, 1), new StreamRecord<>(1, 2), new StreamRecord<>(1, 3), new StreamRecord<>(2, 1), new StreamRecord<>(2, 2), new StreamRecord<>(2, 3))));
}

Example 5 with IntSerializer

use of org.apache.flink.api.common.typeutils.base.IntSerializer in project flink by apache.

the class SerializerComparatorTestData method getOrderedIntTestData.

@SuppressWarnings("unchecked")
static Tuple2<byte[], StreamRecord<Integer>>[] getOrderedIntTestData() {
    IntSerializer intSerializer = new IntSerializer();
    DataOutputSerializer outputSerializer = new DataOutputSerializer(intSerializer.getLength());
    return IntStream.range(-10, 10).mapToObj(idx -> {
        try {
            intSerializer.serialize(idx, outputSerializer);
            byte[] copyOfBuffer = outputSerializer.getCopyOfBuffer();
            outputSerializer.clear();
            return Tuple2.of(copyOfBuffer, new StreamRecord<>(idx, idx));
        } catch (IOException e) {
            throw new AssertionError(e);
        }
    }).toArray(Tuple2[]::new);
}