Examples with TypeComparator org.apache.flink.api.common.typeutils.TypeComparator used on opensource projects

Example 1 with TypeComparator

use of org.apache.flink.api.common.typeutils.TypeComparator in project flink by apache.

the class BatchTask method initInputsSerializersAndComparators.

/**
 * Creates all the serializers and comparators.
 */
protected void initInputsSerializersAndComparators(int numInputs, int numComparators) {
    this.inputSerializers = new TypeSerializerFactory<?>[numInputs];
    this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null;
    this.inputIterators = new MutableObjectIterator<?>[numInputs];
    ClassLoader userCodeClassLoader = getUserCodeClassLoader();
    for (int i = 0; i < numInputs; i++) {
        final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader);
        this.inputSerializers[i] = serializerFactory;
        this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]);
    }
    // ---------------- create the driver's comparators ---------------------
    for (int i = 0; i < numComparators; i++) {
        if (this.inputComparators != null) {
            final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader);
            this.inputComparators[i] = comparatorFactory.createComparator();
        }
    }
}

Example 2 with TypeComparator

use of org.apache.flink.api.common.typeutils.TypeComparator in project flink by apache.

the class NonReusingSortMergeInnerJoinIteratorITCase method beforeTest.

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
    serializer1 = new TupleSerializer<Tuple2<Integer, String>>((Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class, new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
    serializer2 = new TupleSerializer<Tuple2<Integer, String>>((Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class, new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
    comparator1 = new TupleComparator<Tuple2<Integer, String>>(new int[] { 0 }, new TypeComparator<?>[] { new IntComparator(true) }, new TypeSerializer<?>[] { IntSerializer.INSTANCE });
    comparator2 = new TupleComparator<Tuple2<Integer, String>>(new int[] { 0 }, new TypeComparator<?>[] { new IntComparator(true) }, new TypeSerializer<?>[] { IntSerializer.INSTANCE });
    pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
    this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
    this.ioManager = new IOManagerAsync();
}

Example 3 with TypeComparator

use of org.apache.flink.api.common.typeutils.TypeComparator in project flink by apache.

the class ReusingSortMergeInnerJoinIteratorITCase method beforeTest.

@SuppressWarnings("unchecked")
@Before
public void beforeTest() {
    serializer1 = new TupleSerializer<Tuple2<Integer, String>>((Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class, new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
    serializer2 = new TupleSerializer<Tuple2<Integer, String>>((Class<Tuple2<Integer, String>>) (Class<?>) Tuple2.class, new TypeSerializer<?>[] { IntSerializer.INSTANCE, StringSerializer.INSTANCE });
    comparator1 = new TupleComparator<Tuple2<Integer, String>>(new int[] { 0 }, new TypeComparator<?>[] { new IntComparator(true) }, new TypeSerializer<?>[] { IntSerializer.INSTANCE });
    comparator2 = new TupleComparator<Tuple2<Integer, String>>(new int[] { 0 }, new TypeComparator<?>[] { new IntComparator(true) }, new TypeSerializer<?>[] { IntSerializer.INSTANCE });
    pairComparator = new GenericPairComparator<Tuple2<Integer, String>, Tuple2<Integer, String>>(comparator1, comparator2);
    this.memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(MEMORY_SIZE).build();
    this.ioManager = new IOManagerAsync();
}

Example 4 with TypeComparator

use of org.apache.flink.api.common.typeutils.TypeComparator in project flink by apache.

the class MultiInputSortingDataInput method wrapInputs.

public static <K> SelectableSortingInputs wrapInputs(TaskInvokable containingTask, StreamTaskInput<Object>[] sortingInputs, KeySelector<Object, K>[] keySelectors, TypeSerializer<Object>[] inputSerializers, TypeSerializer<K> keySerializer, StreamTaskInput<Object>[] passThroughInputs, MemoryManager memoryManager, IOManager ioManager, boolean objectReuse, double managedMemoryFraction, Configuration jobConfiguration, ExecutionConfig executionConfig) {
    int keyLength = keySerializer.getLength();
    final TypeComparator<Tuple2<byte[], StreamRecord<Object>>> comparator;
    DataOutputSerializer dataOutputSerializer;
    if (keyLength > 0) {
        dataOutputSerializer = new DataOutputSerializer(keyLength);
        comparator = new FixedLengthByteKeyComparator<>(keyLength);
    } else {
        dataOutputSerializer = new DataOutputSerializer(64);
        comparator = new VariableLengthByteKeyComparator<>();
    }
    List<Integer> passThroughInputIndices = Arrays.stream(passThroughInputs).map(StreamTaskInput::getInputIndex).collect(Collectors.toList());
    int numberOfInputs = sortingInputs.length + passThroughInputs.length;
    CommonContext commonContext = new CommonContext(sortingInputs);
    InputSelector inputSelector = new InputSelector(commonContext, numberOfInputs, passThroughInputIndices);
    StreamTaskInput<?>[] wrappedSortingInputs = IntStream.range(0, sortingInputs.length).mapToObj(idx -> {
        try {
            KeyAndValueSerializer<Object> keyAndValueSerializer = new KeyAndValueSerializer<>(inputSerializers[idx], keyLength);
            return new MultiInputSortingDataInput<>(commonContext, sortingInputs[idx], sortingInputs[idx].getInputIndex(), ExternalSorter.newBuilder(memoryManager, containingTask, keyAndValueSerializer, comparator, executionConfig).memoryFraction(managedMemoryFraction / numberOfInputs).enableSpilling(ioManager, jobConfiguration.get(AlgorithmOptions.SORT_SPILLING_THRESHOLD)).maxNumFileHandles(jobConfiguration.get(AlgorithmOptions.SPILLING_MAX_FAN) / numberOfInputs).objectReuse(objectReuse).largeRecords(true).build(), keySelectors[idx], keySerializer, dataOutputSerializer);
        } catch (MemoryAllocationException e) {
            throw new RuntimeException();
        }
    }).toArray(StreamTaskInput[]::new);
    StreamTaskInput<?>[] wrappedPassThroughInputs = Arrays.stream(passThroughInputs).map(input -> new ObservableStreamTaskInput<>(input, inputSelector)).toArray(StreamTaskInput[]::new);
    return new SelectableSortingInputs(wrappedSortingInputs, wrappedPassThroughInputs, inputSelector);
}