Examples with CompactingHashTable org.apache.flink.runtime.operators.hash.CompactingHashTable used on opensource projects

Example 1 with CompactingHashTable

use of org.apache.flink.runtime.operators.hash.CompactingHashTable in project flink by apache.

the class CoGroupWithSolutionSetSecondDriver method initialize.

// --------------------------------------------------------------------------------------------
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
    final TypeComparator<IT2> solutionSetComparator;
    // grab a handle to the hash table from the iteration broker
    if (taskContext instanceof AbstractIterativeTask) {
        AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
        String identifier = iterativeTaskContext.brokerKey();
        Object table = SolutionSetBroker.instance().get(identifier);
        if (table instanceof CompactingHashTable) {
            this.hashTable = (CompactingHashTable<IT2>) table;
            solutionSetSerializer = this.hashTable.getBuildSideSerializer();
            solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
        } else if (table instanceof JoinHashMap) {
            this.objectMap = (JoinHashMap<IT2>) table;
            solutionSetSerializer = this.objectMap.getBuildSerializer();
            solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
        } else {
            throw new RuntimeException("Unrecognized solution set index: " + table);
        }
    } else {
        throw new Exception("The task context of this driver is no iterative task context.");
    }
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
    TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
    this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (objectReuseEnabled) {
        solutionSideRecord = solutionSetSerializer.createInstance();
    }
    ;
    TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
    pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}

Example 2 with CompactingHashTable

use of org.apache.flink.runtime.operators.hash.CompactingHashTable in project flink by apache.

the class JoinWithSolutionSetSecondDriver method initialize.

// --------------------------------------------------------------------------------------------
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
    final TypeSerializer<IT2> solutionSetSerializer;
    final TypeComparator<IT2> solutionSetComparator;
    // grab a handle to the hash table from the iteration broker
    if (taskContext instanceof AbstractIterativeTask) {
        AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
        String identifier = iterativeTaskContext.brokerKey();
        Object table = SolutionSetBroker.instance().get(identifier);
        if (table instanceof CompactingHashTable) {
            this.hashTable = (CompactingHashTable<IT2>) table;
            solutionSetSerializer = this.hashTable.getBuildSideSerializer();
            solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
        } else if (table instanceof JoinHashMap) {
            this.objectMap = (JoinHashMap<IT2>) table;
            solutionSetSerializer = this.objectMap.getBuildSerializer();
            solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
        } else {
            throw new RuntimeException("Unrecognized solution set index: " + table);
        }
    } else {
        throw new Exception("The task context of this driver is no iterative task context.");
    }
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
    TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
    TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (objectReuseEnabled) {
        solutionSideRecord = solutionSetSerializer.createInstance();
        probeSideRecord = probeSideSerializer.createInstance();
    }
    TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
    pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}

Example 3 with CompactingHashTable

use of org.apache.flink.runtime.operators.hash.CompactingHashTable in project flink by apache.

the class CoGroupWithSolutionSetFirstDriver method initialize.

// --------------------------------------------------------------------------------------------
@Override
@SuppressWarnings("unchecked")
public void initialize() {
    final TypeComparator<IT1> solutionSetComparator;
    // grab a handle to the hash table from the iteration broker
    if (taskContext instanceof AbstractIterativeTask) {
        AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
        String identifier = iterativeTaskContext.brokerKey();
        Object table = SolutionSetBroker.instance().get(identifier);
        if (table instanceof CompactingHashTable) {
            this.hashTable = (CompactingHashTable<IT1>) table;
            solutionSetSerializer = this.hashTable.getBuildSideSerializer();
            solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
        } else if (table instanceof JoinHashMap) {
            this.objectMap = (JoinHashMap<IT1>) table;
            solutionSetSerializer = this.objectMap.getBuildSerializer();
            solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
        } else {
            throw new RuntimeException("Unrecognized solution set index: " + table);
        }
    } else {
        throw new RuntimeException("The task context of this driver is no iterative task context.");
    }
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
    TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
    this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (objectReuseEnabled) {
        solutionSideRecord = solutionSetSerializer.createInstance();
    }
    TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
    pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}

Example 4 with CompactingHashTable

use of org.apache.flink.runtime.operators.hash.CompactingHashTable in project flink by apache.

the class JoinWithSolutionSetFirstDriver method initialize.

// --------------------------------------------------------------------------------------------
@Override
@SuppressWarnings("unchecked")
public void initialize() {
    final TypeSerializer<IT1> solutionSetSerializer;
    final TypeComparator<IT1> solutionSetComparator;
    // grab a handle to the hash table from the iteration broker
    if (taskContext instanceof AbstractIterativeTask) {
        AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
        String identifier = iterativeTaskContext.brokerKey();
        Object table = SolutionSetBroker.instance().get(identifier);
        if (table instanceof CompactingHashTable) {
            this.hashTable = (CompactingHashTable<IT1>) table;
            solutionSetSerializer = this.hashTable.getBuildSideSerializer();
            solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
        } else if (table instanceof JoinHashMap) {
            this.objectMap = (JoinHashMap<IT1>) table;
            solutionSetSerializer = this.objectMap.getBuildSerializer();
            solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
        } else {
            throw new RuntimeException("Unrecognized solution set index: " + table);
        }
    } else {
        throw new RuntimeException("The task context of this driver is no iterative task context.");
    }
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
    TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
    TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (objectReuseEnabled) {
        solutionSideRecord = solutionSetSerializer.createInstance();
        probeSideRecord = probeSideSerializer.createInstance();
    }
    TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
    pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}