Search in sources :

Example 11 with Partition

use of com.datatorrent.api.Partitioner.Partition in project apex-core by apache.

the class PhysicalPlan method initPartitioning.

private void initPartitioning(PMapping m, int partitionCnt) {
    Operator operator = m.logicalOperator.getOperator();
    Collection<Partition<Operator>> partitions;
    @SuppressWarnings("unchecked") Partitioner<Operator> partitioner = m.logicalOperator.getAttributes().contains(OperatorContext.PARTITIONER) ? (Partitioner<Operator>) m.logicalOperator.getValue(OperatorContext.PARTITIONER) : operator instanceof Partitioner ? (Partitioner<Operator>) operator : null;
    Collection<Partition<Operator>> collection = new ArrayList<>(1);
    DefaultPartition<Operator> firstPartition = new DefaultPartition<>(operator);
    collection.add(firstPartition);
    if (partitioner != null) {
        partitions = partitioner.definePartitions(collection, new PartitioningContextImpl(m, partitionCnt));
        if (partitions == null || partitions.isEmpty()) {
            throw new IllegalStateException("Partitioner returns null or empty.");
        }
    } else {
        // Non zero in the case of parallel partitioning.
        for (int partitionCounter = 0; partitionCounter < partitionCnt - 1; partitionCounter++) {
            collection.add(firstPartition);
        }
        partitions = collection;
    }
    Collection<StatsListener> statsListeners = m.logicalOperator.getValue(OperatorContext.STATS_LISTENERS);
    if (statsListeners != null && !statsListeners.isEmpty()) {
        if (m.statsHandlers == null) {
            m.statsHandlers = new ArrayList<>(statsListeners.size());
        }
        for (StatsListener sl : statsListeners) {
            m.statsHandlers.add(getStatsListenerAdapter(sl));
        }
    }
    if (m.logicalOperator.getOperator() instanceof StatsListener) {
        if (m.statsHandlers == null) {
            m.statsHandlers = new ArrayList<>(1);
        }
        m.statsHandlers.add(new StatsListenerProxy(m.logicalOperator));
    }
    // create operator instance per partition
    Map<Integer, Partition<Operator>> operatorIdToPartition = Maps.newHashMapWithExpectedSize(partitions.size());
    for (Partition<Operator> partition : partitions) {
        PTOperator p = addPTOperator(m, partition, null);
        operatorIdToPartition.put(p.getId(), partition);
    }
    if (partitioner != null) {
        partitioner.partitioned(operatorIdToPartition);
    }
}
Also used : Operator(com.datatorrent.api.Operator) Partition(com.datatorrent.api.Partitioner.Partition) DefaultPartition(com.datatorrent.api.DefaultPartition) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) StatsListener(com.datatorrent.api.StatsListener) Checkpoint(com.datatorrent.stram.api.Checkpoint) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DefaultPartition(com.datatorrent.api.DefaultPartition) Partitioner(com.datatorrent.api.Partitioner)

Aggregations

Partition (com.datatorrent.api.Partitioner.Partition)11 DefaultPartition (com.datatorrent.api.DefaultPartition)9 Test (org.junit.Test)8 PartitioningContextImpl (org.apache.apex.malhar.lib.partitioner.StatelessPartitionerTest.PartitioningContextImpl)7 File (java.io.File)6 LineByLineFileInputOperator (org.apache.apex.malhar.lib.fs.LineByLineFileInputOperator)6 Path (org.apache.hadoop.fs.Path)6 StatsListener (com.datatorrent.api.StatsListener)4 ArrayList (java.util.ArrayList)4 CollectorTestSink (org.apache.apex.malhar.lib.testbench.CollectorTestSink)4 Operator (com.datatorrent.api.Operator)3 Checkpoint (com.datatorrent.stram.api.Checkpoint)3 Kryo (com.esotericsoftware.kryo.Kryo)3 Partitioner (com.datatorrent.api.Partitioner)2 PartitionKeys (com.datatorrent.api.Partitioner.PartitionKeys)2 HashMap (java.util.HashMap)2 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)2 Configuration (org.apache.hadoop.conf.Configuration)2 CouchbaseClient (com.couchbase.client.CouchbaseClient)1 CouchbaseConnectionFactoryBuilder (com.couchbase.client.CouchbaseConnectionFactoryBuilder)1