Examples with PartitionKeys com.datatorrent.api.Partitioner.PartitionKeys used on opensource projects

Example 6 with PartitionKeys

use of com.datatorrent.api.Partitioner.PartitionKeys in project apex-core by apache.

the class PhysicalPlan method updatePersistOperatorWithSinkPartitions.

private void updatePersistOperatorWithSinkPartitions(InputPortMeta persistInputPort, OperatorMeta persistOperatorMeta, StreamCodecWrapperForPersistance<?> persistCodec, InputPortMeta sinkPortMeta) {
    Collection<PTOperator> ptOperators = getOperators(sinkPortMeta.getOperatorMeta());
    Collection<PartitionKeys> partitionKeysList = new ArrayList<>();
    for (PTOperator p : ptOperators) {
        PartitionKeys keys = p.partitionKeys.get(sinkPortMeta);
        partitionKeysList.add(keys);
    }
    persistCodec.inputPortToPartitionMap.put(sinkPortMeta, partitionKeysList);
}

Example 7 with PartitionKeys

use of com.datatorrent.api.Partitioner.PartitionKeys in project apex-core by apache.

the class DefaultPartition method assignPartitionKeys.

/**
   * Assign partitions keys for the given list of partitions and port of the logical operator.
   * <p>
   * The incoming stream will be partitioned by n keys, with n the nearest power of 2 greater or equal to the
   * number of partition instances provided. If the number of instances does not align with a power of 2, some of the
   * partitions will be assigned 2 keys. This logic is used for default partitioning and can be used to implement
   * {@link Partitioner}.
   *
   * @param <T>        Type of the partitionable object
   * @param partitions
   * @param inputPort
   */
public static <T> void assignPartitionKeys(Collection<Partition<T>> partitions, InputPort<?> inputPort) {
    if (partitions.isEmpty()) {
        throw new IllegalArgumentException("partitions collection cannot be empty");
    }
    int partitionBits = (Integer.numberOfLeadingZeros(0) - Integer.numberOfLeadingZeros(partitions.size() - 1));
    int partitionMask = 0;
    if (partitionBits > 0) {
        partitionMask = -1 >>> (Integer.numberOfLeadingZeros(-1)) - partitionBits;
    }
    Iterator<Partition<T>> iterator = partitions.iterator();
    for (int i = 0; i <= partitionMask; i++) {
        Partition<?> p;
        if (iterator.hasNext()) {
            p = iterator.next();
        } else {
            iterator = partitions.iterator();
            p = iterator.next();
        }
        PartitionKeys pks = p.getPartitionKeys().get(inputPort);
        if (pks == null) {
            p.getPartitionKeys().put(inputPort, new PartitionKeys(partitionMask, Sets.newHashSet(i)));
        } else {
            pks.partitions.add(i);
        }
    }
}

Example 8 with PartitionKeys

use of com.datatorrent.api.Partitioner.PartitionKeys in project apex-core by apache.

the class StreamCodecWrapperForPersistance method shouldCaptureEvent.

public boolean shouldCaptureEvent(T o) {
    for (Entry<InputPortMeta, Collection<PartitionKeys>> entry : inputPortToPartitionMap.entrySet()) {
        StreamCodec<Object> codec = codecsToMerge.get(entry.getKey());
        Collection<PartitionKeys> partitionKeysList = entry.getValue();
        for (PartitionKeys keys : partitionKeysList) {
            if (keys.partitions != null && keys.partitions.contains(keys.mask & codec.getPartition(o))) {
                // So send the event to persist operator
                return true;
            }
        }
    }
    return false;
}

Example 9 with PartitionKeys

use of com.datatorrent.api.Partitioner.PartitionKeys in project apex-core by apache.

the class PhysicalPlanTest method testDefaultPartitioning.

@Test
public void testDefaultPartitioning() {
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    dag.addStream("node1.outport1", node1.outport1, node2.inport2, node2.inport1);
    int initialPartitionCount = 5;
    OperatorMeta node2Decl = dag.getMeta(node2);
    node2Decl.getAttributes().put(OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(initialPartitionCount));
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTOperator> n2Instances = plan.getOperators(node2Decl);
    Assert.assertEquals("partition instances " + n2Instances, initialPartitionCount, n2Instances.size());
    List<Integer> assignedPartitionKeys = Lists.newArrayList();
    for (int i = 0; i < n2Instances.size(); i++) {
        PTOperator n2Partition = n2Instances.get(i);
        Assert.assertNotNull("partition keys null: " + n2Partition, n2Partition.getPartitionKeys());
        Map<InputPort<?>, PartitionKeys> pkeys = n2Partition.getPartitionKeys();
        // one port partitioned
        Assert.assertEquals("partition keys size: " + pkeys, 1, pkeys.size());
        InputPort<?> expectedPort = node2.inport2;
        Assert.assertEquals("partition port: " + pkeys, expectedPort, pkeys.keySet().iterator().next());
        Assert.assertEquals("partition mask: " + pkeys, "111", Integer.toBinaryString(pkeys.get(expectedPort).mask));
        Set<Integer> pks = pkeys.get(expectedPort).partitions;
        Assert.assertTrue("number partition keys: " + pkeys, pks.size() == 1 || pks.size() == 2);
        assignedPartitionKeys.addAll(pks);
    }
    int expectedMask = Integer.parseInt("111", 2);
    Assert.assertEquals("assigned partitions ", expectedMask + 1, assignedPartitionKeys.size());
    for (int i = 0; i <= expectedMask; i++) {
        Assert.assertTrue("" + assignedPartitionKeys, assignedPartitionKeys.contains(i));
    }
}