Examples with MinorFragmentEndpoint org.apache.drill.exec.physical.MinorFragmentEndpoint used on opensource projects

Example 1 with MinorFragmentEndpoint

use of org.apache.drill.exec.physical.MinorFragmentEndpoint in project drill by apache.

the class PartitionerTemplate method setup.

@Override
public final void setup(FragmentContext context, RecordBatch incoming, HashPartitionSender popConfig, OperatorStats stats, OperatorContext oContext, int start, int end) throws SchemaChangeException {
    this.incoming = incoming;
    this.stats = stats;
    this.start = start;
    this.end = end;
    doSetup(context, incoming, null);
    // allocated.
    if (popConfig.getDestinations().size() > 1000) {
        // Always keep the recordCount as (2^x) - 1 to better utilize the memory allocation in ValueVectors
        outgoingRecordBatchSize = (DEFAULT_RECORD_BATCH_SIZE + 1) / 2 - 1;
    }
    int fieldId = 0;
    for (MinorFragmentEndpoint destination : popConfig.getDestinations()) {
        // create outgoingBatches only for subset of Destination Points
        if (fieldId >= start && fieldId < end) {
            logger.debug("start: {}, count: {}, fieldId: {}", start, end, fieldId);
            outgoingBatches.add(newOutgoingRecordBatch(stats, popConfig, context.getDataTunnel(destination.getEndpoint()), context, oContext.getAllocator(), destination.getId()));
        }
        fieldId++;
    }
    for (OutgoingRecordBatch outgoingRecordBatch : outgoingBatches) {
        outgoingRecordBatch.initializeBatch();
    }
    SelectionVectorMode svMode = incoming.getSchema().getSelectionVectorMode();
    switch(svMode) {
        case FOUR_BYTE:
            this.sv4 = incoming.getSelectionVector4();
            break;
        case TWO_BYTE:
            this.sv2 = incoming.getSelectionVector2();
            break;
        case NONE:
            break;
        default:
            throw new UnsupportedOperationException("Unknown selection vector mode: " + svMode.toString());
    }
}

Example 2 with MinorFragmentEndpoint

use of org.apache.drill.exec.physical.MinorFragmentEndpoint in project drill by axbaretto.

the class AbstractDeMuxExchange method createSenderReceiverMapping.

protected void createSenderReceiverMapping() {
    if (isSenderReceiverMappingCreated) {
        return;
    }
    senderToReceiversMapping = ArrayListMultimap.create();
    receiverToSenderMapping = Maps.newHashMap();
    // Find the list of receiver fragment ids assigned to each Drillbit endpoint
    ArrayListMultimap<DrillbitEndpoint, Integer> endpointReceiverList = ArrayListMultimap.create();
    int receiverFragmentId = 0;
    for (DrillbitEndpoint receiverLocation : receiverLocations) {
        endpointReceiverList.put(receiverLocation, receiverFragmentId);
        receiverFragmentId++;
    }
    int senderFragmentId = 0;
    for (DrillbitEndpoint senderLocation : senderLocations) {
        final List<Integer> receiverMinorFragmentIds = endpointReceiverList.get(senderLocation);
        for (Integer receiverId : receiverMinorFragmentIds) {
            receiverToSenderMapping.put(receiverId, new MinorFragmentEndpoint(senderFragmentId, senderLocation));
            senderToReceiversMapping.put(senderFragmentId, new MinorFragmentEndpoint(receiverId, receiverLocations.get(receiverId)));
        }
        senderFragmentId++;
    }
    isSenderReceiverMappingCreated = true;
}

Example 3 with MinorFragmentEndpoint

use of org.apache.drill.exec.physical.MinorFragmentEndpoint in project drill by axbaretto.

the class AbstractMuxExchange method getSender.

@Override
public Sender getSender(int minorFragmentId, PhysicalOperator child) {
    createSenderReceiverMapping();
    MinorFragmentEndpoint receiver = senderToReceiverMapping.get(minorFragmentId);
    if (receiver == null) {
        throw new IllegalStateException(String.format("Failed to find receiver for sender [%d]", minorFragmentId));
    }
    return new SingleSender(receiverMajorFragmentId, receiver.getId(), child, receiver.getEndpoint());
}

Example 4 with MinorFragmentEndpoint

use of org.apache.drill.exec.physical.MinorFragmentEndpoint in project drill by axbaretto.

the class AbstractMuxExchange method createSenderReceiverMapping.

protected void createSenderReceiverMapping() {
    if (isSenderReceiverMappingCreated) {
        return;
    }
    senderToReceiverMapping = Maps.newHashMap();
    receiverToSenderMapping = ArrayListMultimap.create();
    // Find the list of sender fragment ids assigned to each Drillbit endpoint.
    ArrayListMultimap<DrillbitEndpoint, Integer> endpointSenderList = ArrayListMultimap.create();
    int senderFragmentId = 0;
    for (DrillbitEndpoint senderLocation : senderLocations) {
        endpointSenderList.put(senderLocation, senderFragmentId);
        senderFragmentId++;
    }
    int receiverFragmentId = 0;
    for (DrillbitEndpoint receiverLocation : receiverLocations) {
        List<Integer> senderFragmentIds = endpointSenderList.get(receiverLocation);
        for (Integer senderId : senderFragmentIds) {
            senderToReceiverMapping.put(senderId, new MinorFragmentEndpoint(receiverFragmentId, receiverLocation));
            receiverToSenderMapping.put(receiverFragmentId, new MinorFragmentEndpoint(senderId, senderLocations.get(senderId)));
        }
        receiverFragmentId++;
    }
    isSenderReceiverMappingCreated = true;
}

Example 5 with MinorFragmentEndpoint

use of org.apache.drill.exec.physical.MinorFragmentEndpoint in project drill by axbaretto.

the class PartitionSenderRootExec method sendEmptyBatch.

private void sendEmptyBatch(boolean isLast) {
    BatchSchema schema = incoming.getSchema();
    if (schema == null) {
        // If the incoming batch has no schema (possible when there are no input records),
        // create an empty schema to avoid NPE.
        schema = BatchSchema.newBuilder().build();
    }
    FragmentHandle handle = context.getHandle();
    for (MinorFragmentEndpoint destination : popConfig.getDestinations()) {
        AccountingDataTunnel tunnel = context.getDataTunnel(destination.getEndpoint());
        FragmentWritableBatch writableBatch = FragmentWritableBatch.getEmptyBatchWithSchema(isLast, handle.getQueryId(), handle.getMajorFragmentId(), handle.getMinorFragmentId(), operator.getOppositeMajorFragmentId(), destination.getId(), schema);
        stats.startWait();
        try {
            tunnel.sendRecordBatch(writableBatch);
        } finally {
            stats.stopWait();
        }
    }
    stats.addLongStat(Metric.BATCHES_SENT, 1);
}