Examples with EventHubClientManager org.apache.samza.system.eventhub.EventHubClientManager used on opensource projects

Example 6 with EventHubClientManager

use of org.apache.samza.system.eventhub.EventHubClientManager in project samza by apache.

the class EventHubSystemConsumer method initializeEventHubsManagers.

private synchronized void initializeEventHubsManagers() {
    LOG.info("Starting EventHubSystemConsumer. Count of SSPs registered: " + streamPartitionOffsets.entrySet().size());
    eventHubNonTransientError.set(null);
    // Create receivers for Event Hubs
    for (Map.Entry<SystemStreamPartition, String> entry : streamPartitionOffsets.entrySet()) {
        SystemStreamPartition ssp = entry.getKey();
        String streamId = config.getStreamId(ssp.getStream());
        Integer partitionId = ssp.getPartition().getPartitionId();
        String offset = entry.getValue();
        String consumerGroup = config.getStreamConsumerGroup(systemName, streamId);
        String namespace = config.getStreamNamespace(systemName, streamId);
        String entityPath = config.getStreamEntityPath(systemName, streamId);
        EventHubClientManager eventHubClientManager = createOrGetEventHubClientManagerForSSP(streamId, ssp);
        try {
            PartitionReceiver receiver;
            if (END_OF_STREAM.equals(offset)) {
                // If the offset is greater than the newest offset, use the use current Instant as
                // offset to fetch in Eventhub.
                receiver = eventHubClientManager.getEventHubClient().createReceiver(consumerGroup, partitionId.toString(), EventPosition.fromEnqueuedTime(Instant.now())).get(DEFAULT_EVENTHUB_CREATE_RECEIVER_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            } else {
                // EventHub will return the first message AFTER the offset that was specified in the fetch request.
                // If no such offset exists Eventhub will return an error.
                receiver = eventHubClientManager.getEventHubClient().createReceiver(consumerGroup, partitionId.toString(), EventPosition.fromOffset(offset, /* inclusiveFlag */
                false)).get(DEFAULT_EVENTHUB_CREATE_RECEIVER_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
            }
            receiver.setPrefetchCount(prefetchCount);
            PartitionReceiveHandler handler = new PartitionReceiverHandlerImpl(ssp, eventReadRates.get(streamId), eventByteReadRates.get(streamId), consumptionLagMs.get(streamId), readErrors.get(streamId), interceptors.getOrDefault(streamId, null), config.getMaxEventCountPerPoll(systemName));
            // Timeout for EventHubClient receive
            receiver.setReceiveTimeout(DEFAULT_EVENTHUB_RECEIVER_TIMEOUT);
            // Start the receiver thread
            receiver.setReceiveHandler(handler);
            streamPartitionHandlers.put(ssp, handler);
            streamPartitionReceivers.put(ssp, receiver);
        } catch (Exception e) {
            throw new SamzaException(String.format("Failed to create receiver for EventHubs: namespace=%s, entity=%s, partitionId=%d", namespace, entityPath, partitionId), e);
        }
        LOG.info(String.format("Connection successfully started for namespace=%s, entity=%s ", namespace, entityPath));
    }
}

Example 7 with EventHubClientManager

use of org.apache.samza.system.eventhub.EventHubClientManager in project samza by apache.

the class EventHubSystemConsumer method createOrGetEventHubClientManagerForSSP.

// Based on the config PerPartitionConnection, create or get EventHubClientManager for the SSP
// Note: this should be used only when starting up. After initialization, directly use perPartitionEventHubManagers
// to obtain the corresponding EventHubClientManager
private EventHubClientManager createOrGetEventHubClientManagerForSSP(String streamId, SystemStreamPartition ssp) {
    EventHubClientManager eventHubClientManager;
    if (config.getPerPartitionConnection(systemName)) {
        // will create one EventHub client per partition
        if (perPartitionEventHubManagers.containsKey(ssp)) {
            LOG.warn(String.format("Trying to create new EventHubClientManager for ssp=%s. But one already exists", ssp));
            eventHubClientManager = perPartitionEventHubManagers.get(ssp);
        } else {
            LOG.info("Creating EventHub client manager for SSP: " + ssp);
            eventHubClientManager = eventHubClientManagerFactory.getEventHubClientManager(systemName, streamId, config);
            eventHubClientManager.init();
            perPartitionEventHubManagers.put(ssp, eventHubClientManager);
        }
    } else {
        // will share one EventHub client per stream
        if (!perStreamEventHubManagers.containsKey(streamId)) {
            LOG.info("Creating EventHub client manager for stream: " + streamId);
            EventHubClientManager perStreamEventHubClientManager = eventHubClientManagerFactory.getEventHubClientManager(systemName, streamId, config);
            perStreamEventHubClientManager.init();
            perStreamEventHubManagers.put(streamId, perStreamEventHubClientManager);
        }
        eventHubClientManager = perStreamEventHubManagers.get(streamId);
        perPartitionEventHubManagers.put(ssp, eventHubClientManager);
    }
    LOG.info("EventHub client created for ssp: " + ssp);
    Validate.notNull(eventHubClientManager, String.format("Fail to create or get EventHubClientManager for ssp=%s", ssp));
    return eventHubClientManager;
}

Example 8 with EventHubClientManager

use of org.apache.samza.system.eventhub.EventHubClientManager in project samza by apache.

the class EventHubSystemAdmin method getSystemStreamMetadata.

@Override
public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
    Map<String, SystemStreamMetadata> requestedMetadata = new HashMap<>();
    try {
        for (String streamName : streamNames) {
            if (!streamPartitions.containsKey(streamName)) {
                LOG.debug(String.format("Partition ids for Stream=%s not found", streamName));
                EventHubClientManager eventHubClientManager = getOrCreateStreamEventHubClient(streamName);
                EventHubClient ehClient = eventHubClientManager.getEventHubClient();
                CompletableFuture<EventHubRuntimeInformation> runtimeInfo = ehClient.getRuntimeInformation();
                long timeoutMs = eventHubConfig.getRuntimeInfoWaitTimeMS(systemName);
                EventHubRuntimeInformation ehInfo = runtimeInfo.get(timeoutMs, TimeUnit.MILLISECONDS);
                LOG.info(String.format("Adding partition ids=%s for stream=%s. EHRuntimetInfo=%s", Arrays.toString(ehInfo.getPartitionIds()), streamName, printEventHubRuntimeInfo(ehInfo)));
                streamPartitions.put(streamName, ehInfo.getPartitionIds());
            }
            String[] partitionIds = streamPartitions.get(streamName);
            Map<Partition, SystemStreamPartitionMetadata> sspMetadataMap = getPartitionMetadata(streamName, partitionIds);
            SystemStreamMetadata systemStreamMetadata = new SystemStreamMetadata(streamName, sspMetadataMap);
            requestedMetadata.put(streamName, systemStreamMetadata);
        }
        return requestedMetadata;
    } catch (Exception e) {
        String msg = String.format("Error while fetching EventHubRuntimeInfo for System:%s", systemName);
        LOG.error(msg, e);
        throw new SamzaException(msg, e);
    }
}

Example 9 with EventHubClientManager

use of org.apache.samza.system.eventhub.EventHubClientManager in project samza by apache.

the class EventHubSystemConsumer method renewPartitionReceiver.

private void renewPartitionReceiver(SystemStreamPartition ssp) {
    String streamId = config.getStreamId(ssp.getStream());
    EventHubClientManager eventHubClientManager = perPartitionEventHubManagers.get(ssp);
    String offset = streamPartitionOffsets.get(ssp);
    Integer partitionId = ssp.getPartition().getPartitionId();
    String consumerGroup = config.getStreamConsumerGroup(ssp.getSystem(), streamId);
    try {
        // Close current receiver
        streamPartitionReceivers.get(ssp).close().get(DEFAULT_SHUTDOWN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        // Recreate receiver
        PartitionReceiver receiver = eventHubClientManager.getEventHubClient().createReceiverSync(consumerGroup, partitionId.toString(), EventPosition.fromOffset(offset, !offset.equals(EventHubSystemConsumer.START_OF_STREAM)));
        receiver.setPrefetchCount(prefetchCount);
        // Timeout for EventHubClient receive
        receiver.setReceiveTimeout(DEFAULT_EVENTHUB_RECEIVER_TIMEOUT);
        // Create and start receiver thread with handler
        receiver.setReceiveHandler(streamPartitionHandlers.get(ssp));
        streamPartitionReceivers.put(ssp, receiver);
    } catch (Exception e) {
        eventHubNonTransientError.set(new SamzaException(String.format("Failed to recreate receiver for EventHubs after ReceiverHandlerError (ssp=%s)", ssp), e));
    }
}

Example 10 with EventHubClientManager

use of org.apache.samza.system.eventhub.EventHubClientManager in project samza by apache.

the class EventHubSystemProducer method sendAsync.

@Override
public synchronized CompletableFuture<Void> sendAsync(String source, OutgoingMessageEnvelope envelope) {
    LOG.debug(String.format("Trying to send %s", envelope));
    if (!isStarted) {
        throw new SamzaException("Trying to call send before the producer is started.");
    }
    if (!isInitialized) {
        // lazy initialization on the first send
        init();
    }
    String streamId = config.getStreamId(envelope.getSystemStream().getStream());
    if (!perStreamEventHubClientManagers.containsKey(streamId)) {
        String msg = String.format("Trying to send event to a destination {%s} that is not registered.", streamId);
        throw new SamzaException(msg);
    }
    EventData eventData = createEventData(streamId, envelope);
    // SAMZA-1654: waiting for the client library to expose the API to calculate the exact size of the AMQP message
    // https://github.com/Azure/azure-event-hubs-java/issues/305
    int eventDataLength = eventData.getBytes() == null ? 0 : eventData.getBytes().length;
    // If the maxMessageSize is lesser than zero, then it means there is no message size restriction.
    if (this.maxMessageSize > 0 && eventDataLength > this.maxMessageSize) {
        LOG.info("Received a message with size {} > maxMessageSize configured {(}), Skipping it", eventDataLength, this.maxMessageSize);
        eventSkipRate.get(streamId).inc();
        aggEventSkipRate.inc();
        return CompletableFuture.completedFuture(null);
    }
    eventWriteRate.get(streamId).inc();
    aggEventWriteRate.inc();
    eventByteWriteRate.get(streamId).inc(eventDataLength);
    aggEventByteWriteRate.inc(eventDataLength);
    EventHubClientManager ehClient = perStreamEventHubClientManagers.get(streamId);
    // Async send call
    return sendToEventHub(streamId, eventData, getEnvelopePartitionId(envelope), ehClient.getEventHubClient());
}