Examples with StreamConsumer co.cask.cdap.data2.transaction.stream.StreamConsumer used on opensource projects

Example 11 with StreamConsumer

use of co.cask.cdap.data2.transaction.stream.StreamConsumer in project cdap by caskdata.

the class InMemoryStreamConsumerFactory method create.

@Override
public StreamConsumer create(StreamId streamId, String namespace, ConsumerConfig consumerConfig) throws IOException {
    QueueName queueName = QueueName.fromStream(streamId);
    QueueConsumer consumer = queueClientFactory.createConsumer(queueName, consumerConfig, -1);
    return new QueueToStreamConsumer(streamId, consumerConfig, consumer);
}

Example 12 with StreamConsumer

use of co.cask.cdap.data2.transaction.stream.StreamConsumer in project cdap by caskdata.

the class LineageWriterDataFabricFacade method createStreamConsumer.

@Override
public StreamConsumer createStreamConsumer(StreamId streamName, ConsumerConfig consumerConfig) throws IOException {
    String namespace = String.format("%s.%s", programId.getApplicationId(), programId.getId());
    final StreamConsumer consumer = streamConsumerFactory.create(streamName, namespace, consumerConfig);
    datasetCache.addExtraTransactionAware(consumer);
    ProgramContext programContext = this.programContext;
    if (programContext != null) {
        lineageWriter.addAccess(programContext.getProgramRunId(), streamName, AccessType.READ, programContext.getComponentId());
    }
    return new ForwardingStreamConsumer(consumer) {

        @Override
        public void close() throws IOException {
            super.close();
            datasetCache.removeExtraTransactionAware(consumer);
        }
    };
}

Example 13 with StreamConsumer

use of co.cask.cdap.data2.transaction.stream.StreamConsumer in project cdap by caskdata.

the class ConsumerSupplier method open.

/**
   * Updates number of instances for the consumer group that this instance belongs to. It'll close existing
   * consumer and create a new one with the new group size.
   *
   * @param groupSize New group size.
   */
void open(int groupSize) {
    try {
        close();
        ConsumerConfig config = consumerConfig;
        if (groupSize != config.getGroupSize()) {
            config = new ConsumerConfig(consumerConfig.getGroupId(), consumerConfig.getInstanceId(), groupSize, consumerConfig.getDequeueStrategy(), consumerConfig.getHashKey());
        }
        if (queueName.isQueue()) {
            QueueConsumer queueConsumer = dataFabricFacade.createConsumer(queueName, config, numGroups);
            consumerConfig = queueConsumer.getConfig();
            consumer = queueConsumer;
        } else {
            StreamId queueStream = queueName.toStreamId();
            for (EntityId owner : owners) {
                try {
                    runtimeUsageRegistry.register(owner, queueStream);
                } catch (Exception e) {
                    LOG.warn("Failed to register usage of {} -> {}", owner, queueStream, e);
                }
            }
            StreamConsumer streamConsumer = dataFabricFacade.createStreamConsumer(queueName.toStreamId(), config);
            consumerConfig = streamConsumer.getConsumerConfig();
            consumer = streamConsumer;
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}