Search in sources :

Example 11 with KafkaBindingConfig

use of io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig in project zilla by aklivity.

the class KafkaCacheMetaFactory method newStream.

@Override
public MessageConsumer newStream(int msgTypeId, DirectBuffer buffer, int index, int length, MessageConsumer sender) {
    final BeginFW begin = beginRO.wrap(buffer, index, index + length);
    final long routeId = begin.routeId();
    final long initialId = begin.streamId();
    final long authorization = begin.authorization();
    final long affinity = begin.affinity();
    assert (initialId & 0x0000_0000_0000_0001L) != 0L;
    final OctetsFW extension = begin.extension();
    final ExtensionFW beginEx = extension.get(extensionRO::tryWrap);
    assert beginEx != null && beginEx.typeId() == kafkaTypeId;
    final KafkaBeginExFW kafkaBeginEx = extension.get(kafkaBeginExRO::wrap);
    assert kafkaBeginEx.kind() == KafkaBeginExFW.KIND_META;
    final KafkaMetaBeginExFW kafkaMetaBeginEx = kafkaBeginEx.meta();
    final String16FW beginTopic = kafkaMetaBeginEx.topic();
    final String topicName = beginTopic.asString();
    MessageConsumer newStream = null;
    final KafkaBindingConfig binding = supplyBinding.apply(routeId);
    final KafkaRouteConfig resolved = binding != null ? binding.resolve(authorization, topicName) : null;
    if (resolved != null) {
        final long resolvedId = resolved.id;
        final KafkaCacheRoute cacheRoute = supplyCacheRoute.apply(resolvedId);
        final int topicKey = cacheRoute.topicKey(topicName);
        KafkaCacheMetaFanout fanout = cacheRoute.metaFanoutsByTopic.get(topicKey);
        if (fanout == null) {
            final long cacheId = supplyCacheId.applyAsLong(routeId, resolvedId);
            final String cacheName = String.format("%s.%s", supplyNamespace.apply(cacheId), supplyLocalName.apply(cacheId));
            final KafkaCache cache = supplyCache.apply(cacheName);
            final KafkaCacheTopic topic = cache.supplyTopic(topicName);
            final KafkaCacheMetaFanout newFanout = new KafkaCacheMetaFanout(resolvedId, authorization, topic);
            cacheRoute.metaFanoutsByTopic.put(topicKey, newFanout);
            fanout = newFanout;
        }
        if (fanout != null) {
            newStream = new KafkaCacheMetaStream(fanout, sender, routeId, initialId, affinity, authorization)::onMetaMessage;
        }
    }
    return newStream;
}
Also used : ExtensionFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.ExtensionFW) MessageConsumer(io.aklivity.zilla.runtime.engine.binding.function.MessageConsumer) KafkaMetaBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaMetaBeginExFW) OctetsFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.OctetsFW) BeginFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.BeginFW) KafkaBindingConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig) KafkaBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaBeginExFW) KafkaCacheTopic(io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCacheTopic) KafkaCache(io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCache) KafkaRouteConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaRouteConfig) String16FW(io.aklivity.zilla.runtime.binding.kafka.internal.types.String16FW)

Example 12 with KafkaBindingConfig

use of io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig in project zilla by aklivity.

the class KafkaCacheServerFetchFactory method newStream.

@Override
public MessageConsumer newStream(int msgTypeId, DirectBuffer buffer, int index, int length, MessageConsumer sender) {
    final BeginFW begin = beginRO.wrap(buffer, index, index + length);
    final long routeId = begin.routeId();
    final long initialId = begin.streamId();
    final long affinity = begin.affinity();
    final long authorization = begin.authorization();
    assert (initialId & 0x0000_0000_0000_0001L) != 0L;
    final OctetsFW extension = begin.extension();
    final ExtensionFW beginEx = extension.get(extensionRO::wrap);
    assert beginEx != null && beginEx.typeId() == kafkaTypeId;
    final KafkaBeginExFW kafkaBeginEx = extension.get(kafkaBeginExRO::wrap);
    final KafkaFetchBeginExFW kafkaFetchBeginEx = kafkaBeginEx.fetch();
    final String16FW beginTopic = kafkaFetchBeginEx.topic();
    final KafkaOffsetFW progress = kafkaFetchBeginEx.partition();
    final int partitionId = progress.partitionId();
    final long partitionOffset = progress.partitionOffset();
    final KafkaDeltaType deltaType = kafkaFetchBeginEx.deltaType().get();
    final String topicName = beginTopic.asString();
    MessageConsumer newStream = null;
    final KafkaBindingConfig binding = supplyBinding.apply(routeId);
    final KafkaRouteConfig resolved = binding != null ? binding.resolve(authorization, topicName) : null;
    if (resolved != null) {
        final long resolvedId = resolved.id;
        final KafkaCacheRoute cacheRoute = supplyCacheRoute.apply(resolvedId);
        final long partitionKey = cacheRoute.topicPartitionKey(topicName, partitionId);
        KafkaCacheServerFetchFanout fanout = cacheRoute.serverFetchFanoutsByTopicPartition.get(partitionKey);
        if (fanout == null) {
            final KafkaTopicConfig topic = binding.topic(topicName);
            final KafkaDeltaType routeDeltaType = topic != null ? topic.deltaType : deltaType;
            final KafkaOffsetType defaultOffset = topic != null ? topic.defaultOffset : HISTORICAL;
            final String cacheName = String.format("%s.%s", supplyNamespace.apply(routeId), supplyLocalName.apply(routeId));
            final KafkaCache cache = supplyCache.apply(cacheName);
            final KafkaCacheTopic cacheTopic = cache.supplyTopic(topicName);
            final KafkaCachePartition partition = cacheTopic.supplyFetchPartition(partitionId);
            final KafkaCacheServerFetchFanout newFanout = new KafkaCacheServerFetchFanout(resolvedId, authorization, affinity, partition, routeDeltaType, defaultOffset);
            cacheRoute.serverFetchFanoutsByTopicPartition.put(partitionKey, newFanout);
            fanout = newFanout;
        }
        final int leaderId = cacheRoute.leadersByPartitionId.get(partitionId);
        newStream = new KafkaCacheServerFetchStream(fanout, sender, routeId, initialId, leaderId, authorization, partitionOffset)::onServerMessage;
    }
    return newStream;
}
Also used : ExtensionFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.ExtensionFW) MessageConsumer(io.aklivity.zilla.runtime.engine.binding.function.MessageConsumer) KafkaCachePartition(io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCachePartition) KafkaDeltaType(io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaDeltaType) KafkaTopicConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaTopicConfig) OctetsFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.OctetsFW) KafkaFetchBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaFetchBeginExFW) BeginFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.BeginFW) KafkaBindingConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig) KafkaBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaBeginExFW) KafkaCacheTopic(io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCacheTopic) KafkaCache(io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCache) KafkaRouteConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaRouteConfig) KafkaOffsetFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaOffsetFW) KafkaOffsetType(io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaOffsetType) String16FW(io.aklivity.zilla.runtime.binding.kafka.internal.types.String16FW)

Example 13 with KafkaBindingConfig

use of io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig in project zilla by aklivity.

the class KafkaCacheClientDescribeFactory method newStream.

@Override
public MessageConsumer newStream(int msgTypeId, DirectBuffer buffer, int index, int length, MessageConsumer sender) {
    final BeginFW begin = beginRO.wrap(buffer, index, index + length);
    final long routeId = begin.routeId();
    final long initialId = begin.streamId();
    final long authorization = begin.authorization();
    final long affinity = begin.affinity();
    assert (initialId & 0x0000_0000_0000_0001L) != 0L;
    final OctetsFW extension = begin.extension();
    final ExtensionFW beginEx = extension.get(extensionRO::tryWrap);
    assert beginEx != null && beginEx.typeId() == kafkaTypeId;
    final KafkaBeginExFW kafkaBeginEx = extension.get(kafkaBeginExRO::tryWrap);
    assert kafkaBeginEx.kind() == KafkaBeginExFW.KIND_DESCRIBE;
    final KafkaDescribeBeginExFW kafkaDescribeBeginEx = kafkaBeginEx.describe();
    final String16FW topic = kafkaDescribeBeginEx.topic();
    final String topicName = topic.asString();
    MessageConsumer newStream = null;
    final KafkaBindingConfig binding = supplyBinding.apply(routeId);
    final KafkaRouteConfig resolved = binding != null ? binding.resolve(authorization, topicName) : null;
    if (resolved != null) {
        final long resolvedId = resolved.id;
        final KafkaCacheRoute cacheRoute = supplyCacheRoute.apply(resolvedId);
        final int topicKey = cacheRoute.topicKey(topicName);
        KafkaCacheClientDescribeFanout fanout = cacheRoute.clientDescribeFanoutsByTopic.get(topicKey);
        if (fanout == null) {
            final List<String> configNames = new ArrayList<>();
            kafkaDescribeBeginEx.configs().forEach(c -> configNames.add(c.asString()));
            final KafkaCacheClientDescribeFanout newFanout = new KafkaCacheClientDescribeFanout(resolvedId, authorization, topicName, configNames);
            cacheRoute.clientDescribeFanoutsByTopic.put(topicKey, newFanout);
            fanout = newFanout;
        }
        if (fanout != null) {
            newStream = new KafkaCacheClientDescribeStream(fanout, sender, routeId, initialId, affinity, authorization)::onDescribeMessage;
        }
    }
    return newStream;
}
Also used : ExtensionFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.ExtensionFW) MessageConsumer(io.aklivity.zilla.runtime.engine.binding.function.MessageConsumer) OctetsFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.OctetsFW) ArrayList(java.util.ArrayList) BeginFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.BeginFW) KafkaBindingConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig) KafkaBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaBeginExFW) KafkaDescribeBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaDescribeBeginExFW) KafkaRouteConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaRouteConfig) String16FW(io.aklivity.zilla.runtime.binding.kafka.internal.types.String16FW)

Example 14 with KafkaBindingConfig

use of io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig in project zilla by aklivity.

the class KafkaCacheServerAddressFactory method onAttached.

void onAttached(long bindingId) {
    final KafkaBindingConfig binding = supplyBinding.apply(bindingId);
    final List<String> bootstrap = binding.options != null ? binding.options.bootstrap : null;
    if (bootstrap != null) {
        List<KafkaAddressStream> bootstraps = bootstrap.stream().map(t -> new KafkaAddressStream(bindingId, 0, t)).collect(toList());
        streams.put(bindingId, bootstraps);
        bootstraps.forEach(KafkaAddressStream::doKafkaInitialBegin);
    }
}
Also used : KafkaConfiguration(io.aklivity.zilla.runtime.binding.kafka.internal.KafkaConfiguration) WindowFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.WindowFW) EngineContext(io.aklivity.zilla.runtime.engine.EngineContext) EndFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.EndFW) LongFunction(java.util.function.LongFunction) LongSupplier(java.util.function.LongSupplier) BindingHandler(io.aklivity.zilla.runtime.engine.binding.BindingHandler) KafkaBinding(io.aklivity.zilla.runtime.binding.kafka.internal.KafkaBinding) MessageConsumer(io.aklivity.zilla.runtime.engine.binding.function.MessageConsumer) KafkaBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaBeginExFW) KafkaBindingConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig) BeginFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.BeginFW) Consumer(java.util.function.Consumer) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) Long2ObjectHashMap(org.agrona.collections.Long2ObjectHashMap) OctetsFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.OctetsFW) LongUnaryOperator(java.util.function.LongUnaryOperator) AbortFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.AbortFW) MutableDirectBuffer(org.agrona.MutableDirectBuffer) ResetFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.ResetFW) DirectBuffer(org.agrona.DirectBuffer) KafkaBindingConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig)

Example 15 with KafkaBindingConfig

use of io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig in project zilla by aklivity.

the class KafkaCacheServerBootstrapFactory method newStream.

@Override
public MessageConsumer newStream(int msgTypeId, DirectBuffer buffer, int index, int length, MessageConsumer sender) {
    final BeginFW begin = beginRO.wrap(buffer, index, index + length);
    final long routeId = begin.routeId();
    final long initialId = begin.streamId();
    final long authorization = begin.authorization();
    final long affinity = begin.affinity();
    assert (initialId & 0x0000_0000_0000_0001L) != 0L;
    final OctetsFW extension = begin.extension();
    final ExtensionFW beginEx = extensionRO.tryWrap(extension.buffer(), extension.offset(), extension.limit());
    final KafkaBeginExFW kafkaBeginEx = beginEx != null && beginEx.typeId() == kafkaTypeId ? kafkaBeginExRO.tryWrap(extension.buffer(), extension.offset(), extension.limit()) : null;
    assert kafkaBeginEx != null;
    assert kafkaBeginEx.kind() == KafkaBeginExFW.KIND_BOOTSTRAP;
    final KafkaBootstrapBeginExFW kafkaBootstrapBeginEx = kafkaBeginEx.bootstrap();
    final String16FW beginTopic = kafkaBootstrapBeginEx.topic();
    final String topicName = beginTopic != null ? beginTopic.asString() : null;
    MessageConsumer newStream = null;
    final KafkaBindingConfig binding = supplyBinding.apply(routeId);
    if (binding != null && binding.bootstrap(topicName)) {
        final KafkaTopicConfig topic = binding.topic(topicName);
        final long resolvedId = routeId;
        final long defaultOffset = topic != null && topic.defaultOffset != null ? topic.defaultOffset.value() : OFFSET_HISTORICAL;
        newStream = new KafkaBootstrapStream(sender, routeId, initialId, affinity, authorization, topicName, resolvedId, defaultOffset)::onBootstrapInitial;
    }
    return newStream;
}
Also used : ExtensionFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.ExtensionFW) MessageConsumer(io.aklivity.zilla.runtime.engine.binding.function.MessageConsumer) KafkaBootstrapBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaBootstrapBeginExFW) KafkaTopicConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaTopicConfig) OctetsFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.OctetsFW) String16FW(io.aklivity.zilla.runtime.binding.kafka.internal.types.String16FW) BeginFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.BeginFW) KafkaBindingConfig(io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig) KafkaBeginExFW(io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaBeginExFW)

Aggregations

KafkaBindingConfig (io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaBindingConfig)17 OctetsFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.OctetsFW)14 BeginFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.BeginFW)14 KafkaBeginExFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaBeginExFW)14 MessageConsumer (io.aklivity.zilla.runtime.engine.binding.function.MessageConsumer)14 String16FW (io.aklivity.zilla.runtime.binding.kafka.internal.types.String16FW)13 ExtensionFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.ExtensionFW)13 KafkaRouteConfig (io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaRouteConfig)11 KafkaCache (io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCache)6 KafkaCacheTopic (io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCacheTopic)6 KafkaCachePartition (io.aklivity.zilla.runtime.binding.kafka.internal.cache.KafkaCachePartition)4 KafkaOffsetFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaOffsetFW)4 KafkaDeltaType (io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaDeltaType)3 KafkaDescribeBeginExFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaDescribeBeginExFW)3 KafkaFetchBeginExFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaFetchBeginExFW)3 KafkaProduceBeginExFW (io.aklivity.zilla.runtime.binding.kafka.internal.types.stream.KafkaProduceBeginExFW)3 ArrayList (java.util.ArrayList)3 KafkaTopicConfig (io.aklivity.zilla.runtime.binding.kafka.internal.config.KafkaTopicConfig)2 KafkaOffsetType (io.aklivity.zilla.runtime.binding.kafka.internal.types.KafkaOffsetType)2 KafkaBinding (io.aklivity.zilla.runtime.binding.kafka.internal.KafkaBinding)1