Examples with HostInfo org.apache.kafka.streams.state.HostInfo used on opensource projects

Example 46 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method testAssignWithStandbyReplicas.

@Test
public void testAssignWithStandbyReplicas() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addSource(null, "source2", null, null, null, "topic2");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source1", "source2");
    builder.addStateStore(new MockKeyValueStoreBuilder("store1", false), "processor");
    final List<String> topics = asList("topic1", "topic2");
    final Set<TopicPartition> allTopicPartitions = topics.stream().map(topic -> asList(new TopicPartition(topic, 0), new TopicPartition(topic, 1), new TopicPartition(topic, 2))).flatMap(Collection::stream).collect(Collectors.toSet());
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2);
    final Set<TaskId> prevTasks00 = mkSet(TASK_0_0);
    final Set<TaskId> prevTasks01 = mkSet(TASK_0_1);
    final Set<TaskId> prevTasks02 = mkSet(TASK_0_2);
    final Set<TaskId> standbyTasks00 = mkSet(TASK_0_0);
    final Set<TaskId> standbyTasks01 = mkSet(TASK_0_1);
    final Set<TaskId> standbyTasks02 = mkSet(TASK_0_2);
    createMockTaskManager(prevTasks00, standbyTasks01);
    adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(singletonList(APPLICATION_ID + "-store1-changelog"), singletonList(3)));
    configurePartitionAssignorWith(Collections.singletonMap(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1));
    subscriptions.put("consumer10", new Subscription(topics, getInfo(UUID_1, prevTasks00, EMPTY_TASKS, USER_END_POINT).encode()));
    subscriptions.put("consumer11", new Subscription(topics, getInfo(UUID_1, prevTasks01, standbyTasks02, USER_END_POINT).encode()));
    subscriptions.put("consumer20", new Subscription(topics, getInfo(UUID_2, prevTasks02, standbyTasks00, OTHER_END_POINT).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    // the first consumer
    final AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10"));
    final Set<TaskId> allActiveTasks = new HashSet<>(info10.activeTasks());
    final Set<TaskId> allStandbyTasks = new HashSet<>(info10.standbyTasks().keySet());
    // the second consumer
    final AssignmentInfo info11 = checkAssignment(allTopics, assignments.get("consumer11"));
    allActiveTasks.addAll(info11.activeTasks());
    allStandbyTasks.addAll(info11.standbyTasks().keySet());
    assertNotEquals("same processId has same set of standby tasks", info11.standbyTasks().keySet(), info10.standbyTasks().keySet());
    // check active tasks assigned to the first client
    assertEquals(mkSet(TASK_0_0, TASK_0_1), new HashSet<>(allActiveTasks));
    assertEquals(mkSet(TASK_0_2), new HashSet<>(allStandbyTasks));
    // the third consumer
    final AssignmentInfo info20 = checkAssignment(allTopics, assignments.get("consumer20"));
    allActiveTasks.addAll(info20.activeTasks());
    allStandbyTasks.addAll(info20.standbyTasks().keySet());
    // all task ids are in the active tasks and also in the standby tasks
    assertEquals(3, allActiveTasks.size());
    assertEquals(allTasks, allActiveTasks);
    assertEquals(3, allStandbyTasks.size());
    assertEquals(allTasks, allStandbyTasks);
    // Check host partition assignments
    final Map<HostInfo, Set<TopicPartition>> partitionsByHost = info10.partitionsByHost();
    assertEquals(2, partitionsByHost.size());
    assertEquals(allTopicPartitions, partitionsByHost.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()));
    final Map<HostInfo, Set<TopicPartition>> standbyPartitionsByHost = info10.standbyPartitionByHost();
    assertEquals(2, standbyPartitionsByHost.size());
    assertEquals(allTopicPartitions, standbyPartitionsByHost.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()));
    for (final HostInfo hostInfo : partitionsByHost.keySet()) {
        assertTrue(Collections.disjoint(partitionsByHost.get(hostInfo), standbyPartitionsByHost.get(hostInfo)));
    }
    // All consumers got the same host info
    assertEquals(partitionsByHost, info11.partitionsByHost());
    assertEquals(partitionsByHost, info20.partitionsByHost());
    assertEquals(standbyPartitionsByHost, info11.standbyPartitionByHost());
    assertEquals(standbyPartitionsByHost, info20.standbyPartitionByHost());
}

Example 47 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamsMetadataStateTest method before.

@Before
public void before() {
    builder = new StreamsBuilder();
    final KStream<Object, Object> one = builder.stream("topic-one");
    one.groupByKey().count(Materialized.as("table-one"));
    final KStream<Object, Object> two = builder.stream("topic-two");
    two.groupByKey().count(Materialized.as("table-two"));
    builder.stream("topic-three").groupByKey().count(Materialized.as("table-three"));
    one.merge(two).groupByKey().count(Materialized.as("merged-table"));
    builder.stream("topic-four").mapValues(value -> value);
    builder.globalTable("global-topic", Consumed.with(null, null), Materialized.as(globalTable));
    TopologyWrapper.getInternalTopologyBuilder(builder.build()).setApplicationId("appId");
    topic1P0 = new TopicPartition("topic-one", 0);
    topic1P1 = new TopicPartition("topic-one", 1);
    topic2P0 = new TopicPartition("topic-two", 0);
    topic2P1 = new TopicPartition("topic-two", 1);
    topic3P0 = new TopicPartition("topic-three", 0);
    topic4P0 = new TopicPartition("topic-four", 0);
    hostOne = new HostInfo("host-one", 8080);
    hostTwo = new HostInfo("host-two", 9090);
    hostThree = new HostInfo("host-three", 7070);
    hostToActivePartitions = new HashMap<>();
    hostToActivePartitions.put(hostOne, mkSet(topic1P0, topic2P1, topic4P0));
    hostToActivePartitions.put(hostTwo, mkSet(topic2P0, topic1P1));
    hostToActivePartitions.put(hostThree, Collections.singleton(topic3P0));
    hostToStandbyPartitions = new HashMap<>();
    hostToStandbyPartitions.put(hostThree, mkSet(topic1P0, topic2P1, topic4P0));
    hostToStandbyPartitions.put(hostOne, mkSet(topic2P0, topic1P1));
    hostToStandbyPartitions.put(hostTwo, Collections.singleton(topic3P0));
    final List<PartitionInfo> partitionInfos = Arrays.asList(new PartitionInfo("topic-one", 0, null, null, null), new PartitionInfo("topic-one", 1, null, null, null), new PartitionInfo("topic-two", 0, null, null, null), new PartitionInfo("topic-two", 1, null, null, null), new PartitionInfo("topic-three", 0, null, null, null), new PartitionInfo("topic-four", 0, null, null, null));
    cluster = new Cluster(null, Collections.<Node>emptyList(), partitionInfos, Collections.<String>emptySet(), Collections.<String>emptySet());
    final TopologyMetadata topologyMetadata = new TopologyMetadata(TopologyWrapper.getInternalTopologyBuilder(builder.build()), new DummyStreamsConfig());
    topologyMetadata.buildAndRewriteTopology();
    metadataState = new StreamsMetadataState(topologyMetadata, hostOne, logContext);
    metadataState.onChange(hostToActivePartitions, hostToStandbyPartitions, cluster);
    partitioner = (topic, key, value, numPartitions) -> 1;
    storeNames = mkSet("table-one", "table-two", "merged-table", globalTable);
}

Example 48 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project ksql by confluentinc.

the class ServerUtilTest method shouldReturnServerPortWithTrailingSlash.

@Test
public void shouldReturnServerPortWithTrailingSlash() {
    // When:
    final HostInfo hostInfo = ServerUtil.parseHostInfo("http://localhost:8088/");
    // Then:
    assertThat(hostInfo.port(), Matchers.is(8088));
}

Example 49 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project ksql by confluentinc.

the class ActiveHostFilterTest method setUp.

@Before
public void setUp() {
    activeHost = new KsqlHostInfo("activeHost", 2345);
    HostInfo activeHostInfo = new HostInfo("activeHost", 2345);
    standByHost = new KsqlHostInfo("standby1", 1234);
    activeHostFilter = new ActiveHostFilter(activeHostInfo);
}

Example 50 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project ksql by confluentinc.

the class KsqlResource method configure.

@Override
public void configure(final KsqlConfig config) {
    if (!config.getKsqlStreamConfigProps().containsKey(StreamsConfig.APPLICATION_SERVER_CONFIG)) {
        throw new IllegalArgumentException("Need KS application server set");
    }
    final String applicationServer = (String) config.getKsqlStreamConfigProps().get(StreamsConfig.APPLICATION_SERVER_CONFIG);
    final HostInfo hostInfo = ServerUtil.parseHostInfo(applicationServer);
    this.localHost = new KsqlHostInfo(hostInfo.host(), hostInfo.port());
    try {
        this.localUrl = new URL(applicationServer);
    } catch (final Exception e) {
        throw new IllegalStateException("Failed to convert remote host info to URL." + " remoteInfo: " + localHost.host() + ":" + localHost.host());
    }
    this.validator = new RequestValidator(CustomValidators.VALIDATOR_MAP, injectorFactory, ksqlEngine::createSandbox, new ValidatedCommandFactory());
    this.handler = new RequestHandler(customExecutors.EXECUTOR_MAP, new DistributingExecutor(config, commandRunner.getCommandQueue(), distributedCmdResponseTimeout, injectorFactory, authorizationValidator, new ValidatedCommandFactory(), errorHandler, commandRunnerWarning), ksqlEngine, new DefaultCommandQueueSync(commandRunner.getCommandQueue(), this::shouldSynchronize, distributedCmdResponseTimeout));
}