Examples with TaskId org.apache.kafka.streams.processor.TaskId used on opensource projects

Example 1 with TaskId

use of org.apache.kafka.streams.processor.TaskId in project kafka by apache.

the class StreamPartitionAssignor method onAssignment.

/**
     * @throws TaskAssignmentException if there is no task id for one of the partitions specified
     */
@Override
public void onAssignment(Assignment assignment) {
    List<TopicPartition> partitions = new ArrayList<>(assignment.partitions());
    Collections.sort(partitions, PARTITION_COMPARATOR);
    AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
    this.standbyTasks = info.standbyTasks;
    this.activeTasks = new HashMap<>();
    // could be duplicated if one task has more than one assigned partitions
    if (partitions.size() != info.activeTasks.size()) {
        throw new TaskAssignmentException(String.format("stream-thread [%s] Number of assigned partitions %d is not equal to the number of active taskIds %d" + ", assignmentInfo=%s", streamThread.getName(), partitions.size(), info.activeTasks.size(), info.toString()));
    }
    for (int i = 0; i < partitions.size(); i++) {
        TopicPartition partition = partitions.get(i);
        TaskId id = info.activeTasks.get(i);
        Set<TopicPartition> assignedPartitions = activeTasks.get(id);
        if (assignedPartitions == null) {
            assignedPartitions = new HashSet<>();
            activeTasks.put(id, assignedPartitions);
        }
        assignedPartitions.add(partition);
    }
    this.partitionsByHostState = info.partitionsByHost;
    final Collection<Set<TopicPartition>> values = partitionsByHostState.values();
    final Map<TopicPartition, PartitionInfo> topicToPartitionInfo = new HashMap<>();
    for (Set<TopicPartition> value : values) {
        for (TopicPartition topicPartition : value) {
            topicToPartitionInfo.put(topicPartition, new PartitionInfo(topicPartition.topic(), topicPartition.partition(), null, new Node[0], new Node[0]));
        }
    }
    metadataWithInternalTopics = Cluster.empty().withPartitions(topicToPartitionInfo);
}

Example 2 with TaskId

use of org.apache.kafka.streams.processor.TaskId in project kafka by apache.

the class StreamThread method cachedTasks.

/**
     * Returns ids of tasks whose states are kept on the local storage.
     */
public Set<TaskId> cachedTasks() {
    // A client could contain some inactive tasks whose states are still kept on the local storage in the following scenarios:
    // 1) the client is actively maintaining standby tasks by maintaining their states from the change log.
    // 2) the client has just got some tasks migrated out of itself to other clients while these task states
    //    have not been cleaned up yet (this can happen in a rolling bounce upgrade, for example).
    HashSet<TaskId> tasks = new HashSet<>();
    File[] stateDirs = stateDirectory.listTaskDirectories();
    if (stateDirs != null) {
        for (File dir : stateDirs) {
            try {
                TaskId id = TaskId.parse(dir.getName());
                // if the checkpoint file exists, the state is valid.
                if (new File(dir, ProcessorStateManager.CHECKPOINT_FILE_NAME).exists())
                    tasks.add(id);
            } catch (TaskIdFormatException e) {
            // there may be some unknown files that sits in the same directory,
            // we should ignore these files instead trying to delete them as well
            }
        }
    }
    return tasks;
}

Example 3 with TaskId

use of org.apache.kafka.streams.processor.TaskId in project kafka by apache.

the class ProcessorStateManagerTest method testGetStore.

@Test
public void testGetStore() throws IOException {
    final MockStateStoreSupplier.MockStateStore mockStateStore = new MockStateStoreSupplier.MockStateStore(nonPersistentStoreName, false);
    final ProcessorStateManager stateMgr = new ProcessorStateManager(new TaskId(0, 1), noPartitions, false, stateDirectory, Collections.<String, String>emptyMap(), changelogReader);
    try {
        stateMgr.register(mockStateStore, true, mockStateStore.stateRestoreCallback);
        assertNull(stateMgr.getStore("noSuchStore"));
        assertEquals(mockStateStore, stateMgr.getStore(nonPersistentStoreName));
    } finally {
        stateMgr.close(Collections.<TopicPartition, Long>emptyMap());
    }
}

Example 4 with TaskId

use of org.apache.kafka.streams.processor.TaskId in project kafka by apache.

the class ProcessorStateManagerTest method testRegisterNonPersistentStore.

@Test
public void testRegisterNonPersistentStore() throws IOException {
    // non persistent store
    MockStateStoreSupplier.MockStateStore nonPersistentStore = new MockStateStoreSupplier.MockStateStore(nonPersistentStoreName, false);
    ProcessorStateManager stateMgr = new ProcessorStateManager(new TaskId(0, 2), noPartitions, false, stateDirectory, new HashMap<String, String>() {

        {
            put(persistentStoreName, persistentStoreTopicName);
            put(nonPersistentStoreName, nonPersistentStoreTopicName);
        }
    }, changelogReader);
    try {
        stateMgr.register(nonPersistentStore, true, nonPersistentStore.stateRestoreCallback);
        assertTrue(changelogReader.wasRegistered(new TopicPartition(nonPersistentStoreTopicName, 2)));
    } finally {
        stateMgr.close(Collections.<TopicPartition, Long>emptyMap());
    }
}

Example 5 with TaskId

use of org.apache.kafka.streams.processor.TaskId in project kafka by apache.

the class StreamPartitionAssignorTest method testOnAssignment.

@Test
public void testOnAssignment() throws Exception {
    TopicPartition t2p3 = new TopicPartition("topic2", 3);
    TopologyBuilder builder = new TopologyBuilder();
    builder.addSource("source1", "topic1");
    builder.addSource("source2", "topic2");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
    UUID uuid = UUID.randomUUID();
    String client1 = "client1";
    StreamThread thread = new StreamThread(builder, config, mockClientSupplier, "test", client1, uuid, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(thread, "test", client1));
    List<TaskId> activeTaskList = Utils.mkList(task0, task3);
    Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
    activeTasks.put(task0, Utils.mkSet(t1p0));
    activeTasks.put(task3, Utils.mkSet(t2p3));
    standbyTasks.put(task1, Utils.mkSet(t1p0));
    standbyTasks.put(task2, Utils.mkSet(t2p0));
    AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, new HashMap<HostInfo, Set<TopicPartition>>());
    PartitionAssignor.Assignment assignment = new PartitionAssignor.Assignment(Utils.mkList(t1p0, t2p3), info.encode());
    partitionAssignor.onAssignment(assignment);
    assertEquals(activeTasks, partitionAssignor.activeTasks());
    assertEquals(standbyTasks, partitionAssignor.standbyTasks());
}