Examples with TaskDirectory org.apache.kafka.streams.processor.internals.StateDirectory.TaskDirectory used on opensource projects

Example 1 with TaskDirectory

use of org.apache.kafka.streams.processor.internals.StateDirectory.TaskDirectory in project kafka by apache.

the class TaskManagerTest method makeTaskFolders.

private void makeTaskFolders(final String... names) throws Exception {
    final ArrayList<TaskDirectory> taskFolders = new ArrayList<>(names.length);
    for (int i = 0; i < names.length; ++i) {
        taskFolders.add(new TaskDirectory(testFolder.newFolder(names[i]), null));
    }
    expect(stateDirectory.listNonEmptyTaskDirectories()).andReturn(taskFolders).once();
}

Example 2 with TaskDirectory

use of org.apache.kafka.streams.processor.internals.StateDirectory.TaskDirectory in project kafka by apache.

the class StateDirectoryTest method shouldOnlyListNonEmptyTaskDirectories.

@Test
public void shouldOnlyListNonEmptyTaskDirectories() throws IOException {
    TestUtils.tempDirectory(stateDir.toPath(), "foo");
    final TaskDirectory taskDir1 = new TaskDirectory(directory.getOrCreateDirectoryForTask(new TaskId(0, 0)), null);
    final TaskDirectory taskDir2 = new TaskDirectory(directory.getOrCreateDirectoryForTask(new TaskId(0, 1)), null);
    final File storeDir = new File(taskDir1.file(), "store");
    assertTrue(storeDir.mkdir());
    assertThat(mkSet(taskDir1, taskDir2), equalTo(new HashSet<>(directory.listAllTaskDirectories())));
    assertThat(singletonList(taskDir1), equalTo(directory.listNonEmptyTaskDirectories()));
    Utils.delete(taskDir1.file());
    assertThat(singleton(taskDir2), equalTo(new HashSet<>(directory.listAllTaskDirectories())));
    assertThat(emptyList(), equalTo(directory.listNonEmptyTaskDirectories()));
}

Example 3 with TaskDirectory

use of org.apache.kafka.streams.processor.internals.StateDirectory.TaskDirectory in project kafka by apache.

the class TaskManager method tryToLockAllNonEmptyTaskDirectories.

/**
 * Makes a weak attempt to lock all non-empty task directories in the state dir. We are responsible for computing and
 * reporting the offset sum for any unassigned tasks we obtain the lock for in the upcoming rebalance. Tasks
 * that we locked but didn't own will be released at the end of the rebalance (unless of course we were
 * assigned the task as a result of the rebalance). This method should be idempotent.
 */
private void tryToLockAllNonEmptyTaskDirectories() {
    // Always clear the set at the beginning as we're always dealing with the
    // current set of actually-locked tasks.
    lockedTaskDirectories.clear();
    for (final TaskDirectory taskDir : stateDirectory.listNonEmptyTaskDirectories()) {
        final File dir = taskDir.file();
        final String namedTopology = taskDir.namedTopology();
        try {
            final TaskId id = parseTaskDirectoryName(dir.getName(), namedTopology);
            if (stateDirectory.lock(id)) {
                lockedTaskDirectories.add(id);
                if (!tasks.owned(id)) {
                    log.debug("Temporarily locked unassigned task {} for the upcoming rebalance", id);
                }
            }
        } catch (final TaskIdFormatException e) {
        // ignore any unknown files that sit in the same directory
        }
    }
}

Example 4 with TaskDirectory

use of org.apache.kafka.streams.processor.internals.StateDirectory.TaskDirectory in project kafka by apache.

the class StateDirectoryTest method shouldOnlyListNonEmptyTaskDirectoriesInNamedTopologies.

@Test
public void shouldOnlyListNonEmptyTaskDirectoriesInNamedTopologies() throws IOException {
    initializeStateDirectory(true, true);
    TestUtils.tempDirectory(appDir.toPath(), "foo");
    final TaskDirectory taskDir1 = new TaskDirectory(directory.getOrCreateDirectoryForTask(new TaskId(0, 0, "topology1")), "topology1");
    final TaskDirectory taskDir2 = new TaskDirectory(directory.getOrCreateDirectoryForTask(new TaskId(0, 1, "topology1")), "topology1");
    final TaskDirectory taskDir3 = new TaskDirectory(directory.getOrCreateDirectoryForTask(new TaskId(0, 0, "topology2")), "topology2");
    final File storeDir = new File(taskDir1.file(), "store");
    assertTrue(storeDir.mkdir());
    assertThat(new HashSet<>(directory.listAllTaskDirectories()), equalTo(mkSet(taskDir1, taskDir2, taskDir3)));
    assertThat(directory.listNonEmptyTaskDirectories(), equalTo(singletonList(taskDir1)));
    Utils.delete(taskDir1.file());
    assertThat(new HashSet<>(directory.listAllTaskDirectories()), equalTo(mkSet(taskDir2, taskDir3)));
    assertThat(directory.listNonEmptyTaskDirectories(), equalTo(emptyList()));
}

Example 5 with TaskDirectory

use of org.apache.kafka.streams.processor.internals.StateDirectory.TaskDirectory in project kafka by apache.

the class StateDirectoryTest method shouldCleanUpTaskStateDirectoriesThatAreNotCurrentlyLocked.

@Test
public void shouldCleanUpTaskStateDirectoriesThatAreNotCurrentlyLocked() {
    final TaskId task0 = new TaskId(0, 0);
    final TaskId task1 = new TaskId(1, 0);
    final TaskId task2 = new TaskId(2, 0);
    try {
        assertTrue(new File(directory.getOrCreateDirectoryForTask(task0), "store").mkdir());
        assertTrue(new File(directory.getOrCreateDirectoryForTask(task1), "store").mkdir());
        assertTrue(new File(directory.getOrCreateDirectoryForTask(task2), "store").mkdir());
        directory.lock(task0);
        directory.lock(task1);
        final TaskDirectory dir0 = new TaskDirectory(new File(appDir, toTaskDirString(task0)), null);
        final TaskDirectory dir1 = new TaskDirectory(new File(appDir, toTaskDirString(task1)), null);
        final TaskDirectory dir2 = new TaskDirectory(new File(appDir, toTaskDirString(task2)), null);
        List<TaskDirectory> files = directory.listAllTaskDirectories();
        assertEquals(mkSet(dir0, dir1, dir2), new HashSet<>(files));
        files = directory.listNonEmptyTaskDirectories();
        assertEquals(mkSet(dir0, dir1, dir2), new HashSet<>(files));
        time.sleep(5000);
        directory.cleanRemovedTasks(0);
        files = directory.listAllTaskDirectories();
        assertEquals(mkSet(dir0, dir1), new HashSet<>(files));
        files = directory.listNonEmptyTaskDirectories();
        assertEquals(mkSet(dir0, dir1), new HashSet<>(files));
    } finally {
        directory.unlock(task0);
        directory.unlock(task1);
    }
}