Examples with DeletionServiceDeleteTaskProto org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto used on opensource projects

Example 1 with DeletionServiceDeleteTaskProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto in project hadoop by apache.

the class NMLeveldbStateStoreService method loadDeletionServiceState.

@Override
public RecoveredDeletionServiceState loadDeletionServiceState() throws IOException {
    RecoveredDeletionServiceState state = new RecoveredDeletionServiceState();
    state.tasks = new ArrayList<DeletionServiceDeleteTaskProto>();
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(DELETION_TASK_KEY_PREFIX));
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.next();
            String key = asString(entry.getKey());
            if (!key.startsWith(DELETION_TASK_KEY_PREFIX)) {
                break;
            }
            state.tasks.add(DeletionServiceDeleteTaskProto.parseFrom(entry.getValue()));
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    return state;
}

Example 2 with DeletionServiceDeleteTaskProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto in project hadoop by apache.

the class DeletionService method recover.

private void recover(RecoveredDeletionServiceState state) throws IOException {
    List<DeletionServiceDeleteTaskProto> taskProtos = state.getTasks();
    Map<Integer, DeletionTaskRecoveryInfo> idToInfoMap = new HashMap<Integer, DeletionTaskRecoveryInfo>(taskProtos.size());
    Set<Integer> successorTasks = new HashSet<Integer>();
    for (DeletionServiceDeleteTaskProto proto : taskProtos) {
        DeletionTaskRecoveryInfo info = parseTaskProto(proto);
        idToInfoMap.put(info.task.taskId, info);
        nextTaskId.set(Math.max(nextTaskId.get(), info.task.taskId));
        successorTasks.addAll(info.successorTaskIds);
    }
    // restore the task dependencies and schedule the deletion tasks that
    // have no predecessors
    final long now = System.currentTimeMillis();
    for (DeletionTaskRecoveryInfo info : idToInfoMap.values()) {
        for (Integer successorId : info.successorTaskIds) {
            DeletionTaskRecoveryInfo successor = idToInfoMap.get(successorId);
            if (successor != null) {
                info.task.addFileDeletionTaskDependency(successor.task);
            } else {
                LOG.error("Unable to locate dependency task for deletion task " + info.task.taskId + " at " + info.task.getSubDir());
            }
        }
        if (!successorTasks.contains(info.task.taskId)) {
            long msecTilDeletion = info.deletionTimestamp - now;
            sched.schedule(info.task, msecTilDeletion, TimeUnit.MILLISECONDS);
        }
    }
}

Example 3 with DeletionServiceDeleteTaskProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testDeletionTaskStorage.

@Test
public void testDeletionTaskStorage() throws IOException {
    // test empty when no state
    RecoveredDeletionServiceState state = stateStore.loadDeletionServiceState();
    assertTrue(state.getTasks().isEmpty());
    // store a deletion task and verify recovered
    DeletionServiceDeleteTaskProto proto = DeletionServiceDeleteTaskProto.newBuilder().setId(7).setUser("someuser").setSubdir("some/subdir").addBasedirs("some/dir/path").addBasedirs("some/other/dir/path").setDeletionTime(123456L).addSuccessorIds(8).addSuccessorIds(9).build();
    stateStore.storeDeletionTask(proto.getId(), proto);
    restartStateStore();
    state = stateStore.loadDeletionServiceState();
    assertEquals(1, state.getTasks().size());
    assertEquals(proto, state.getTasks().get(0));
    // store another deletion task
    DeletionServiceDeleteTaskProto proto2 = DeletionServiceDeleteTaskProto.newBuilder().setId(8).setUser("user2").setSubdir("subdir2").setDeletionTime(789L).build();
    stateStore.storeDeletionTask(proto2.getId(), proto2);
    restartStateStore();
    state = stateStore.loadDeletionServiceState();
    assertEquals(2, state.getTasks().size());
    assertTrue(state.getTasks().contains(proto));
    assertTrue(state.getTasks().contains(proto2));
    // delete a task and verify gone after recovery
    stateStore.removeDeletionTask(proto2.getId());
    restartStateStore();
    state = stateStore.loadDeletionServiceState();
    assertEquals(1, state.getTasks().size());
    assertEquals(proto, state.getTasks().get(0));
    // delete the last task and verify none left
    stateStore.removeDeletionTask(proto.getId());
    restartStateStore();
    state = stateStore.loadDeletionServiceState();
    assertTrue(state.getTasks().isEmpty());
}