Examples with LogDeleterProto org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto used on opensource projects

Example 1 with LogDeleterProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto in project hadoop by apache.

the class NMLeveldbStateStoreService method loadLogDeleterState.

@Override
public RecoveredLogDeleterState loadLogDeleterState() throws IOException {
    RecoveredLogDeleterState state = new RecoveredLogDeleterState();
    state.logDeleterMap = new HashMap<ApplicationId, LogDeleterProto>();
    LeveldbIterator iter = null;
    try {
        iter = new LeveldbIterator(db);
        iter.seek(bytes(LOG_DELETER_KEY_PREFIX));
        final int logDeleterKeyPrefixLength = LOG_DELETER_KEY_PREFIX.length();
        while (iter.hasNext()) {
            Entry<byte[], byte[]> entry = iter.next();
            String fullKey = asString(entry.getKey());
            if (!fullKey.startsWith(LOG_DELETER_KEY_PREFIX)) {
                break;
            }
            String appIdStr = fullKey.substring(logDeleterKeyPrefixLength);
            ApplicationId appId = null;
            try {
                appId = ApplicationId.fromString(appIdStr);
            } catch (IllegalArgumentException e) {
                LOG.warn("Skipping unknown log deleter key " + fullKey);
                continue;
            }
            LogDeleterProto proto = LogDeleterProto.parseFrom(entry.getValue());
            state.logDeleterMap.put(appId, proto);
        }
    } catch (DBException e) {
        throw new IOException(e);
    } finally {
        if (iter != null) {
            iter.close();
        }
    }
    return state;
}

Example 2 with LogDeleterProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto in project hadoop by apache.

the class NonAggregatingLogHandler method recover.

private void recover() throws IOException {
    if (stateStore.canRecover()) {
        RecoveredLogDeleterState state = stateStore.loadLogDeleterState();
        long now = System.currentTimeMillis();
        for (Map.Entry<ApplicationId, LogDeleterProto> entry : state.getLogDeleterMap().entrySet()) {
            ApplicationId appId = entry.getKey();
            LogDeleterProto proto = entry.getValue();
            long deleteDelayMsec = proto.getDeletionTime() - now;
            if (LOG.isDebugEnabled()) {
                LOG.debug("Scheduling deletion of " + appId + " logs in " + deleteDelayMsec + " msec");
            }
            LogDeleterRunnable logDeleter = new LogDeleterRunnable(proto.getUser(), appId);
            try {
                sched.schedule(logDeleter, deleteDelayMsec, TimeUnit.MILLISECONDS);
            } catch (RejectedExecutionException e) {
                // Handling this event in local thread before starting threads
                // or after calling sched.shutdownNow().
                logDeleter.run();
            }
        }
    }
}

Example 3 with LogDeleterProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto in project hadoop by apache.

the class NonAggregatingLogHandler method handle.

@SuppressWarnings("unchecked")
@Override
public void handle(LogHandlerEvent event) {
    switch(event.getType()) {
        case APPLICATION_STARTED:
            LogHandlerAppStartedEvent appStartedEvent = (LogHandlerAppStartedEvent) event;
            this.appOwners.put(appStartedEvent.getApplicationId(), appStartedEvent.getUser());
            this.dispatcher.getEventHandler().handle(new ApplicationEvent(appStartedEvent.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_INITED));
            break;
        case CONTAINER_FINISHED:
            // Ignore
            break;
        case APPLICATION_FINISHED:
            LogHandlerAppFinishedEvent appFinishedEvent = (LogHandlerAppFinishedEvent) event;
            ApplicationId appId = appFinishedEvent.getApplicationId();
            // Schedule - so that logs are available on the UI till they're deleted.
            LOG.info("Scheduling Log Deletion for application: " + appId + ", with delay of " + this.deleteDelaySeconds + " seconds");
            String user = appOwners.remove(appId);
            if (user == null) {
                LOG.error("Unable to locate user for " + appId);
                // send LOG_HANDLING_FAILED out
                NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
                break;
            }
            LogDeleterRunnable logDeleter = new LogDeleterRunnable(user, appId);
            long deletionTimestamp = System.currentTimeMillis() + this.deleteDelaySeconds * 1000;
            LogDeleterProto deleterProto = LogDeleterProto.newBuilder().setUser(user).setDeletionTime(deletionTimestamp).build();
            try {
                stateStore.storeLogDeleter(appId, deleterProto);
            } catch (IOException e) {
                LOG.error("Unable to record log deleter state", e);
            }
            try {
                sched.schedule(logDeleter, this.deleteDelaySeconds, TimeUnit.SECONDS);
            } catch (RejectedExecutionException e) {
                // Handling this event in local thread before starting threads
                // or after calling sched.shutdownNow().
                logDeleter.run();
            }
            break;
        default:
            // Ignore
            ;
    }
}

Example 4 with LogDeleterProto

use of org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto in project hadoop by apache.

the class TestNMLeveldbStateStoreService method testLogDeleterStorage.

@Test
public void testLogDeleterStorage() throws IOException {
    // test empty when no state
    RecoveredLogDeleterState state = stateStore.loadLogDeleterState();
    assertTrue(state.getLogDeleterMap().isEmpty());
    // store log deleter state
    final ApplicationId appId1 = ApplicationId.newInstance(1, 1);
    LogDeleterProto proto1 = LogDeleterProto.newBuilder().setUser("user1").setDeletionTime(1234).build();
    stateStore.storeLogDeleter(appId1, proto1);
    // restart state store and verify recovered
    restartStateStore();
    state = stateStore.loadLogDeleterState();
    assertEquals(1, state.getLogDeleterMap().size());
    assertEquals(proto1, state.getLogDeleterMap().get(appId1));
    // store another log deleter
    final ApplicationId appId2 = ApplicationId.newInstance(2, 2);
    LogDeleterProto proto2 = LogDeleterProto.newBuilder().setUser("user2").setDeletionTime(5678).build();
    stateStore.storeLogDeleter(appId2, proto2);
    // restart state store and verify recovered
    restartStateStore();
    state = stateStore.loadLogDeleterState();
    assertEquals(2, state.getLogDeleterMap().size());
    assertEquals(proto1, state.getLogDeleterMap().get(appId1));
    assertEquals(proto2, state.getLogDeleterMap().get(appId2));
    // remove a deleter and verify removed after restart and recovery
    stateStore.removeLogDeleter(appId1);
    restartStateStore();
    state = stateStore.loadLogDeleterState();
    assertEquals(1, state.getLogDeleterMap().size());
    assertEquals(proto2, state.getLogDeleterMap().get(appId2));
    // remove last deleter and verify empty after restart and recovery
    stateStore.removeLogDeleter(appId2);
    restartStateStore();
    state = stateStore.loadLogDeleterState();
    assertTrue(state.getLogDeleterMap().isEmpty());
}