Examples with LifecycleStart org.apache.druid.java.util.common.lifecycle.LifecycleStart used on opensource projects

Example 11 with LifecycleStart

use of org.apache.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class ExecutorLifecycle method start.

@LifecycleStart
public void start() throws InterruptedException {
    final File taskFile = Preconditions.checkNotNull(taskExecutorConfig.getTaskFile(), "taskFile");
    final File statusFile = Preconditions.checkNotNull(taskExecutorConfig.getStatusFile(), "statusFile");
    final InputStream parentStream = Preconditions.checkNotNull(taskExecutorConfig.getParentStream(), "parentStream");
    try {
        task = jsonMapper.readValue(taskFile, Task.class);
        log.info("Running with task: %s", jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(task));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    // Avoid running the same task twice on the same machine by locking the task base directory.
    final File taskLockFile = taskConfig.getTaskLockFile(task.getId());
    try {
        synchronized (this) {
            if (taskLockChannel == null && taskLockFileLock == null) {
                taskLockChannel = FileChannel.open(taskLockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
                log.info("Attempting to lock file[%s].", taskLockFile);
                final long startLocking = System.currentTimeMillis();
                final long timeout = DateTimes.utc(startLocking).plus(taskConfig.getDirectoryLockTimeout()).getMillis();
                while (taskLockFileLock == null && System.currentTimeMillis() < timeout) {
                    taskLockFileLock = taskLockChannel.tryLock();
                    if (taskLockFileLock == null) {
                        Thread.sleep(100);
                    }
                }
                if (taskLockFileLock == null) {
                    throw new ISE("Could not acquire lock file[%s] within %,dms.", taskLockFile, timeout - startLocking);
                } else {
                    log.info("Acquired lock file[%s] in %,dms.", taskLockFile, System.currentTimeMillis() - startLocking);
                }
            } else {
                throw new ISE("Already started!");
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    if (taskExecutorConfig.isParentStreamDefined()) {
        // Spawn monitor thread to keep a watch on parent's stdin
        // If stdin reaches eof, the parent is gone, and we should shut down
        parentMonitorExec.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    while (parentStream.read() != -1) {
                    // Toss the byte
                    }
                } catch (Exception e) {
                    log.error(e, "Failed to read from stdin");
                }
                // Kind of gross, but best way to kill the JVM as far as I know
                log.info("Triggering JVM shutdown.");
                System.exit(2);
            }
        });
    }
    // Won't hurt in remote mode, and is required for setting up locks in local mode:
    try {
        if (!task.isReady(taskActionClientFactory.create(task))) {
            throw new ISE("Task[%s] is not ready to run yet!", task.getId());
        }
    } catch (Exception e) {
        throw new ISE(e, "Failed to run task[%s] isReady", task.getId());
    }
    statusFuture = Futures.transform(taskRunner.run(task), new Function<TaskStatus, TaskStatus>() {

        @Override
        public TaskStatus apply(TaskStatus taskStatus) {
            try {
                log.info("Task completed with status: %s", jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(taskStatus));
                final File statusFileParent = statusFile.getParentFile();
                if (statusFileParent != null) {
                    FileUtils.mkdirp(statusFileParent);
                }
                jsonMapper.writeValue(statusFile, taskStatus);
                return taskStatus;
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    });
}

Example 12 with LifecycleStart

use of org.apache.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class CoordinatorBasicAuthorizerMetadataStorageUpdater method start.

@LifecycleStart
public void start() {
    if (!lifecycleLock.canStart()) {
        throw new ISE("can't start.");
    }
    if (authorizerMapper == null || authorizerMapper.getAuthorizerMap() == null) {
        return;
    }
    try {
        LOG.info("Starting CoordinatorBasicAuthorizerMetadataStorageUpdater");
        BasicAuthUtils.maybeInitialize(() -> {
            for (Map.Entry<String, Authorizer> entry : authorizerMapper.getAuthorizerMap().entrySet()) {
                Authorizer authorizer = entry.getValue();
                if (authorizer instanceof BasicRoleBasedAuthorizer) {
                    BasicRoleBasedAuthorizer basicRoleBasedAuthorizer = (BasicRoleBasedAuthorizer) authorizer;
                    BasicAuthDBConfig dbConfig = basicRoleBasedAuthorizer.getDbConfig();
                    String authorizerName = entry.getKey();
                    authorizerNames.add(authorizerName);
                    byte[] userMapBytes = getCurrentUserMapBytes(authorizerName);
                    Map<String, BasicAuthorizerUser> userMap = BasicAuthUtils.deserializeAuthorizerUserMap(objectMapper, userMapBytes);
                    cachedUserMaps.put(authorizerName, new BasicAuthorizerUserMapBundle(userMap, userMapBytes));
                    byte[] groupMappingMapBytes = getCurrentGroupMappingMapBytes(authorizerName);
                    Map<String, BasicAuthorizerGroupMapping> groupMappingMap = BasicAuthUtils.deserializeAuthorizerGroupMappingMap(objectMapper, groupMappingMapBytes);
                    cachedGroupMappingMaps.put(authorizerName, new BasicAuthorizerGroupMappingMapBundle(groupMappingMap, groupMappingMapBytes));
                    byte[] roleMapBytes = getCurrentRoleMapBytes(authorizerName);
                    Map<String, BasicAuthorizerRole> roleMap = BasicAuthUtils.deserializeAuthorizerRoleMap(objectMapper, roleMapBytes);
                    cachedRoleMaps.put(authorizerName, new BasicAuthorizerRoleMapBundle(roleMap, roleMapBytes));
                    initSuperUsersAndGroupMapping(authorizerName, userMap, roleMap, groupMappingMap, dbConfig.getInitialAdminUser(), dbConfig.getInitialAdminRole(), dbConfig.getInitialAdminGroupMapping());
                }
            }
            return true;
        });
        ScheduledExecutors.scheduleWithFixedDelay(exec, new Duration(commonCacheConfig.getPollingPeriod()), new Duration(commonCacheConfig.getPollingPeriod()), () -> {
            if (stopped) {
                return ScheduledExecutors.Signal.STOP;
            }
            try {
                LOG.debug("Scheduled db poll is running");
                for (String authorizerName : authorizerNames) {
                    byte[] userMapBytes = getCurrentUserMapBytes(authorizerName);
                    Map<String, BasicAuthorizerUser> userMap = BasicAuthUtils.deserializeAuthorizerUserMap(objectMapper, userMapBytes);
                    if (userMapBytes != null) {
                        synchronized (cachedUserMaps) {
                            cachedUserMaps.put(authorizerName, new BasicAuthorizerUserMapBundle(userMap, userMapBytes));
                        }
                    }
                    byte[] groupMappingMapBytes = getCurrentGroupMappingMapBytes(authorizerName);
                    Map<String, BasicAuthorizerGroupMapping> groupMappingMap = BasicAuthUtils.deserializeAuthorizerGroupMappingMap(objectMapper, groupMappingMapBytes);
                    if (groupMappingMapBytes != null) {
                        synchronized (cachedGroupMappingMaps) {
                            cachedGroupMappingMaps.put(authorizerName, new BasicAuthorizerGroupMappingMapBundle(groupMappingMap, groupMappingMapBytes));
                        }
                    }
                    byte[] roleMapBytes = getCurrentRoleMapBytes(authorizerName);
                    Map<String, BasicAuthorizerRole> roleMap = BasicAuthUtils.deserializeAuthorizerRoleMap(objectMapper, roleMapBytes);
                    if (roleMapBytes != null) {
                        synchronized (cachedRoleMaps) {
                            cachedRoleMaps.put(authorizerName, new BasicAuthorizerRoleMapBundle(roleMap, roleMapBytes));
                        }
                    }
                }
                LOG.debug("Scheduled db poll is done");
            } catch (Throwable t) {
                LOG.makeAlert(t, "Error occured while polling for cachedUserMaps, cachedGroupMappingMaps, cachedRoleMaps.").emit();
            }
            return ScheduledExecutors.Signal.REPEAT;
        });
        lifecycleLock.started();
    } finally {
        lifecycleLock.exitStart();
    }
}

Example 13 with LifecycleStart

use of org.apache.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class CoordinatorPollingBasicAuthenticatorCacheManager method start.

@LifecycleStart
public void start() {
    if (!lifecycleLock.canStart()) {
        throw new ISE("can't start.");
    }
    LOG.info("Starting CoordinatorPollingBasicAuthenticatorCacheManager.");
    try {
        initUserMaps();
        ScheduledExecutors.scheduleWithFixedDelay(exec, new Duration(commonCacheConfig.getPollingPeriod()), new Duration(commonCacheConfig.getPollingPeriod()), () -> {
            try {
                long randomDelay = ThreadLocalRandom.current().nextLong(0, commonCacheConfig.getMaxRandomDelay());
                LOG.debug("Inserting cachedUserMaps random polling delay of [%s] ms", randomDelay);
                Thread.sleep(randomDelay);
                LOG.debug("Scheduled user cache poll is running");
                for (String authenticatorPrefix : authenticatorPrefixes) {
                    Map<String, BasicAuthenticatorUser> userMap = fetchUserMapFromCoordinator(authenticatorPrefix, false);
                    if (userMap != null) {
                        cachedUserMaps.put(authenticatorPrefix, userMap);
                    }
                }
                LOG.debug("Scheduled user cache poll is done");
            } catch (Throwable t) {
                LOG.makeAlert(t, "Error occured while polling for cachedUserMaps.").emit();
            }
        });
        lifecycleLock.started();
        LOG.info("Started CoordinatorPollingBasicAuthenticatorCacheManager.");
    } finally {
        lifecycleLock.exitStart();
    }
}

Example 14 with LifecycleStart

use of org.apache.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class CoordinatorBasicAuthenticatorMetadataStorageUpdater method start.

@LifecycleStart
public void start() {
    if (!lifecycleLock.canStart()) {
        throw new ISE("can't start.");
    }
    if (authenticatorMapper == null || authenticatorMapper.getAuthenticatorMap() == null) {
        return;
    }
    try {
        LOG.info("Starting CoordinatorBasicAuthenticatorMetadataStorageUpdater.");
        BasicAuthUtils.maybeInitialize(() -> {
            for (Map.Entry<String, Authenticator> entry : authenticatorMapper.getAuthenticatorMap().entrySet()) {
                Authenticator authenticator = entry.getValue();
                if (authenticator instanceof BasicHTTPAuthenticator) {
                    String authenticatorName = entry.getKey();
                    authenticatorPrefixes.add(authenticatorName);
                    BasicHTTPAuthenticator basicHTTPAuthenticator = (BasicHTTPAuthenticator) authenticator;
                    BasicAuthDBConfig dbConfig = basicHTTPAuthenticator.getDbConfig();
                    byte[] userMapBytes = getCurrentUserMapBytes(authenticatorName);
                    Map<String, BasicAuthenticatorUser> userMap = BasicAuthUtils.deserializeAuthenticatorUserMap(objectMapper, userMapBytes);
                    cachedUserMaps.put(authenticatorName, new BasicAuthenticatorUserMapBundle(userMap, userMapBytes));
                    if (dbConfig.getInitialAdminPassword() != null && !userMap.containsKey(BasicAuthUtils.ADMIN_NAME)) {
                        createUserInternal(authenticatorName, BasicAuthUtils.ADMIN_NAME);
                        setUserCredentialsInternal(authenticatorName, BasicAuthUtils.ADMIN_NAME, new BasicAuthenticatorCredentialUpdate(dbConfig.getInitialAdminPassword().getPassword(), BasicAuthUtils.DEFAULT_KEY_ITERATIONS));
                    }
                    if (dbConfig.getInitialInternalClientPassword() != null && !userMap.containsKey(BasicAuthUtils.INTERNAL_USER_NAME)) {
                        createUserInternal(authenticatorName, BasicAuthUtils.INTERNAL_USER_NAME);
                        setUserCredentialsInternal(authenticatorName, BasicAuthUtils.INTERNAL_USER_NAME, new BasicAuthenticatorCredentialUpdate(dbConfig.getInitialInternalClientPassword().getPassword(), BasicAuthUtils.DEFAULT_KEY_ITERATIONS));
                    }
                }
            }
            return true;
        });
        ScheduledExecutors.scheduleWithFixedDelay(exec, new Duration(commonCacheConfig.getPollingPeriod()), new Duration(commonCacheConfig.getPollingPeriod()), new Callable<ScheduledExecutors.Signal>() {

            @Override
            public ScheduledExecutors.Signal call() {
                if (stopped) {
                    return ScheduledExecutors.Signal.STOP;
                }
                try {
                    LOG.debug("Scheduled db userMap poll is running");
                    for (String authenticatorPrefix : authenticatorPrefixes) {
                        byte[] userMapBytes = getCurrentUserMapBytes(authenticatorPrefix);
                        Map<String, BasicAuthenticatorUser> userMap = BasicAuthUtils.deserializeAuthenticatorUserMap(objectMapper, userMapBytes);
                        if (userMapBytes != null) {
                            cachedUserMaps.put(authenticatorPrefix, new BasicAuthenticatorUserMapBundle(userMap, userMapBytes));
                        }
                    }
                    LOG.debug("Scheduled db userMap poll is done");
                } catch (Throwable t) {
                    LOG.makeAlert(t, "Error occured while polling for cachedUserMaps.").emit();
                }
                return ScheduledExecutors.Signal.REPEAT;
            }
        });
        lifecycleLock.started();
    } finally {
        lifecycleLock.exitStart();
    }
}

Example 15 with LifecycleStart

use of org.apache.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class CoordinatorRuleManager method start.

@LifecycleStart
public void start() {
    synchronized (lock) {
        if (started) {
            return;
        }
        this.exec = Execs.scheduledSingleThreaded("CoordinatorRuleManager-Exec--%d");
        ScheduledExecutors.scheduleWithFixedDelay(exec, new Duration(0), config.get().getPollPeriod().toStandardDuration(), this::poll);
        started = true;
    }
}