Examples with LifecycleStart io.druid.java.util.common.lifecycle.LifecycleStart used on opensource projects

Example 6 with LifecycleStart

use of io.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class RemoteTaskRunner method start.

@Override
@LifecycleStart
public void start() {
    try {
        if (started) {
            return;
        }
        final MutableInt waitingFor = new MutableInt(1);
        final Object waitingForMonitor = new Object();
        // Add listener for creation/deletion of workers
        workerPathCache.getListenable().addListener(new PathChildrenCacheListener() {

            @Override
            public void childEvent(CuratorFramework client, final PathChildrenCacheEvent event) throws Exception {
                final Worker worker;
                switch(event.getType()) {
                    case CHILD_ADDED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        synchronized (waitingForMonitor) {
                            waitingFor.increment();
                        }
                        Futures.addCallback(addWorker(worker), new FutureCallback<ZkWorker>() {

                            @Override
                            public void onSuccess(ZkWorker zkWorker) {
                                synchronized (waitingForMonitor) {
                                    waitingFor.decrement();
                                    waitingForMonitor.notifyAll();
                                }
                            }

                            @Override
                            public void onFailure(Throwable throwable) {
                                synchronized (waitingForMonitor) {
                                    waitingFor.decrement();
                                    waitingForMonitor.notifyAll();
                                }
                            }
                        });
                        break;
                    case CHILD_UPDATED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        updateWorker(worker);
                        break;
                    case CHILD_REMOVED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        removeWorker(worker);
                        break;
                    case INITIALIZED:
                        // Schedule cleanup for task status of the workers that might have disconnected while overlord was not running
                        List<String> workers;
                        try {
                            workers = cf.getChildren().forPath(indexerZkConfig.getStatusPath());
                        } catch (KeeperException.NoNodeException e) {
                            // statusPath doesn't exist yet; can occur if no middleManagers have started.
                            workers = ImmutableList.of();
                        }
                        for (String workerId : workers) {
                            final String workerAnnouncePath = JOINER.join(indexerZkConfig.getAnnouncementsPath(), workerId);
                            final String workerStatusPath = JOINER.join(indexerZkConfig.getStatusPath(), workerId);
                            if (!zkWorkers.containsKey(workerId) && cf.checkExists().forPath(workerAnnouncePath) == null) {
                                try {
                                    scheduleTasksCleanupForWorker(workerId, cf.getChildren().forPath(workerStatusPath));
                                } catch (Exception e) {
                                    log.warn(e, "Could not schedule cleanup for worker[%s] during startup (maybe someone removed the status znode[%s]?). Skipping.", workerId, workerStatusPath);
                                }
                            }
                        }
                        synchronized (waitingForMonitor) {
                            waitingFor.decrement();
                            waitingForMonitor.notifyAll();
                        }
                    default:
                        break;
                }
            }
        });
        workerPathCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
        synchronized (waitingForMonitor) {
            while (waitingFor.intValue() > 0) {
                waitingForMonitor.wait();
            }
        }
        scheduleBlackListedNodesCleanUp();
        resourceManagement.startManagement(this);
        started = true;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 7 with LifecycleStart

use of io.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class HdfsStorageAuthentication method authenticate.

/**
   * Dose authenticate against a secured hadoop cluster
   * In case of any bug fix make sure to fix the code in JobHelper#authenticate as well.
   */
@LifecycleStart
public void authenticate() {
    String principal = hdfsKerberosConfig.getPrincipal();
    String keytab = hdfsKerberosConfig.getKeytab();
    if (!Strings.isNullOrEmpty(principal) && !Strings.isNullOrEmpty(keytab)) {
        UserGroupInformation.setConfiguration(hadoopConf);
        if (UserGroupInformation.isSecurityEnabled()) {
            try {
                if (UserGroupInformation.getCurrentUser().hasKerberosCredentials() == false || !UserGroupInformation.getCurrentUser().getUserName().equals(principal)) {
                    log.info("Trying to authenticate user [%s] with keytab [%s]..", principal, keytab);
                    UserGroupInformation.loginUserFromKeytab(principal, keytab);
                }
            } catch (IOException e) {
                throw new ISE(e, "Failed to authenticate user principal [%s] with keytab [%s]", principal, keytab);
            }
        }
    }
}

Example 8 with LifecycleStart

use of io.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class ConfigManager method start.

@LifecycleStart
public void start() {
    synchronized (lock) {
        if (started) {
            return;
        }
        poller = new PollingCallable();
        ScheduledExecutors.scheduleWithFixedDelay(exec, new Duration(0), config.get().getPollDuration().toStandardDuration(), poller);
        started = true;
    }
}

Example 9 with LifecycleStart

use of io.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class RealtimeManager method start.

@LifecycleStart
public void start() throws IOException {
    for (final FireDepartment fireDepartment : fireDepartments) {
        final DataSchema schema = fireDepartment.getDataSchema();
        final FireChief chief = new FireChief(fireDepartment, conglomerate);
        Map<Integer, FireChief> partitionChiefs = chiefs.get(schema.getDataSource());
        if (partitionChiefs == null) {
            partitionChiefs = new HashMap<>();
            chiefs.put(schema.getDataSource(), partitionChiefs);
        }
        partitionChiefs.put(fireDepartment.getTuningConfig().getShardSpec().getPartitionNum(), chief);
        chief.setName(String.format("chief-%s[%s]", schema.getDataSource(), fireDepartment.getTuningConfig().getShardSpec().getPartitionNum()));
        chief.setDaemon(true);
        chief.start();
    }
}

Example 10 with LifecycleStart

use of io.druid.java.util.common.lifecycle.LifecycleStart in project druid by druid-io.

the class ExecutorLifecycle method start.

@LifecycleStart
public void start() throws InterruptedException {
    final File taskFile = Preconditions.checkNotNull(taskExecutorConfig.getTaskFile(), "taskFile");
    final File statusFile = Preconditions.checkNotNull(taskExecutorConfig.getStatusFile(), "statusFile");
    final InputStream parentStream = Preconditions.checkNotNull(taskExecutorConfig.getParentStream(), "parentStream");
    try {
        task = jsonMapper.readValue(taskFile, Task.class);
        log.info("Running with task: %s", jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(task));
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
    // Avoid running the same task twice on the same machine by locking the task base directory.
    final File taskLockFile = taskConfig.getTaskLockFile(task.getId());
    try {
        synchronized (this) {
            if (taskLockChannel == null && taskLockFileLock == null) {
                taskLockChannel = FileChannel.open(taskLockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
                log.info("Attempting to lock file[%s].", taskLockFile);
                final long startLocking = System.currentTimeMillis();
                final long timeout = new DateTime(startLocking).plus(taskConfig.getDirectoryLockTimeout()).getMillis();
                while (taskLockFileLock == null && System.currentTimeMillis() < timeout) {
                    taskLockFileLock = taskLockChannel.tryLock();
                    if (taskLockFileLock == null) {
                        Thread.sleep(100);
                    }
                }
                if (taskLockFileLock == null) {
                    throw new ISE("Could not acquire lock file[%s] within %,dms.", taskLockFile, timeout - startLocking);
                } else {
                    log.info("Acquired lock file[%s] in %,dms.", taskLockFile, System.currentTimeMillis() - startLocking);
                }
            } else {
                throw new ISE("Already started!");
            }
        }
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
    // Spawn monitor thread to keep a watch on parent's stdin
    // If stdin reaches eof, the parent is gone, and we should shut down
    parentMonitorExec.submit(new Runnable() {

        @Override
        public void run() {
            try {
                while (parentStream.read() != -1) {
                // Toss the byte
                }
            } catch (Exception e) {
                log.error(e, "Failed to read from stdin");
            }
            // Kind of gross, but best way to kill the JVM as far as I know
            log.info("Triggering JVM shutdown.");
            System.exit(2);
        }
    });
    // Won't hurt in remote mode, and is required for setting up locks in local mode:
    try {
        if (!task.isReady(taskActionClientFactory.create(task))) {
            throw new ISE("Task is not ready to run yet!", task.getId());
        }
    } catch (Exception e) {
        throw new ISE(e, "Failed to run isReady", task.getId());
    }
    statusFuture = Futures.transform(taskRunner.run(task), new Function<TaskStatus, TaskStatus>() {

        @Override
        public TaskStatus apply(TaskStatus taskStatus) {
            try {
                log.info("Task completed with status: %s", jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(taskStatus));
                final File statusFileParent = statusFile.getParentFile();
                if (statusFileParent != null) {
                    statusFileParent.mkdirs();
                }
                jsonMapper.writeValue(statusFile, taskStatus);
                return taskStatus;
            } catch (Exception e) {
                throw Throwables.propagate(e);
            }
        }
    });
}