Examples with TaskShutdownDameon com.alibaba.jstorm.task.TaskShutdownDameon used on opensource projects

Example 1 with TaskShutdownDameon

use of com.alibaba.jstorm.task.TaskShutdownDameon in project jstorm by alibaba.

the class RefreshActive method run.

@Override
public void run() {
    try {
        StatusType newTopologyStatus;
        // /ZK-DIR/topology
        StormBase base = zkCluster.storm_base(topologyId, this);
        if (base == null) {
            // normally the topology has been removed
            LOG.warn("Failed to get StormBase from ZK of " + topologyId);
            newTopologyStatus = StatusType.killed;
        } else {
            newTopologyStatus = base.getStatus().getStatusType();
        }
        // Process the topology status change
        StatusType oldTopologyStatus = workerData.getTopologyStatus();
        List<TaskShutdownDameon> tasks = workerData.getShutdownTasks();
        if (tasks == null) {
            LOG.info("Tasks aren't ready or are beginning to shutdown");
            return;
        }
        // If all connections were done, start to update topology status. Otherwise, just return.
        if (oldTopologyStatus == null) {
            if (!workerData.getWorkeInitConnectionStatus().get()) {
                return;
            }
        }
        if (oldTopologyStatus == null || !newTopologyStatus.equals(oldTopologyStatus)) {
            LOG.info("Old TopologyStatus:" + oldTopologyStatus + ", new TopologyStatus:" + newTopologyStatus);
            if (newTopologyStatus.equals(StatusType.active)) {
                for (TaskShutdownDameon task : tasks) {
                    if (task.getTask().getTaskStatus().isInit()) {
                        task.getTask().getTaskStatus().setStatus(TaskStatus.RUN);
                    } else {
                        task.active();
                    }
                }
            } else if (oldTopologyStatus == null || !oldTopologyStatus.equals(StatusType.inactive)) {
                for (TaskShutdownDameon task : tasks) {
                    if (task.getTask().getTaskStatus().isInit()) {
                        task.getTask().getTaskStatus().setStatus(TaskStatus.PAUSE);
                    } else {
                        task.deactive();
                    }
                }
            }
            workerData.setTopologyStatus(newTopologyStatus);
            if (base != null) {
                boolean newMonitorEnable = base.isEnableMonitor();
                boolean oldMonitorEnable = monitorEnable.get();
                if (newMonitorEnable != oldMonitorEnable) {
                    LOG.info("Change MonitorEnable from " + oldMonitorEnable + " to " + newMonitorEnable);
                    monitorEnable.set(newMonitorEnable);
                }
            }
        }
    } catch (Exception e) {
        LOG.error("Failed to get topology from ZK ", e);
    }
}

Example 2 with TaskShutdownDameon

use of com.alibaba.jstorm.task.TaskShutdownDameon in project jstorm by alibaba.

the class RefreshConnections method shutdownTasks.

private void shutdownTasks(Set<Integer> tasks) {
    if (tasks == null)
        return;
    List<TaskShutdownDameon> shutdowns = workerData.getShutdownDaemonbyTaskIds(tasks);
    List<Future> futures = new ArrayList<>();
    for (ShutdownableDameon task : shutdowns) {
        Future<?> future = workerData.getFlusherPool().submit(task);
        futures.add(future);
    }
    // To be assure all tasks are closed rightly
    for (Future future : futures) {
        if (future != null) {
            try {
                future.get();
            } catch (Exception ex) {
                LOG.error("Failed to shutdown task {}", ex);
            }
        }
    }
}

Example 3 with TaskShutdownDameon

use of com.alibaba.jstorm.task.TaskShutdownDameon in project jstorm by alibaba.

the class RefreshConnections method run.

@Override
public void run() {
    try {
        //
        synchronized (this) {
            Integer recordedVersion = zkCluster.assignment_version(topologyId, this);
            boolean isUpdateAssignment = !(recordedVersion != null && recordedVersion.equals(assignmentVersion));
            boolean isUpdateSupervisorTimeStamp = false;
            Long localAssignmentTS = null;
            try {
                localAssignmentTS = StormConfig.read_supervisor_topology_timestamp(conf, topologyId);
                isUpdateSupervisorTimeStamp = localAssignmentTS > workerData.getAssignmentTs();
            } catch (FileNotFoundException e) {
                LOG.warn("Failed to read supervisor topology timeStamp for " + topologyId + " port=" + workerData.getPort(), e);
            }
            if (isUpdateAssignment || isUpdateSupervisorTimeStamp) {
                LOG.info("update worker data due to changed assignment!!!");
                Assignment assignment = zkCluster.assignment_info(topologyId, this);
                if (assignment == null) {
                    String errMsg = "Failed to get Assignment of " + topologyId;
                    LOG.error(errMsg);
                    // throw new RuntimeException(errMsg);
                    return;
                }
                // task map should be updated accordingly.
                if (isUpdateSupervisorTimeStamp) {
                    try {
                        if (assignment.getAssignmentType() == AssignmentType.UpdateTopology) {
                            LOG.info("Get config reload request for " + topologyId);
                            // If config was updated, notify all tasks
                            List<TaskShutdownDameon> taskShutdowns = workerData.getShutdownTasks();
                            Map newConf = StormConfig.read_supervisor_topology_conf(conf, topologyId);
                            workerData.getStormConf().putAll(newConf);
                            for (TaskShutdownDameon taskSD : taskShutdowns) {
                                taskSD.update(newConf);
                            }
                            // disable/enable metrics on the fly
                            workerData.getUpdateListener().update(newConf);
                            workerData.setAssignmentType(AssignmentType.UpdateTopology);
                        } else {
                            Set<Integer> addedTasks = getAddedTasks(assignment);
                            Set<Integer> removedTasks = getRemovedTasks(assignment);
                            Set<Integer> updatedTasks = getUpdatedTasks(assignment);
                            workerData.updateWorkerData(assignment);
                            workerData.updateKryoSerializer();
                            shutdownTasks(removedTasks);
                            createTasks(addedTasks);
                            updateTasks(updatedTasks);
                            Set<Integer> tmpOutboundTasks = Worker.worker_output_tasks(workerData);
                            if (outboundTasks.equals(tmpOutboundTasks) == false) {
                                for (int taskId : tmpOutboundTasks) {
                                    if (outboundTasks.contains(taskId) == false)
                                        workerData.addOutboundTaskStatusIfAbsent(taskId);
                                }
                                for (int taskId : workerData.getOutboundTaskStatus().keySet()) {
                                    if (tmpOutboundTasks.contains(taskId) == false) {
                                        workerData.removeOutboundTaskStatus(taskId);
                                    }
                                }
                                workerData.setOutboundTasks(tmpOutboundTasks);
                                outboundTasks = tmpOutboundTasks;
                            }
                            workerData.setAssignmentType(AssignmentType.Assign);
                        }
                        // going to update the related data.
                        if (localAssignmentTS != null)
                            workerData.setAssignmentTs(localAssignmentTS);
                    } catch (Exception e) {
                        LOG.warn("Failed to update worker data", e);
                    }
                }
                Set<ResourceWorkerSlot> workers = assignment.getWorkers();
                if (workers == null) {
                    String errMsg = "Failed to get taskToResource of " + topologyId;
                    LOG.error(errMsg);
                    return;
                }
                workerData.updateWorkerToResource(workers);
                Map<Integer, WorkerSlot> taskNodeportTmp = new HashMap<Integer, WorkerSlot>();
                Map<String, String> node = assignment.getNodeHost();
                // only reserve outboundTasks
                Set<WorkerSlot> need_connections = new HashSet<WorkerSlot>();
                Set<Integer> localTasks = new HashSet<Integer>();
                Set<Integer> localNodeTasks = new HashSet<Integer>();
                if (outboundTasks != null) {
                    for (ResourceWorkerSlot worker : workers) {
                        if (supervisorId.equals(worker.getNodeId()))
                            localNodeTasks.addAll(worker.getTasks());
                        if (supervisorId.equals(worker.getNodeId()) && worker.getPort() == workerData.getPort())
                            localTasks.addAll(worker.getTasks());
                        for (Integer id : worker.getTasks()) {
                            taskNodeportTmp.put(id, worker);
                            if (outboundTasks.contains(id)) {
                                need_connections.add(worker);
                            }
                        }
                    }
                }
                taskNodeport.putAll(taskNodeportTmp);
                //workerData.setLocalTasks(localTasks);
                workerData.setLocalNodeTasks(localNodeTasks);
                // get which connection need to be remove or add
                Set<WorkerSlot> current_connections = nodeportSocket.keySet();
                Set<WorkerSlot> new_connections = new HashSet<WorkerSlot>();
                Set<WorkerSlot> remove_connections = new HashSet<WorkerSlot>();
                for (WorkerSlot node_port : need_connections) {
                    if (!current_connections.contains(node_port)) {
                        new_connections.add(node_port);
                    }
                }
                for (WorkerSlot node_port : current_connections) {
                    if (!need_connections.contains(node_port)) {
                        remove_connections.add(node_port);
                    }
                }
                // create new connection
                for (WorkerSlot nodePort : new_connections) {
                    String host = node.get(nodePort.getNodeId());
                    int port = nodePort.getPort();
                    IConnection conn = context.connect(topologyId, host, port);
                    nodeportSocket.put(nodePort, conn);
                    LOG.info("Add connection to " + nodePort);
                }
                // close useless connection
                for (WorkerSlot node_port : remove_connections) {
                    LOG.info("Remove connection to " + node_port);
                    nodeportSocket.remove(node_port).close();
                }
            }
            // check the status of connections to all outbound tasks
            boolean allConnectionReady = true;
            for (Integer taskId : outboundTasks) {
                boolean isConnected = isOutTaskConnected(taskId);
                if (!isConnected)
                    allConnectionReady = isConnected;
                workerData.updateOutboundTaskStatus(taskId, isConnected);
            }
            if (allConnectionReady) {
                workerData.getWorkeInitConnectionStatus().getAndSet(allConnectionReady);
            }
            if (recordedVersion != null)
                assignmentVersion = recordedVersion;
        }
    } catch (Exception e) {
        LOG.error("Failed to refresh worker Connection", e);
        throw new RuntimeException(e);
    }
// finally {
// endpoint_socket_lock.writeLock().unlock();
// }
}

Example 4 with TaskShutdownDameon

use of com.alibaba.jstorm.task.TaskShutdownDameon in project jstorm by alibaba.

the class Worker method execute.

public WorkerShutdown execute() throws Exception {
    List<AsyncLoopThread> threads = new ArrayList<>();
    // create recv connection, reduce the count of netty client reconnect
    AsyncLoopThread controlRvthread = startDispatchThread();
    threads.add(controlRvthread);
    // create client before create task
    // so create client connection before create task
    // refresh connection
    RefreshConnections refreshConn = makeRefreshConnections();
    AsyncLoopThread refreshconn = new AsyncLoopThread(refreshConn, false, Thread.MIN_PRIORITY, true);
    threads.add(refreshconn);
    // refresh ZK active status
    RefreshActive refreshZkActive = new RefreshActive(workerData);
    AsyncLoopThread refreshzk = new AsyncLoopThread(refreshZkActive, false, Thread.MIN_PRIORITY, true);
    threads.add(refreshzk);
    //create send control message thread
    DrainerCtrlRunable drainerCtrlRunable;
    drainerCtrlRunable = new DrainerCtrlRunable(workerData, MetricDef.SEND_THREAD);
    AsyncLoopThread controlSendThread = new AsyncLoopThread(drainerCtrlRunable, false, Thread.MAX_PRIORITY, true);
    threads.add(controlSendThread);
    // Sync heartbeat to Apsara Container
    AsyncLoopThread syncContainerHbThread = SyncContainerHb.mkWorkerInstance(workerData.getStormConf());
    if (syncContainerHbThread != null) {
        threads.add(syncContainerHbThread);
    }
    JStormMetricsReporter metricReporter = new JStormMetricsReporter(workerData);
    metricReporter.init();
    workerData.setMetricsReporter(metricReporter);
    // refresh hearbeat to Local dir
    RunnableCallback heartbeatFn = new WorkerHeartbeatRunable(workerData);
    AsyncLoopThread hb = new AsyncLoopThread(heartbeatFn, false, null, Thread.NORM_PRIORITY, true);
    threads.add(hb);
    // shutdown task callbacks
    List<TaskShutdownDameon> shutdownTasks = createTasks();
    workerData.setShutdownTasks(shutdownTasks);
    //create worker serializes/deserializes
    List<AsyncLoopThread> serializeThreads = workerData.setSerializeThreads();
    threads.addAll(serializeThreads);
    List<AsyncLoopThread> deserializeThreads = workerData.setDeserializeThreads();
    threads.addAll(deserializeThreads);
    return new WorkerShutdown(workerData, threads);
}

Example 5 with TaskShutdownDameon

use of com.alibaba.jstorm.task.TaskShutdownDameon in project jstorm by alibaba.

the class Worker method createTasks.

private List<TaskShutdownDameon> createTasks() throws Exception {
    List<TaskShutdownDameon> shutdownTasks = new ArrayList<>();
    Set<Integer> taskIds = workerData.getTaskids();
    Set<Thread> threads = new HashSet<>();
    List<Task> taskArrayList = new ArrayList<>();
    for (int taskId : taskIds) {
        Task task = new Task(workerData, taskId);
        Thread thread = new Thread(task);
        threads.add(thread);
        taskArrayList.add(task);
        thread.start();
    }
    for (Thread thread : threads) {
        thread.join();
    }
    for (Task t : taskArrayList) {
        shutdownTasks.add(t.getTaskShutdownDameon());
    }
    return shutdownTasks;
}