Examples with BoltCollector com.alibaba.jstorm.task.execute.BoltCollector used on opensource projects

Example 1 with BoltCollector

use of com.alibaba.jstorm.task.execute.BoltCollector in project jstorm by alibaba.

the class SnapshotStateMaster method process.

public synchronized void process(Tuple tuple) {
    int taskId = tuple.getSourceTask();
    TopoMasterCtrlEvent event = (TopoMasterCtrlEvent) tuple.getValues().get(0);
    BatchGroupId batchGroupId = null;
    if (event.getEventValue() != null && event.getEventValue().size() > 0) {
        batchGroupId = (BatchGroupId) event.getEventValue().get(0);
    }
    SnapshotState snapshotState = null;
    // If batchGroupId == null, it is initState request
    if (batchGroupId != null) {
        snapshotState = topologySnapshotState.get(batchGroupId.groupId);
        if (snapshotState == null) {
            LOG.warn("unexpected event from task-{}, event={}", taskId, event.toString());
            ;
            return;
        }
    }
    LOG.debug("Received control event from task-{}, event={}", taskId, event);
    boolean isFinished = false;
    switch(event.getEventType()) {
        case transactionInitState:
            for (Entry<Integer, SnapshotState> entry : topologySnapshotState.entrySet()) {
                TransactionState state = entry.getValue().getInitState(taskId);
                int groupId = entry.getKey();
                if (state != null) {
                    TopoMasterCtrlEvent initStateResp = new TopoMasterCtrlEvent(EventType.transactionInitState);
                    initStateResp.addEventValue(state);
                    ((BoltCollector) (outputCollector.getDelegate())).emitDirectCtrl(taskId, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(initStateResp));
                }
            }
            break;
        case transactionCommit:
            TransactionState commitState = (TransactionState) event.getEventValue().get(1);
            isFinished = snapshotState.commit(batchGroupId.batchId, taskId, commitState);
            break;
        case transactionAck:
            isFinished = snapshotState.ackEndBolt(batchGroupId.batchId, taskId);
            break;
        case transactionRollback:
            // If receiving rollback request, TM will rollback all tasks which are on the same stream as this task
            stopAndRollback(batchGroupId.groupId, snapshotState.rollback());
            break;
        default:
            LOG.warn("unexpected event type from task-{}, event={}", taskId, event.toString());
            break;
    }
    if (isFinished) {
        finishSnapshotStateCommit(batchGroupId, snapshotState);
    }
    LOG.debug("snapshotState: {}", snapshotState);
}

Example 2 with BoltCollector

use of com.alibaba.jstorm.task.execute.BoltCollector in project jstorm by alibaba.

the class JStormMetricsReporter method uploadMetricData.

public void uploadMetricData(WorkerUploadMetrics metrics) {
    if (inTopology) {
        //in Worker, we upload data via netty transport
        if (boltOutput != null) {
            LOG.debug("emit metrics through bolt collector.");
            ((BoltCollector) boltOutput.getDelegate()).emitCtrl(Common.TOPOLOGY_MASTER_METRICS_STREAM_ID, null, new Values(JStormServerUtils.getName(host, port), metrics));
        } else if (spoutOutput != null) {
            LOG.debug("emit metrics through spout collector.");
            ((SpoutCollector) spoutOutput.getDelegate()).emitCtrl(Common.TOPOLOGY_MASTER_METRICS_STREAM_ID, new Values(JStormServerUtils.getName(host, port), metrics), null);
        } else {
            LOG.warn("topology:{}, both spout/bolt collectors are null, don't know what to do...", topologyId);
        }
    } else {
        // in supervisor or nimbus, we upload metric data via thrift
        LOG.debug("emit metrics through nimbus client.");
        TopologyMetric tpMetric = MetricUtils.mkTopologyMetric();
        tpMetric.set_workerMetric(metrics.get_allMetrics());
        //UpdateEvent.pushEvent(topologyId, tpMetric);
        try {
            // push metrics via nimbus client
            if (client == null) {
                LOG.warn("nimbus client is null...");
                client = new NimbusClientWrapper();
                client.init(conf);
            }
            client.getClient().uploadTopologyMetrics(topologyId, tpMetric);
        } catch (Throwable ex) {
            LOG.error("upload metrics error:", ex);
            if (client != null) {
                client.cleanup();
                client = null;
            }
        }
    }
//MetricUtils.logMetrics(metrics.get_allMetrics());
}

Example 3 with BoltCollector

use of com.alibaba.jstorm.task.execute.BoltCollector in project jstorm by alibaba.

the class SnapshotStateMaster method sendRollbackRequest.

private void sendRollbackRequest(int groupId, Map<Integer, TransactionState> tasksToStates) {
    // Rollback for stateful tasks
    Iterator<Integer> iter = tasksToStates.keySet().iterator();
    while (iter.hasNext()) {
        int taskId = iter.next();
        TransactionState state = tasksToStates.get(taskId) != null ? tasksToStates.get(taskId) : new TransactionState(groupId, 0, null, null);
        TopoMasterCtrlEvent rollbackRequest = new TopoMasterCtrlEvent(EventType.transactionRollback);
        rollbackRequest.addEventValue(state);
        ((BoltCollector) (outputCollector.getDelegate())).emitDirectCtrl(taskId, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(rollbackRequest));
    }
    LOG.info("Send rollback request to group:{}, tasks:{}", groupIdToNames.get(groupId), tasksToStates.keySet());
    // Rollback for non-stateful tasks
    SnapshotState snapshot = topologySnapshotState.get(groupId);
    TransactionState state = new TransactionState(groupId, snapshot.getLastSuccessfulBatchId(), null, null);
    TopoMasterCtrlEvent rollbackRequest = new TopoMasterCtrlEvent(EventType.transactionRollback);
    rollbackRequest.addEventValue(state);
    for (Integer nonStatefulTaskId : snapshot.getNonStatefulTasks()) {
        ((BoltCollector) (outputCollector.getDelegate())).emitDirectCtrl(nonStatefulTaskId, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(rollbackRequest));
    }
}

Example 4 with BoltCollector

use of com.alibaba.jstorm.task.execute.BoltCollector in project jstorm by alibaba.

the class SnapshotStateMaster method stopAndRollback.

private void stopAndRollback(final int groupId, final Map<Integer, TransactionState> tasksToStates) {
    List<String> spoutNames = groupIdToNames.get(groupId);
    for (String spoutName : spoutNames) {
        for (int taskId : context.getComponentTasks(spoutName)) {
            TopoMasterCtrlEvent stop = new TopoMasterCtrlEvent(EventType.transactionStop);
            stop.addEventValue(groupId);
            ((BoltCollector) (outputCollector.getDelegate())).emitDirectCtrl(taskId, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(stop));
        }
    }
    LOG.info("Stop spouts={}, tasks={}", spoutNames, context.getComponentsTasks(new HashSet<String>(spoutNames)));
    scheduledService.schedule(new Runnable() {

        @Override
        public void run() {
            sendRollbackRequest(groupId, tasksToStates);
        }
    }, 10, TimeUnit.SECONDS);
}

Example 5 with BoltCollector

use of com.alibaba.jstorm.task.execute.BoltCollector in project jstorm by alibaba.

the class SnapshotStateMaster method finishSnapshotStateCommit.

private void finishSnapshotStateCommit(BatchGroupId batchGroupId, SnapshotState snapshotState) {
    Set<Integer> statefulTasks = new HashSet<Integer>();
    statefulTasks.addAll(snapshotState.getSpoutTasks());
    TopoMasterCtrlEvent resp = null;
    boolean isCommitSuccess = false;
    if (batchGroupId.batchId != TransactionCommon.INIT_BATCH_ID) {
        if (snapshotState.isActive()) {
            resp = new TopoMasterCtrlEvent(EventType.transactionCommit);
            resp.addEventValue(batchGroupId);
            statefulTasks.addAll(snapshotState.getStatefulTasks());
            snapshotState.successBatch(batchGroupId.batchId);
            // Try to persist the topology snapshot state. But if any failure happened, just continue.
            try {
                isCommitSuccess = stateOperator.commitState(topologyName, batchGroupId.groupId, topologySnapshotState);
                if (!isCommitSuccess) {
                    LOG.warn("Failed to commit topology state for batch-{}", batchGroupId);
                }
            } catch (Exception e) {
                LOG.warn("Got exception, when committing topology state for batch-{}, {}", batchGroupId, e);
            }
        }
    } else {
        snapshotState.setActive();
        resp = new TopoMasterCtrlEvent(EventType.transactionStart);
        snapshotState.successBatch(batchGroupId.batchId);
        isCommitSuccess = true;
    }
    if (isCommitSuccess) {
        // Ack the init or commit request from spout and stateful bolt
        LOG.debug("Send ack to spouts-{}, event={}", statefulTasks, resp);
        for (Integer spoutTask : statefulTasks) {
            ((BoltCollector) (outputCollector.getDelegate())).emitDirectCtrl(spoutTask, Common.TOPOLOGY_MASTER_CONTROL_STREAM_ID, null, new Values(resp));
        }
    }
    long nextPendingSuccessBatch = snapshotState.getPendingSuccessBatch();
    if (nextPendingSuccessBatch != -1) {
        LOG.info("Try to commit a pending successful batch-{}", nextPendingSuccessBatch);
        finishSnapshotStateCommit(new BatchGroupId(batchGroupId.groupId, nextPendingSuccessBatch), snapshotState);
    }
}