Examples with KVSerializable com.alibaba.jstorm.metric.KVSerializable used on opensource projects

Example 1 with KVSerializable

use of com.alibaba.jstorm.metric.KVSerializable in project jstorm by alibaba.

the class HBaseMetricSendClient method updateTasks.

private void updateTasks(TaskDeadEvent event) {
    Map<Integer, ResourceWorkerSlot> deadTasks = event.getDeadTasks();
    List<KVSerializable> taskTrackList = new ArrayList<>(deadTasks.size());
    for (Map.Entry<Integer, ResourceWorkerSlot> task : deadTasks.entrySet()) {
        TaskTrack taskTrack = new TaskTrack(event.getClusterMetricsContext().getClusterName(), event.getTopologyId());
        taskTrack.setEnd(new Date(event.getTimestamp()));
        taskTrack.setTaskId(task.getKey());
        taskTrack.setHost(task.getValue().getHostname());
        taskTrack.setPort(task.getValue().getPort());
        taskTrackList.add(taskTrack);
    }
    batchAdd(taskTrackList, TABLE_TASK_TRACK);
}

Example 2 with KVSerializable

use of com.alibaba.jstorm.metric.KVSerializable in project jstorm by alibaba.

the class HBaseMetricSendClient method batchAdd.

protected boolean batchAdd(Collection<KVSerializable> items, String tableName) {
    int size = items.size();
    List<Put> batch = new ArrayList<>(size);
    HTableInterface table = getHTableInterface(tableName);
    for (KVSerializable v : items) {
        byte[] rowKey = v.getKey();
        Put put = new Put(rowKey);
        put.add(CF, V_DATA, v.getValue());
        batch.add(put);
    }
    try {
        table.put(batch);
        table.flushCommits();
        succCounter.update(size);
        hbaseSendTps.update(size);
    } catch (Throwable ex) {
        logger.error("Error", ex);
        failedCounter.update(size);
        return false;
    } finally {
        closeTable(table);
    }
    return true;
}

Example 3 with KVSerializable

use of com.alibaba.jstorm.metric.KVSerializable in project jstorm by alibaba.

the class HBaseMetricSendClient method insertOrUpdateTasks.

/**
 * insert tasks on new assign, update tasks on rebalance.
 */
private void insertOrUpdateTasks(TaskStartEvent event) {
    Assignment old = event.getOldAssignment();
    Assignment current = event.getNewAssignment();
    Map<Integer, String> task2Component = event.getTask2Component();
    List<KVSerializable> taskTrackList = new ArrayList<>();
    // assign
    if (old == null) {
        Set<ResourceWorkerSlot> workers = current.getWorkers();
        logger.info("old workers are null, assigned workers:{}", Joiner.on(",").join(workers));
        for (ResourceWorkerSlot worker : workers) {
            Set<Integer> tasks = worker.getTasks();
            for (Integer task : tasks) {
                TaskTrack track = new TaskTrack(event.getClusterMetricsContext().getClusterName(), event.getTopologyId());
                track.setStart(new Date(event.getTimestamp()));
                track.setComponent(task2Component.get(task));
                track.setHost(worker.getHostname());
                track.setPort(worker.getPort());
                track.setTaskId(task);
                taskTrackList.add(track);
            }
        }
    } else {
        // rebalance, we only insert newly assigned tasks
        Set<ResourceWorkerSlot> oldWorkers = old.getWorkers();
        Joiner joiner = Joiner.on(",");
        logger.info("old workers:{}, new workers:{}", joiner.join(oldWorkers), joiner.join(current.getWorkers()));
        for (ResourceWorkerSlot worker : current.getWorkers()) {
            // a new worker, insert all tasks
            if (!oldWorkers.contains(worker)) {
                for (Integer task : worker.getTasks()) {
                    TaskTrack track = new TaskTrack(event.getClusterMetricsContext().getClusterName(), event.getTopologyId());
                    track.setStart(new Date(event.getTimestamp()));
                    track.setComponent(task2Component.get(task));
                    track.setHost(worker.getHostname());
                    track.setPort(worker.getPort());
                    track.setTaskId(task);
                    taskTrackList.add(track);
                }
            } else {
                for (Integer task : worker.getTasks()) {
                    ResourceWorkerSlot oldWorker = old.getWorkerByTaskId(task);
                    if (oldWorker != null) {
                        // update end time of old task
                        TaskTrack oldTrack = new TaskTrack(event.getClusterMetricsContext().getClusterName(), event.getTopologyId());
                        oldTrack.setEnd(new Date(event.getTimestamp()));
                        oldTrack.setTaskId(task);
                        oldTrack.setHost(oldWorker.getHostname());
                        oldTrack.setPort(oldWorker.getPort());
                        taskTrackList.add(oldTrack);
                        // insert new task
                        TaskTrack track = new TaskTrack(event.getClusterMetricsContext().getClusterName(), event.getTopologyId());
                        track.setStart(new Date());
                        track.setComponent(task2Component.get(task));
                        track.setHost(worker.getHostname());
                        track.setPort(worker.getPort());
                        track.setTaskId(task);
                        taskTrackList.add(track);
                    }
                }
            }
        }
    }
    if (taskTrackList.size() > 0) {
        batchAdd(taskTrackList, TABLE_TASK_TRACK);
    }
}

Example 4 with KVSerializable

use of com.alibaba.jstorm.metric.KVSerializable in project jstorm by alibaba.

the class HBaseMetricSendClient method registerMetrics.

@Override
public boolean registerMetrics(String clusterName, String topologyId, Map<String, Long> metricMeta) throws Exception {
    List<KVSerializable> metaList = new ArrayList<>(metricMeta.size());
    for (Map.Entry<String, Long> entry : metricMeta.entrySet()) {
        MetricMeta meta = MetricMeta.parse(entry.getKey());
        meta.setClusterName(clusterName);
        meta.setId(entry.getValue());
        metaList.add(meta);
    }
    return batchAdd(metaList, TABLE_METRIC_META);
}

Example 5 with KVSerializable

use of com.alibaba.jstorm.metric.KVSerializable in project jstorm by alibaba.

the class HBaseMetricSendClient method sendMetricInfo.

private void sendMetricInfo(final String topologyId, final MetricInfo metricInfo, final MetaType metaType, final Date time) {
    if (metricUploadFilter.filter(metaType)) {
        return;
    }
    int cnt = 0;
    int BATCH = 100;
    logger.info("HBase insert thread, topology:{}, type:{}, time:{}, size:{}", topologyId, metaType, TimeUtils.toTimeStr(time), metricInfo.get_metrics_size());
    List<KVSerializable> metricDataList = new ArrayList<>(BATCH);
    try {
        Map<String, Map<Integer, MetricSnapshot>> metrics = metricInfo.get_metrics();
        for (Map.Entry<String, Map<Integer, MetricSnapshot>> metricEntry : metrics.entrySet()) {
            Map<Integer, MetricSnapshot> metric = metricEntry.getValue();
            for (Map.Entry<Integer, MetricSnapshot> entry : metric.entrySet()) {
                Integer win = entry.getKey();
                MetricSnapshot snapshot = entry.getValue();
                if (MetricUtils.isValidId(snapshot.get_metricId())) {
                    MetricType metricType = MetricType.parse(snapshot.get_metricType());
                    if (metricType == MetricType.COUNTER) {
                        metricDataList.add(MetricDataConverter.toCounterData(snapshot, win));
                    } else if (metricType == MetricType.GAUGE) {
                        metricDataList.add(MetricDataConverter.toGaugeData(snapshot, win));
                    } else if (metricType == MetricType.METER) {
                        metricDataList.add(MetricDataConverter.toMeterData(snapshot, win));
                    } else if (metricType == MetricType.HISTOGRAM) {
                        metricDataList.add(MetricDataConverter.toHistogramData(snapshot, win));
                    }
                    cnt++;
                }
                if (cnt >= BATCH) {
                    batchAdd(metricDataList, TABLE_METRIC_DATA);
                    cnt = 0;
                    metricDataList.clear();
                }
            }
        }
        if (cnt > 0) {
            batchAdd(metricDataList, TABLE_METRIC_DATA);
        }
    } catch (Throwable ex) {
        logger.error("transaction error", ex);
    }
}