Examples with EntityDataUpdate org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate used on opensource projects

Example 6 with EntityDataUpdate

use of org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate in project thingsboard by thingsboard.

the class DefaultTbEntityDataSubscriptionService method handleLatestCmd.

private void handleLatestCmd(TbEntityDataSubCtx ctx, LatestValueCmd latestCmd) {
    log.trace("[{}][{}] Going to process latest command: {}", ctx.getSessionId(), ctx.getCmdId(), latestCmd);
    // Fetch the latest values for telemetry keys (in case they are not copied from NoSQL to SQL DB in hybrid mode.
    if (!tsInSqlDB) {
        log.trace("[{}][{}] Going to fetch missing latest values: {}", ctx.getSessionId(), ctx.getCmdId(), latestCmd);
        List<String> allTsKeys = latestCmd.getKeys().stream().filter(key -> key.getType().equals(EntityKeyType.TIME_SERIES)).map(EntityKey::getKey).collect(Collectors.toList());
        Map<EntityData, ListenableFuture<Map<String, TsValue>>> missingTelemetryFutures = new HashMap<>();
        for (EntityData entityData : ctx.getData().getData()) {
            Map<EntityKeyType, Map<String, TsValue>> latestEntityData = entityData.getLatest();
            Map<String, TsValue> tsEntityData = latestEntityData.get(EntityKeyType.TIME_SERIES);
            Set<String> missingTsKeys = new LinkedHashSet<>(allTsKeys);
            if (tsEntityData != null) {
                missingTsKeys.removeAll(tsEntityData.keySet());
            } else {
                tsEntityData = new HashMap<>();
                latestEntityData.put(EntityKeyType.TIME_SERIES, tsEntityData);
            }
            ListenableFuture<List<TsKvEntry>> missingTsData = tsService.findLatest(ctx.getTenantId(), entityData.getEntityId(), missingTsKeys);
            missingTelemetryFutures.put(entityData, Futures.transform(missingTsData, this::toTsValue, MoreExecutors.directExecutor()));
        }
        Futures.addCallback(Futures.allAsList(missingTelemetryFutures.values()), new FutureCallback<List<Map<String, TsValue>>>() {

            @Override
            public void onSuccess(@Nullable List<Map<String, TsValue>> result) {
                missingTelemetryFutures.forEach((key, value) -> {
                    try {
                        key.getLatest().get(EntityKeyType.TIME_SERIES).putAll(value.get());
                    } catch (InterruptedException | ExecutionException e) {
                        log.warn("[{}][{}] Failed to lookup latest telemetry: {}:{}", ctx.getSessionId(), ctx.getCmdId(), key.getEntityId(), allTsKeys, e);
                    }
                });
                EntityDataUpdate update;
                if (!ctx.isInitialDataSent()) {
                    update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription());
                    ctx.setInitialDataSent(true);
                } else {
                    update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription());
                }
                wsService.sendWsMsg(ctx.getSessionId(), update);
                ctx.createLatestValuesSubscriptions(latestCmd.getKeys());
            }

            @Override
            public void onFailure(Throwable t) {
                log.warn("[{}][{}] Failed to process websocket command: {}:{}", ctx.getSessionId(), ctx.getCmdId(), ctx.getQuery(), latestCmd, t);
                wsService.sendWsMsg(ctx.getSessionId(), new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to process websocket command!"));
            }
        }, wsCallBackExecutor);
    } else {
        if (!ctx.isInitialDataSent()) {
            EntityDataUpdate update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription());
            wsService.sendWsMsg(ctx.getSessionId(), update);
            ctx.setInitialDataSent(true);
        }
        ctx.createLatestValuesSubscriptions(latestCmd.getKeys());
    }
}

Example 7 with EntityDataUpdate

use of org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate in project thingsboard by thingsboard.

the class DefaultTbEntityDataSubscriptionService method handleCmd.

@Override
public void handleCmd(TelemetryWebSocketSessionRef session, EntityDataCmd cmd) {
    TbEntityDataSubCtx ctx = getSubCtx(session.getSessionId(), cmd.getCmdId());
    if (ctx != null) {
        log.debug("[{}][{}] Updating existing subscriptions using: {}", session.getSessionId(), cmd.getCmdId(), cmd);
        if (cmd.getLatestCmd() != null || cmd.getTsCmd() != null || cmd.getHistoryCmd() != null) {
            ctx.clearEntitySubscriptions();
        }
    } else {
        log.debug("[{}][{}] Creating new subscription using: {}", session.getSessionId(), cmd.getCmdId(), cmd);
        ctx = createSubCtx(session, cmd);
    }
    ctx.setCurrentCmd(cmd);
    if (cmd.getQuery() != null) {
        if (ctx.getQuery() == null) {
            log.debug("[{}][{}] Initializing data using query: {}", session.getSessionId(), cmd.getCmdId(), cmd.getQuery());
        } else {
            log.debug("[{}][{}] Updating data using query: {}", session.getSessionId(), cmd.getCmdId(), cmd.getQuery());
        }
        ctx.setAndResolveQuery(cmd.getQuery());
        EntityDataQuery query = ctx.getQuery();
        // Step 1. Update existing query with the contents of LatestValueCmd
        if (cmd.getLatestCmd() != null) {
            cmd.getLatestCmd().getKeys().forEach(key -> {
                if (!query.getLatestValues().contains(key)) {
                    query.getLatestValues().add(key);
                }
            });
        }
        long start = System.currentTimeMillis();
        ctx.fetchData();
        long end = System.currentTimeMillis();
        stats.getRegularQueryInvocationCnt().incrementAndGet();
        stats.getRegularQueryTimeSpent().addAndGet(end - start);
        ctx.cancelTasks();
        if (ctx.getQuery().getPageLink().isDynamic()) {
            // TODO: validate number of dynamic page links against rate limits. Ignore dynamic flag if limit is reached.
            TbEntityDataSubCtx finalCtx = ctx;
            ScheduledFuture<?> task = scheduler.scheduleWithFixedDelay(() -> refreshDynamicQuery(finalCtx), dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS);
            finalCtx.setRefreshTask(task);
        }
    }
    ListenableFuture<TbEntityDataSubCtx> historyFuture;
    if (cmd.getHistoryCmd() != null) {
        log.trace("[{}][{}] Going to process history command: {}", session.getSessionId(), cmd.getCmdId(), cmd.getHistoryCmd());
        try {
            historyFuture = handleHistoryCmd(ctx, cmd.getHistoryCmd());
        } catch (RuntimeException e) {
            handleWsCmdRuntimeException(ctx.getSessionId(), e, cmd);
            return;
        }
    } else {
        historyFuture = Futures.immediateFuture(ctx);
    }
    Futures.addCallback(historyFuture, new FutureCallback<>() {

        @Override
        public void onSuccess(@Nullable TbEntityDataSubCtx theCtx) {
            try {
                if (cmd.getLatestCmd() != null) {
                    handleLatestCmd(theCtx, cmd.getLatestCmd());
                } else if (cmd.getTsCmd() != null) {
                    handleTimeSeriesCmd(theCtx, cmd.getTsCmd());
                } else if (!theCtx.isInitialDataSent()) {
                    EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription());
                    wsService.sendWsMsg(theCtx.getSessionId(), update);
                    theCtx.setInitialDataSent(true);
                }
            } catch (RuntimeException e) {
                handleWsCmdRuntimeException(theCtx.getSessionId(), e, cmd);
            }
        }

        @Override
        public void onFailure(Throwable t) {
            log.warn("[{}][{}] Failed to process command", session.getSessionId(), cmd.getCmdId());
        }
    }, wsCallBackExecutor);
}

Example 8 with EntityDataUpdate

use of org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate in project thingsboard by thingsboard.

the class DefaultTbEntityDataSubscriptionService method handleGetTsCmd.

private ListenableFuture<TbEntityDataSubCtx> handleGetTsCmd(TbEntityDataSubCtx ctx, GetTsCmd cmd, boolean subscribe) {
    List<String> keys = cmd.getKeys();
    List<ReadTsKvQuery> finalTsKvQueryList;
    List<ReadTsKvQuery> tsKvQueryList = cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery(key, cmd.getStartTs(), cmd.getEndTs(), cmd.getInterval(), getLimit(cmd.getLimit()), cmd.getAgg())).collect(Collectors.toList());
    if (cmd.isFetchLatestPreviousPoint()) {
        finalTsKvQueryList = new ArrayList<>(tsKvQueryList);
        finalTsKvQueryList.addAll(cmd.getKeys().stream().map(key -> new BaseReadTsKvQuery(key, cmd.getStartTs() - TimeUnit.DAYS.toMillis(365), cmd.getStartTs(), cmd.getInterval(), 1, cmd.getAgg())).collect(Collectors.toList()));
    } else {
        finalTsKvQueryList = tsKvQueryList;
    }
    Map<EntityData, ListenableFuture<List<TsKvEntry>>> fetchResultMap = new HashMap<>();
    ctx.getData().getData().forEach(entityData -> fetchResultMap.put(entityData, tsService.findAll(ctx.getTenantId(), entityData.getEntityId(), finalTsKvQueryList)));
    return Futures.transform(Futures.allAsList(fetchResultMap.values()), f -> {
        fetchResultMap.forEach((entityData, future) -> {
            Map<String, List<TsValue>> keyData = new LinkedHashMap<>();
            cmd.getKeys().forEach(key -> keyData.put(key, new ArrayList<>()));
            try {
                List<TsKvEntry> entityTsData = future.get();
                if (entityTsData != null) {
                    entityTsData.forEach(entry -> keyData.get(entry.getKey()).add(new TsValue(entry.getTs(), entry.getValueAsString())));
                }
                keyData.forEach((k, v) -> entityData.getTimeseries().put(k, v.toArray(new TsValue[v.size()])));
                if (cmd.isFetchLatestPreviousPoint()) {
                    entityData.getTimeseries().values().forEach(dataArray -> {
                        Arrays.sort(dataArray, (o1, o2) -> Long.compare(o2.getTs(), o1.getTs()));
                    });
                }
            } catch (InterruptedException | ExecutionException e) {
                log.warn("[{}][{}][{}] Failed to fetch historical data", ctx.getSessionId(), ctx.getCmdId(), entityData.getEntityId(), e);
                wsService.sendWsMsg(ctx.getSessionId(), new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), "Failed to fetch historical data!"));
            }
        });
        EntityDataUpdate update;
        if (!ctx.isInitialDataSent()) {
            update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription());
            ctx.setInitialDataSent(true);
        } else {
            update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription());
        }
        wsService.sendWsMsg(ctx.getSessionId(), update);
        if (subscribe) {
            ctx.createTimeseriesSubscriptions(keys.stream().map(key -> new EntityKey(EntityKeyType.TIME_SERIES, key)).collect(Collectors.toList()), cmd.getStartTs(), cmd.getEndTs());
        }
        ctx.getData().getData().forEach(ed -> ed.getTimeseries().clear());
        return ctx;
    }, wsCallBackExecutor);
}

Example 9 with EntityDataUpdate

use of org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate in project thingsboard by thingsboard.

the class TbEntityDataSubCtx method sendLatestWsMsg.

private void sendLatestWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) {
    Map<String, TsValue> latestUpdate = new HashMap<>();
    subscriptionUpdate.getData().forEach((k, v) -> {
        Object[] data = (Object[]) v.get(0);
        latestUpdate.put(k, new TsValue((Long) data[0], (String) data[1]));
    });
    EntityData entityData = getDataForEntity(entityId);
    if (entityData != null && entityData.getLatest() != null) {
        Map<String, TsValue> latestCtxValues = entityData.getLatest().get(keyType);
        log.trace("[{}][{}][{}] Going to compare update with {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), latestCtxValues);
        if (latestCtxValues != null) {
            latestCtxValues.forEach((k, v) -> {
                TsValue update = latestUpdate.get(k);
                if (update != null) {
                    // Ignore notifications about deleted keys
                    if (!(update.getTs() == 0 && (update.getValue() == null || update.getValue().isEmpty()))) {
                        if (update.getTs() < v.getTs()) {
                            log.trace("[{}][{}][{}] Removed stale update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs());
                            latestUpdate.remove(k);
                        } else if ((update.getTs() == v.getTs() && update.getValue().equals(v.getValue()))) {
                            log.trace("[{}][{}][{}] Removed duplicate update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs());
                            latestUpdate.remove(k);
                        }
                    } else {
                        log.trace("[{}][{}][{}] Received deleted notification for: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k);
                    }
                }
            });
            // Setting new values
            latestUpdate.forEach(latestCtxValues::put);
        }
    }
    if (!latestUpdate.isEmpty()) {
        Map<EntityKeyType, Map<String, TsValue>> latestMap = Collections.singletonMap(keyType, latestUpdate);
        entityData = new EntityData(entityId, latestMap, null);
        wsService.sendWsMsg(sessionId, new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData), maxEntitiesPerDataSubscription));
    }
}

Example 10 with EntityDataUpdate

use of org.thingsboard.server.service.telemetry.cmd.v2.EntityDataUpdate in project thingsboard by thingsboard.

the class TbEntityDataSubCtx method doUpdate.

@Override
public synchronized void doUpdate(Map<EntityId, EntityData> newDataMap) {
    List<Integer> subIdsToCancel = new ArrayList<>();
    List<TbSubscription> subsToAdd = new ArrayList<>();
    Set<EntityId> currentSubs = new HashSet<>();
    subToEntityIdMap.forEach((subId, entityId) -> {
        if (!newDataMap.containsKey(entityId)) {
            subIdsToCancel.add(subId);
        } else {
            currentSubs.add(entityId);
        }
    });
    log.trace("[{}][{}] Subscriptions that are invalid: {}", sessionRef.getSessionId(), cmdId, subIdsToCancel);
    subIdsToCancel.forEach(subToEntityIdMap::remove);
    List<EntityData> newSubsList = newDataMap.entrySet().stream().filter(entry -> !currentSubs.contains(entry.getKey())).map(Map.Entry::getValue).collect(Collectors.toList());
    if (!newSubsList.isEmpty()) {
        // NOTE: We ignore the TS subscriptions for new entities here, because widgets will re-init it's content and will create new subscriptions.
        if (curTsCmd == null && latestValueCmd != null) {
            List<EntityKey> keys = latestValueCmd.getKeys();
            if (keys != null && !keys.isEmpty()) {
                Map<EntityKeyType, List<EntityKey>> keysByType = getEntityKeyByTypeMap(keys);
                newSubsList.forEach(entity -> {
                    log.trace("[{}][{}] Found new subscription for entity: {}", sessionRef.getSessionId(), cmdId, entity.getEntityId());
                    subsToAdd.addAll(addSubscriptions(entity, keysByType, true, 0, 0));
                });
            }
        }
    }
    wsService.sendWsMsg(sessionRef.getSessionId(), new EntityDataUpdate(cmdId, data, null, maxEntitiesPerDataSubscription));
    subIdsToCancel.forEach(subId -> localSubscriptionService.cancelSubscription(getSessionId(), subId));
    subsToAdd.forEach(localSubscriptionService::addSubscription);
}