use of org.thingsboard.server.common.data.query.EntityDataSortOrder in project thingsboard by thingsboard.
the class BaseAlarmServiceTest method testFindCustomerAlarm.
@Test
public void testFindCustomerAlarm() throws ExecutionException, InterruptedException {
Customer customer = new Customer();
customer.setTitle("TestCustomer");
customer.setTenantId(tenantId);
customer = customerService.saveCustomer(customer);
Device tenantDevice = new Device();
tenantDevice.setName("TestTenantDevice");
tenantDevice.setType("default");
tenantDevice.setTenantId(tenantId);
tenantDevice = deviceService.saveDevice(tenantDevice);
Device customerDevice = new Device();
customerDevice.setName("TestCustomerDevice");
customerDevice.setType("default");
customerDevice.setTenantId(tenantId);
customerDevice.setCustomerId(customer.getId());
customerDevice = deviceService.saveDevice(customerDevice);
long ts = System.currentTimeMillis();
Alarm tenantAlarm = Alarm.builder().tenantId(tenantId).originator(tenantDevice.getId()).type(TEST_ALARM).propagate(true).severity(AlarmSeverity.CRITICAL).status(AlarmStatus.ACTIVE_UNACK).startTs(ts).build();
AlarmOperationResult result = alarmService.createOrUpdateAlarm(tenantAlarm);
tenantAlarm = result.getAlarm();
Alarm deviceAlarm = Alarm.builder().tenantId(tenantId).originator(customerDevice.getId()).type(TEST_ALARM).propagate(true).severity(AlarmSeverity.CRITICAL).status(AlarmStatus.ACTIVE_UNACK).startTs(ts).build();
result = alarmService.createOrUpdateAlarm(deviceAlarm);
deviceAlarm = result.getAlarm();
AlarmDataPageLink pageLink = new AlarmDataPageLink();
pageLink.setPage(0);
pageLink.setPageSize(10);
pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.ALARM_FIELD, "createdTime")));
pageLink.setStartTs(0L);
pageLink.setEndTs(System.currentTimeMillis());
pageLink.setSearchPropagatedAlarms(true);
pageLink.setSeverityList(Arrays.asList(AlarmSeverity.CRITICAL, AlarmSeverity.WARNING));
pageLink.setStatusList(Arrays.asList(AlarmSearchStatus.ACTIVE));
PageData<AlarmData> tenantAlarms = alarmService.findAlarmDataByQueryForEntities(tenantId, toQuery(pageLink), Arrays.asList(tenantDevice.getId(), customerDevice.getId()));
Assert.assertEquals(2, tenantAlarms.getData().size());
PageData<AlarmData> customerAlarms = alarmService.findAlarmDataByQueryForEntities(tenantId, toQuery(pageLink), Collections.singletonList(customerDevice.getId()));
Assert.assertEquals(1, customerAlarms.getData().size());
Assert.assertEquals(deviceAlarm, customerAlarms.getData().get(0));
PageData<AlarmInfo> alarms = alarmService.findAlarms(tenantId, AlarmQuery.builder().affectedEntityId(tenantDevice.getId()).status(AlarmStatus.ACTIVE_UNACK).pageLink(new TimePageLink(10, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), 0L, System.currentTimeMillis())).build()).get();
Assert.assertNotNull(alarms.getData());
Assert.assertEquals(1, alarms.getData().size());
Assert.assertEquals(tenantAlarm, alarms.getData().get(0));
}
use of org.thingsboard.server.common.data.query.EntityDataSortOrder in project thingsboard by thingsboard.
the class DefaultEntityQueryRepository method findEntityDataByQuery.
@Override
public PageData<EntityData> findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) {
return transactionTemplate.execute(status -> {
EntityType entityType = resolveEntityType(query.getEntityFilter());
QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType));
EntityDataPageLink pageLink = query.getPageLink();
List<EntityKeyMapping> mappings = EntityKeyMapping.prepareKeyMapping(query);
List<EntityKeyMapping> selectionMapping = mappings.stream().filter(EntityKeyMapping::isSelection).collect(Collectors.toList());
List<EntityKeyMapping> entityFieldsSelectionMapping = selectionMapping.stream().filter(mapping -> !mapping.isLatest()).collect(Collectors.toList());
List<EntityKeyMapping> latestSelectionMapping = selectionMapping.stream().filter(EntityKeyMapping::isLatest).collect(Collectors.toList());
List<EntityKeyMapping> filterMapping = mappings.stream().filter(EntityKeyMapping::hasFilter).collect(Collectors.toList());
List<EntityKeyMapping> entityFieldsFiltersMapping = filterMapping.stream().filter(mapping -> !mapping.isLatest()).collect(Collectors.toList());
List<EntityKeyMapping> allLatestMappings = mappings.stream().filter(EntityKeyMapping::isLatest).collect(Collectors.toList());
String entityWhereClause = DefaultEntityQueryRepository.this.buildEntityWhere(ctx, query.getEntityFilter(), entityFieldsFiltersMapping);
String latestJoinsCnt = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, true);
String latestJoinsData = EntityKeyMapping.buildLatestJoins(ctx, query.getEntityFilter(), entityType, allLatestMappings, false);
String textSearchQuery = DefaultEntityQueryRepository.this.buildTextSearchQuery(ctx, selectionMapping, pageLink.getTextSearch());
String entityFieldsSelection = EntityKeyMapping.buildSelections(entityFieldsSelectionMapping, query.getEntityFilter().getType(), entityType);
String entityTypeStr;
if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) {
entityTypeStr = "e.entity_type";
} else {
entityTypeStr = "'" + entityType.name() + "'";
}
if (!StringUtils.isEmpty(entityFieldsSelection)) {
entityFieldsSelection = String.format("e.id id, %s entity_type, %s", entityTypeStr, entityFieldsSelection);
} else {
entityFieldsSelection = String.format("e.id id, %s entity_type", entityTypeStr);
}
String latestSelection = EntityKeyMapping.buildSelections(latestSelectionMapping, query.getEntityFilter().getType(), entityType);
String topSelection = "entities.*";
if (!StringUtils.isEmpty(latestSelection)) {
topSelection = topSelection + ", " + latestSelection;
}
String fromClauseCount = String.format("from (select %s from (select %s from %s e where %s) entities %s ) result %s", "entities.*", entityFieldsSelection, addEntityTableQuery(ctx, query.getEntityFilter()), entityWhereClause, latestJoinsCnt, textSearchQuery);
String fromClauseData = String.format("from (select %s from (select %s from %s e where %s) entities %s ) result %s", topSelection, entityFieldsSelection, addEntityTableQuery(ctx, query.getEntityFilter()), entityWhereClause, latestJoinsData, textSearchQuery);
if (!StringUtils.isEmpty(pageLink.getTextSearch())) {
// Unfortunately, we need to sacrifice performance in case of full text search, because it is applied to all joined records.
fromClauseCount = fromClauseData;
}
String countQuery = String.format("select count(id) %s", fromClauseCount);
long startTs = System.currentTimeMillis();
int totalElements;
try {
totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
} finally {
queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - startTs);
}
if (totalElements == 0) {
return new PageData<>();
}
String dataQuery = String.format("select * %s", fromClauseData);
EntityDataSortOrder sortOrder = pageLink.getSortOrder();
if (sortOrder != null) {
Optional<EntityKeyMapping> sortOrderMappingOpt = mappings.stream().filter(EntityKeyMapping::isSortOrder).findFirst();
if (sortOrderMappingOpt.isPresent()) {
EntityKeyMapping sortOrderMapping = sortOrderMappingOpt.get();
String direction = sortOrder.getDirection() == EntityDataSortOrder.Direction.ASC ? "asc" : "desc";
if (sortOrderMapping.getEntityKey().getType() == EntityKeyType.ENTITY_FIELD) {
dataQuery = String.format("%s order by %s %s, result.id %s", dataQuery, sortOrderMapping.getValueAlias(), direction, direction);
} else {
dataQuery = String.format("%s order by %s %s, %s %s, result.id %s", dataQuery, sortOrderMapping.getSortOrderNumAlias(), direction, sortOrderMapping.getSortOrderStrAlias(), direction, direction);
}
}
}
int startIndex = pageLink.getPageSize() * pageLink.getPage();
if (pageLink.getPageSize() > 0) {
dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex);
}
startTs = System.currentTimeMillis();
List<Map<String, Object>> rows;
try {
rows = jdbcTemplate.queryForList(dataQuery, ctx);
} finally {
queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - startTs);
}
return EntityDataAdapter.createEntityData(pageLink, selectionMapping, rows, totalElements);
});
}
use of org.thingsboard.server.common.data.query.EntityDataSortOrder in project thingsboard by thingsboard.
the class DefaultAlarmQueryRepository method findAlarmDataByQueryForEntities.
@Override
public PageData<AlarmData> findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection<EntityId> orderedEntityIds) {
return transactionTemplate.execute(status -> {
AlarmDataPageLink pageLink = query.getPageLink();
QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM));
ctx.addUuidListParameter("entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList()));
StringBuilder selectPart = new StringBuilder(FIELDS_SELECTION);
StringBuilder fromPart = new StringBuilder(" from alarm a ");
StringBuilder wherePart = new StringBuilder(" where ");
StringBuilder sortPart = new StringBuilder(" order by ");
StringBuilder joinPart = new StringBuilder();
boolean addAnd = false;
if (pageLink.isSearchPropagatedAlarms()) {
selectPart.append(" ea.entity_id as entity_id ");
fromPart.append(JOIN_ENTITY_ALARMS);
wherePart.append(buildPermissionsQuery(tenantId, ctx));
addAnd = true;
} else {
selectPart.append(" a.originator_id as entity_id ");
}
EntityDataSortOrder sortOrder = pageLink.getSortOrder();
if (sortOrder != null && sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) {
String sortOrderKey = sortOrder.getKey().getKey();
sortPart.append(alarmFieldColumnMap.getOrDefault(sortOrderKey, sortOrderKey)).append(" ").append(sortOrder.getDirection().name());
if (pageLink.isSearchPropagatedAlarms()) {
wherePart.append(" and ea.entity_id in (:entity_ids)");
} else {
addAndIfNeeded(wherePart, addAnd);
addAnd = true;
wherePart.append(" a.originator_id in (:entity_ids)");
}
} else {
joinPart.append(" inner join (select * from (VALUES");
int entityIdIdx = 0;
int lastEntityIdIdx = orderedEntityIds.size() - 1;
for (EntityId entityId : orderedEntityIds) {
joinPart.append("(uuid('").append(entityId.getId().toString()).append("'), ").append(entityIdIdx).append(")");
if (entityIdIdx != lastEntityIdIdx) {
joinPart.append(",");
} else {
joinPart.append(")");
}
entityIdIdx++;
}
joinPart.append(" as e(id, priority)) e ");
if (pageLink.isSearchPropagatedAlarms()) {
joinPart.append("on ea.entity_id = e.id");
} else {
joinPart.append("on a.originator_id = e.id");
}
sortPart.append("e.priority");
}
long startTs;
long endTs;
if (pageLink.getTimeWindow() > 0) {
endTs = System.currentTimeMillis();
startTs = endTs - pageLink.getTimeWindow();
} else {
startTs = pageLink.getStartTs();
endTs = pageLink.getEndTs();
}
if (startTs > 0) {
addAndIfNeeded(wherePart, addAnd);
addAnd = true;
ctx.addLongParameter("startTime", startTs);
wherePart.append("a.created_time >= :startTime");
if (pageLink.isSearchPropagatedAlarms()) {
wherePart.append(" and ea.created_time >= :startTime");
}
}
if (endTs > 0) {
addAndIfNeeded(wherePart, addAnd);
addAnd = true;
ctx.addLongParameter("endTime", endTs);
wherePart.append("a.created_time <= :endTime");
if (pageLink.isSearchPropagatedAlarms()) {
wherePart.append(" and ea.created_time <= :endTime");
}
}
if (pageLink.getTypeList() != null && !pageLink.getTypeList().isEmpty()) {
addAndIfNeeded(wherePart, addAnd);
addAnd = true;
ctx.addStringListParameter("alarmTypes", pageLink.getTypeList());
wherePart.append("a.type in (:alarmTypes)");
if (pageLink.isSearchPropagatedAlarms()) {
wherePart.append(" and ea.alarm_type in (:alarmTypes)");
}
}
if (pageLink.getSeverityList() != null && !pageLink.getSeverityList().isEmpty()) {
addAndIfNeeded(wherePart, addAnd);
addAnd = true;
ctx.addStringListParameter("alarmSeverities", pageLink.getSeverityList().stream().map(AlarmSeverity::name).collect(Collectors.toList()));
wherePart.append("a.severity in (:alarmSeverities)");
}
if (pageLink.getStatusList() != null && !pageLink.getStatusList().isEmpty()) {
Set<AlarmStatus> statusSet = toStatusSet(pageLink.getStatusList());
if (!statusSet.isEmpty()) {
addAndIfNeeded(wherePart, addAnd);
addAnd = true;
ctx.addStringListParameter("alarmStatuses", statusSet.stream().map(AlarmStatus::name).collect(Collectors.toList()));
wherePart.append(" a.status in (:alarmStatuses)");
}
}
String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch());
String mainQuery;
if (!textSearchQuery.isEmpty()) {
mainQuery = selectPart.toString() + fromPart.toString() + wherePart.toString();
mainQuery = String.format("select * from (%s) a %s WHERE %s", mainQuery, joinPart, textSearchQuery);
} else {
mainQuery = selectPart.toString() + fromPart.toString() + joinPart.toString() + wherePart.toString();
}
String countQuery = String.format("select count(*) from (%s) result", mainQuery);
long queryTs = System.currentTimeMillis();
int totalElements;
try {
totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
} finally {
queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - queryTs);
}
if (totalElements == 0) {
return AlarmDataAdapter.createAlarmData(pageLink, Collections.emptyList(), totalElements, orderedEntityIds);
}
String dataQuery = mainQuery + sortPart;
int startIndex = pageLink.getPageSize() * pageLink.getPage();
if (pageLink.getPageSize() > 0) {
dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex);
}
queryTs = System.currentTimeMillis();
List<Map<String, Object>> rows;
try {
rows = jdbcTemplate.queryForList(dataQuery, ctx);
} finally {
queryLog.logQuery(ctx, dataQuery, System.currentTimeMillis() - queryTs);
}
return AlarmDataAdapter.createAlarmData(pageLink, rows, totalElements, orderedEntityIds);
});
}
use of org.thingsboard.server.common.data.query.EntityDataSortOrder in project thingsboard by thingsboard.
the class BaseEntityServiceTest method testFindEntityDataByQueryWithTimeseries.
@Test
public void testFindEntityDataByQueryWithTimeseries() throws ExecutionException, InterruptedException {
List<Device> devices = new ArrayList<>();
List<Double> temperatures = new ArrayList<>();
List<Double> highTemperatures = new ArrayList<>();
for (int i = 0; i < 67; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
devices.add(deviceService.saveDevice(device));
// TO make sure devices have different created time
Thread.sleep(1);
double temperature = (double) (Math.random() * 100.0);
temperatures.add(temperature);
if (temperature > 45.0) {
highTemperatures.add(temperature);
}
}
List<ListenableFuture<Integer>> timeseriesFutures = new ArrayList<>();
for (int i = 0; i < devices.size(); i++) {
Device device = devices.get(i);
timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i)));
}
Futures.successfulAsList(timeseriesFutures).get();
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
List<EntityKey> latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(67, loadedEntities.size());
List<String> loadedTemperatures = new ArrayList<>();
for (Device device : devices) {
loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue());
}
List<String> deviceTemperatures = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
KeyFilter highTemperatureFilter = new KeyFilter();
highTemperatureFilter.setKey(new EntityKey(EntityKeyType.TIME_SERIES, "temperature"));
NumericFilterPredicate predicate = new NumericFilterPredicate();
predicate.setValue(FilterPredicateValue.fromDouble(45));
predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER);
highTemperatureFilter.setPredicate(predicate);
List<KeyFilter> keyFilters = Collections.singletonList(highTemperatureFilter);
query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(highTemperatures.size(), loadedEntities.size());
List<String> loadedHighTemperatures = loadedEntities.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceHighTemperatures = highTemperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
deviceService.deleteDevicesByTenantId(tenantId);
}
use of org.thingsboard.server.common.data.query.EntityDataSortOrder in project thingsboard by thingsboard.
the class BaseEntityServiceTest method testSimpleFindEntityDataByQuery.
@Test
public void testSimpleFindEntityDataByQuery() throws InterruptedException {
List<Device> devices = new ArrayList<>();
for (int i = 0; i < 97; i++) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("Device" + i);
device.setType("default");
device.setLabel("testLabel" + (int) (Math.random() * 1000));
// TO make sure devices have different created time
Thread.sleep(1);
devices.add(deviceService.saveDevice(device));
}
DeviceTypeFilter filter = new DeviceTypeFilter();
filter.setDeviceType("default");
filter.setDeviceNameFilter("");
EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC);
EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
List<EntityKey> entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
PageData<EntityData> data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
Assert.assertEquals(97, data.getTotalElements());
Assert.assertEquals(10, data.getTotalPages());
Assert.assertTrue(data.hasNext());
Assert.assertEquals(10, data.getData().size());
List<EntityData> loadedEntities = new ArrayList<>(data.getData());
while (data.hasNext()) {
query = query.next();
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
loadedEntities.addAll(data.getData());
}
Assert.assertEquals(97, loadedEntities.size());
List<EntityId> loadedIds = loadedEntities.stream().map(EntityData::getEntityId).collect(Collectors.toList());
List<EntityId> deviceIds = devices.stream().map(Device::getId).collect(Collectors.toList());
deviceIds.sort(Comparator.comparing(EntityId::getId));
loadedIds.sort(Comparator.comparing(EntityId::getId));
Assert.assertEquals(deviceIds, loadedIds);
List<String> loadedNames = loadedEntities.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).collect(Collectors.toList());
List<String> deviceNames = devices.stream().map(Device::getName).collect(Collectors.toList());
Collections.sort(loadedNames);
Collections.sort(deviceNames);
Assert.assertEquals(deviceNames, loadedNames);
sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), EntityDataSortOrder.Direction.DESC);
pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder);
query = new EntityDataQuery(filter, pageLink, entityFields, null, null);
data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query);
Assert.assertEquals(11, data.getTotalElements());
Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
deviceService.deleteDevicesByTenantId(tenantId);
}
Aggregations