Examples with MillisQuery me.retrodaredevil.solarthing.database.MillisQuery used on opensource projects

Example 1 with MillisQuery

use of me.retrodaredevil.solarthing.database.MillisQuery in project solarthing by wildmountainfarms.

the class SimpleQueryHandler method queryPackets.

/**
 * @param from The date millis from
 * @param to The date millis to
 * @param sourceId The source ID or null. If null, the returned List may contain packet groups from multiple sources
 * @return The resulting packets
 */
private List<? extends InstancePacketGroup> queryPackets(MillisDatabase database, long from, long to, String sourceId) {
    MillisQuery millisQuery = new MillisQueryBuilder().startKey(from).endKey(to).build();
    UniqueQuery uniqueQuery = new UniqueQuery(database, millisQuery);
    final Future<? extends List<? extends PacketGroup>> future;
    {
        // Many times a Grafana dashboard will make many graphql requests with the same from and to parameters.
        // Without this code, each graphql request would result in a separate request to the database.
        // Most of the time, these requests are being executed at the same time.
        // This piece of code takes advantage of the fact that we are requesting the same data at the same time.
        // If we find a Future that is already executing for a given query, we wait for that to complete instead of performing a separate request.
        // This is sort of a caching mechanism, but it's a VERY temporary caching mechanism since data is not kept after is it queried.
        executingQueryMutex.lock();
        var currentFuture = executingQueryMap.get(uniqueQuery);
        if (currentFuture != null) {
            future = currentFuture;
            executingQueryMutex.unlock();
        } else {
            RunnableFuture<? extends List<? extends PacketGroup>> runnableFuture = new FutureTask<>(() -> {
                try {
                    return database.query(millisQuery);
                } catch (SolarThingDatabaseException e) {
                    throw new DatabaseException("Exception querying from " + from + " to " + to, e);
                }
            });
            executingQueryMap.put(uniqueQuery, runnableFuture);
            executingQueryMutex.unlock();
            runnableFuture.run();
            future = runnableFuture;
            executingQueryMutex.lock();
            executingQueryMap.remove(uniqueQuery);
            executingQueryMutex.unlock();
        }
    }
    final List<? extends PacketGroup> rawPacketGroups;
    try {
        rawPacketGroups = future.get();
    } catch (InterruptedException e) {
        throw new DatabaseException("Interrupted!", e);
    } catch (ExecutionException e) {
        Throwable cause = e.getCause();
        if (cause instanceof RuntimeException) {
            throw (RuntimeException) cause;
        }
        throw new DatabaseException("Unknown execution exception", e);
    }
    if (rawPacketGroups.isEmpty()) {
        if (to - from > 60 * 1000) {
            // Only debug this message if the requester is actually asking for a decent chunk of data
            LOGGER.debug("No packets were queried between " + from + " and " + to);
        }
        return Collections.emptyList();
    }
    if (sourceId == null) {
        return PacketGroups.parseToInstancePacketGroups(rawPacketGroups, defaultInstanceOptions);
    }
    Map<String, List<InstancePacketGroup>> map = PacketGroups.parsePackets(rawPacketGroups, defaultInstanceOptions);
    if (map.containsKey(sourceId)) {
        List<InstancePacketGroup> instancePacketGroupList = map.get(sourceId);
        return PacketGroups.orderByFragment(instancePacketGroupList);
    }
    throw new NoSuchElementException("No element with sourceId: '" + sourceId + "' available keys are: " + map.keySet());
}

Example 2 with MillisQuery

use of me.retrodaredevil.solarthing.database.MillisQuery in project solarthing by wildmountainfarms.

the class DatabaseCacheTest method test.

@Test
void test() {
    Instant instant = Instant.now();
    Clock clock = Clock.fixed(instant, ZoneOffset.UTC);
    SimpleDatabaseCache cache = SimpleDatabaseCache.createDefault(clock);
    assertTrue(cache.getAllCachedPackets().isEmpty());
    {
        MillisQuery firstRecommendedQuery = cache.getRecommendedQuery();
        assertEquals(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).toEpochMilli(), firstRecommendedQuery.getStartKey());
        assertEquals(instant.toEpochMilli(), firstRecommendedQuery.getEndKey());
        cache.feed(Arrays.asList(create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(30)), create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(40)), create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(50)), create(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).minusSeconds(20)), create(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(20)), create(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(40))), firstRecommendedQuery.getStartKey(), firstRecommendedQuery.getEndKey());
    }
    assertEquals(6, cache.getAllCachedPackets().size());
    assertEquals(6, cache.getCachedPacketsInRange(TimeRange.create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(30), instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(40)), false).size());
    assertEquals(5, cache.getCachedPacketsInRange(TimeRange.create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(31), instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(40)), false).size());
    assertEquals(4, cache.getCachedPacketsInRange(TimeRange.create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(31), instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(39)), false).size());
    {
        MillisQuery millisQuery = cache.getRecommendedQuery();
        assertEquals(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).toEpochMilli(), millisQuery.getStartKey());
        assertEquals(instant.toEpochMilli(), millisQuery.getEndKey());
        cache.feed(Arrays.asList(create(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).minusSeconds(20)), create(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(20)), create(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(41)), create(instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(50))), millisQuery.getStartKey(), millisQuery.getEndKey());
    }
    assertEquals(7, cache.getAllCachedPackets().size());
    assertEquals(5, cache.getCachedPacketsInRange(TimeRange.create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(30), instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(40)), false).size());
    assertEquals(6, cache.getCachedPacketsInRange(TimeRange.create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(30), instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(41)), false).size());
    assertEquals(7, cache.getCachedPacketsInRange(TimeRange.create(instant.minus(SimpleDatabaseCache.DEFAULT_MINIMUM_DURATION).plusSeconds(30), instant.minus(SimpleDatabaseCache.DEFAULT_VOLATILE_WINDOW_DURATION).plusSeconds(50)), false).size());
}

Example 3 with MillisQuery

use of me.retrodaredevil.solarthing.database.MillisQuery in project solarthing by wildmountainfarms.

the class CacheHandler method calculatePeriod.

private List<CacheDataPacket> calculatePeriod(long startPeriodNumber, long endPeriodNumber) {
    Instant firstPeriodStart = getPeriodStartFromNumber(startPeriodNumber);
    Instant lastPeriodEnd = getPeriodStartFromNumber(endPeriodNumber).plus(duration);
    Instant queryStart = firstPeriodStart.minus(INFO_DURATION);
    MillisQuery millisQuery = new MillisQueryBuilder().startKey(queryStart.toEpochMilli()).endKey(lastPeriodEnd.toEpochMilli()).inclusiveEnd(false).build();
    final List<? extends PacketGroup> packetGroups;
    try {
        packetGroups = database.getStatusDatabase().query(millisQuery);
    } catch (SolarThingDatabaseException e) {
        // The consumers of this API may be ok if there are holes in the data rather than getting no data at all, so maybe change this later?
        throw new DatabaseException("Couldn't query status packets for period. startPeriodNumber: " + startPeriodNumber + " endPeriodNumber: " + endPeriodNumber + " firstPeriodStart: " + firstPeriodStart, e);
    }
    List<CacheDataPacket> r = new ArrayList<>();
    Map<String, List<InstancePacketGroup>> sourceMap = PacketGroups.parsePackets(packetGroups, defaultInstanceOptions);
    for (Map.Entry<String, List<InstancePacketGroup>> entry : sourceMap.entrySet()) {
        String sourceId = entry.getKey();
        List<InstancePacketGroup> packets = entry.getValue();
        for (long periodNumber = startPeriodNumber; periodNumber <= endPeriodNumber; periodNumber++) {
            Instant periodStart = getPeriodStartFromNumber(periodNumber);
            for (CacheCreator creator : CACHE_CREATORS) {
                r.add(creator.createFrom(sourceId, packets, periodStart, duration));
            }
        }
    }
    return r;
}