Examples with MillisQueryBuilder me.retrodaredevil.solarthing.database.MillisQueryBuilder used on opensource projects

Example 1 with MillisQueryBuilder

use of me.retrodaredevil.solarthing.database.MillisQueryBuilder in project solarthing by wildmountainfarms.

the class SimpleQueryHandler method queryPackets.

/**
 * @param from The date millis from
 * @param to The date millis to
 * @param sourceId The source ID or null. If null, the returned List may contain packet groups from multiple sources
 * @return The resulting packets
 */
private List<? extends InstancePacketGroup> queryPackets(MillisDatabase database, long from, long to, String sourceId) {
    MillisQuery millisQuery = new MillisQueryBuilder().startKey(from).endKey(to).build();
    UniqueQuery uniqueQuery = new UniqueQuery(database, millisQuery);
    final Future<? extends List<? extends PacketGroup>> future;
    {
        // Many times a Grafana dashboard will make many graphql requests with the same from and to parameters.
        // Without this code, each graphql request would result in a separate request to the database.
        // Most of the time, these requests are being executed at the same time.
        // This piece of code takes advantage of the fact that we are requesting the same data at the same time.
        // If we find a Future that is already executing for a given query, we wait for that to complete instead of performing a separate request.
        // This is sort of a caching mechanism, but it's a VERY temporary caching mechanism since data is not kept after is it queried.
        executingQueryMutex.lock();
        var currentFuture = executingQueryMap.get(uniqueQuery);
        if (currentFuture != null) {
            future = currentFuture;
            executingQueryMutex.unlock();
        } else {
            RunnableFuture<? extends List<? extends PacketGroup>> runnableFuture = new FutureTask<>(() -> {
                try {
                    return database.query(millisQuery);
                } catch (SolarThingDatabaseException e) {
                    throw new DatabaseException("Exception querying from " + from + " to " + to, e);
                }
            });
            executingQueryMap.put(uniqueQuery, runnableFuture);
            executingQueryMutex.unlock();
            runnableFuture.run();
            future = runnableFuture;
            executingQueryMutex.lock();
            executingQueryMap.remove(uniqueQuery);
            executingQueryMutex.unlock();
        }
    }
    final List<? extends PacketGroup> rawPacketGroups;
    try {
        rawPacketGroups = future.get();
    } catch (InterruptedException e) {
        throw new DatabaseException("Interrupted!", e);
    } catch (ExecutionException e) {
        Throwable cause = e.getCause();
        if (cause instanceof RuntimeException) {
            throw (RuntimeException) cause;
        }
        throw new DatabaseException("Unknown execution exception", e);
    }
    if (rawPacketGroups.isEmpty()) {
        if (to - from > 60 * 1000) {
            // Only debug this message if the requester is actually asking for a decent chunk of data
            LOGGER.debug("No packets were queried between " + from + " and " + to);
        }
        return Collections.emptyList();
    }
    if (sourceId == null) {
        return PacketGroups.parseToInstancePacketGroups(rawPacketGroups, defaultInstanceOptions);
    }
    Map<String, List<InstancePacketGroup>> map = PacketGroups.parsePackets(rawPacketGroups, defaultInstanceOptions);
    if (map.containsKey(sourceId)) {
        List<InstancePacketGroup> instancePacketGroupList = map.get(sourceId);
        return PacketGroups.orderByFragment(instancePacketGroupList);
    }
    throw new NoSuchElementException("No element with sourceId: '" + sourceId + "' available keys are: " + map.keySet());
}

Example 2 with MillisQueryBuilder

use of me.retrodaredevil.solarthing.database.MillisQueryBuilder in project solarthing by wildmountainfarms.

the class SimpleDatabaseCache method createRecommendedQueryBuilder.

/**
 * Note that the end key will not be null, and will always be set to now. Many times, this is not ideal, and you should set it to null yourself.
 * <p>
 * One of the main reasons that an end key is not always ideal is because a fragment will overwrite its most recent content with data from "now".
 * Many times, that "now" actually ends up being after the end key that is returned here. That can be because of clocks that are slightly off, or because
 * this builder was created too long before actually querying data.
 * @return A builder for the recommended query
 */
public MillisQueryBuilder createRecommendedQueryBuilder() {
    Data data = this.data;
    Instant now = clock.instant();
    long minimumStart = now.minus(minimumDuration).toEpochMilli();
    long end = now.toEpochMilli();
    final long start;
    if (data == null || data.firstQueryStartDateMillis > minimumStart) {
        start = minimumStart;
    } else {
        start = Math.max(minimumStart, data.volatileAfterDateMillis);
    }
    return new MillisQueryBuilder().startKey(start).endKey(end).inclusiveEnd(true);
}

Example 3 with MillisQueryBuilder

use of me.retrodaredevil.solarthing.database.MillisQueryBuilder in project solarthing by wildmountainfarms.

the class CacheHandler method calculatePeriod.

private List<CacheDataPacket> calculatePeriod(long startPeriodNumber, long endPeriodNumber) {
    Instant firstPeriodStart = getPeriodStartFromNumber(startPeriodNumber);
    Instant lastPeriodEnd = getPeriodStartFromNumber(endPeriodNumber).plus(duration);
    Instant queryStart = firstPeriodStart.minus(INFO_DURATION);
    MillisQuery millisQuery = new MillisQueryBuilder().startKey(queryStart.toEpochMilli()).endKey(lastPeriodEnd.toEpochMilli()).inclusiveEnd(false).build();
    final List<? extends PacketGroup> packetGroups;
    try {
        packetGroups = database.getStatusDatabase().query(millisQuery);
    } catch (SolarThingDatabaseException e) {
        // The consumers of this API may be ok if there are holes in the data rather than getting no data at all, so maybe change this later?
        throw new DatabaseException("Couldn't query status packets for period. startPeriodNumber: " + startPeriodNumber + " endPeriodNumber: " + endPeriodNumber + " firstPeriodStart: " + firstPeriodStart, e);
    }
    List<CacheDataPacket> r = new ArrayList<>();
    Map<String, List<InstancePacketGroup>> sourceMap = PacketGroups.parsePackets(packetGroups, defaultInstanceOptions);
    for (Map.Entry<String, List<InstancePacketGroup>> entry : sourceMap.entrySet()) {
        String sourceId = entry.getKey();
        List<InstancePacketGroup> packets = entry.getValue();
        for (long periodNumber = startPeriodNumber; periodNumber <= endPeriodNumber; periodNumber++) {
            Instant periodStart = getPeriodStartFromNumber(periodNumber);
            for (CacheCreator creator : CACHE_CREATORS) {
                r.add(creator.createFrom(sourceId, packets, periodStart, duration));
            }
        }
    }
    return r;
}

Example 4 with MillisQueryBuilder

use of me.retrodaredevil.solarthing.database.MillisQueryBuilder in project solarthing by wildmountainfarms.

the class PVOutputUploadMain method startRangeUpload.

private static int startRangeUpload(SimpleDate fromDate, SimpleDate toDate, PVOutputUploadProgramOptions options, SolarThingDatabase database, PVOutputHandler handler, PVOutputService service, ZoneId zoneId) {
    List<AddOutputParameters> addOutputParameters = new ArrayList<>();
    SimpleDate date = fromDate;
    while (date.compareTo(toDate) <= 0) {
        System.out.println("Doing " + date);
        SimpleDate tomorrow = date.tomorrow();
        long dayStart = date.getDayStartDateMillis(zoneId);
        long dayEnd = tomorrow.getDayStartDateMillis(zoneId);
        List<? extends PacketGroup> rawPacketGroups = null;
        try {
            rawPacketGroups = database.getStatusDatabase().query(new MillisQueryBuilder().startKey(dayStart).endKey(dayEnd).inclusiveEnd(false).build());
            System.out.println("Got " + rawPacketGroups.size() + " packets for date: " + date.toPVOutputString());
        } catch (SolarThingDatabaseException e) {
            e.printStackTrace();
            System.err.println("Couldn't query packets. Skipping " + date.toPVOutputString());
        }
        if (rawPacketGroups != null) {
            List<FragmentedPacketGroup> packetGroups = PacketUtil.getPacketGroups(options.getSourceId(), options.getDefaultInstanceOptions(), rawPacketGroups);
            if (packetGroups != null) {
                if (!handler.checkPackets(dayStart, packetGroups)) {
                    System.err.println("Unsuccessfully checked packets for " + date.toPVOutputString());
                    try {
                        System.out.println(MAPPER.writeValueAsString(packetGroups.get(packetGroups.size() - 1)));
                    } catch (JsonProcessingException e) {
                        e.printStackTrace();
                    }
                } else {
                    AddStatusParameters statusParameters = handler.getStatus(dayStart, packetGroups);
                    AddOutputParametersBuilder outputParametersBuilder = new AddOutputParametersBuilder(statusParameters.getDate()).setGenerated(statusParameters.getEnergyGeneration()).setConsumption(statusParameters.getEnergyConsumption());
                    PVOutputHandler.setImportedExported(outputParametersBuilder, packetGroups, AccumulationConfig.createDefault(dayStart), options.isIncludeImport(), options.isIncludeExport());
                    AddOutputParameters outputParameters = outputParametersBuilder.build();
                    addOutputParameters.add(outputParameters);
                    System.out.println("Added parameters for " + date.toPVOutputString() + " to queue.");
                    System.out.println("Generated: " + statusParameters.getEnergyGeneration());
                    System.out.println(Arrays.toString(outputParameters.toCsvArray()));
                    System.out.println(CsvUtil.toCsvString(outputParameters.toCsvArray()));
                }
            } else {
                System.err.println("Didn't find any packets with source: " + options.getSourceId() + " for date: " + date.toPVOutputString());
            }
        }
        date = tomorrow;
    }
    System.out.println("Going to upload in batches of 30...");
    for (int i = 0; i < addOutputParameters.size(); i += 30) {
        if (i != 0) {
            System.out.println("Sleeping...");
            try {
                // noinspection BusyWait
                Thread.sleep(7000);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                System.err.println("Interrupted");
                return SolarThingConstants.EXIT_CODE_INTERRUPTED;
            }
        }
        int endIndex = Math.min(i + 30, addOutputParameters.size());
        List<AddOutputParameters> parameters = addOutputParameters.subList(i, endIndex);
        System.out.println("Going to upload from " + parameters.get(0).getOutputDate().toPVOutputString() + " to " + parameters.get(parameters.size() - 1).getOutputDate().toPVOutputString());
        AddBatchOutputParameters batchOutputParameters = new ImmutableAddBatchOutputParameters(parameters);
        try {
            LOGGER.debug("Batch Output parameters as JSON: " + MAPPER.writeValueAsString(batchOutputParameters));
        } catch (JsonProcessingException e) {
            LOGGER.error("Got error serializing JSON. This should never happen.", e);
        }
        boolean successful = false;
        for (int j = 0; j < 5; j++) {
            if (j != 0) {
                System.out.println("Sleeping before trying again");
                try {
                    // noinspection BusyWait
                    Thread.sleep(j * 7000);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    System.err.println("Interrupted");
                    return SolarThingConstants.EXIT_CODE_INTERRUPTED;
                }
            }
            Call<String> call = service.addBatchOutput(batchOutputParameters);
            final Response<String> response;
            try {
                response = call.execute();
            } catch (IOException e) {
                e.printStackTrace();
                System.err.println("Error while executing");
                continue;
            }
            if (response.isSuccessful()) {
                System.out.println("Executed successfully. Result: " + response.body());
                successful = true;
                break;
            } else {
                System.err.println("Unsuccessful. Message: " + response.message() + " code: " + response.code());
            }
        }
        if (!successful) {
            System.err.println("All tries were unsuccessful. Ending");
            return SolarThingConstants.EXIT_CODE_FAIL;
        }
    }
    System.out.println("Done!");
    return 0;
}

Example 5 with MillisQueryBuilder

use of me.retrodaredevil.solarthing.database.MillisQueryBuilder in project solarthing by wildmountainfarms.

the class PVOutputUploadMain method startRealTimeProgram.

private static int startRealTimeProgram(PVOutputUploadProgramOptions options, SolarThingDatabase database, PVOutputHandler handler, PVOutputService service, ZoneId zoneId) {
    if (options.isJoinTeams()) {
        LOGGER.info("Going to join SolarThing team...");
        Call<String> call = service.joinTeam(PVOutputConstants.SOLARTHING_TEAM_ID);
        LOGGER.debug("Executing call");
        Response<String> response = null;
        try {
            response = call.execute();
        } catch (IOException e) {
            LOGGER.error("Exception while executing", e);
        }
        if (response != null) {
            int code = response.code();
            String errorBody;
            try {
                ResponseBody responseBody = response.errorBody();
                if (responseBody != null) {
                    errorBody = responseBody.string();
                } else {
                    errorBody = "null";
                }
            } catch (IOException e) {
                e.printStackTrace();
                errorBody = "exception occurred";
            }
            if (code == 200) {
                LOGGER.info("Joined the SolarThing team! Response: " + response.body());
            } else if (code == 400) {
                if (errorBody.contains("already")) {
                    LOGGER.info("Already joined SolarThing team. Response: " + errorBody);
                } else if (errorBody.contains("must have at least")) {
                    LOGGER.info("We will try joining SolarThing team later once we have more outputs. Response: " + errorBody);
                } else {
                    LOGGER.error("Error joining SolarThing team! Response: " + errorBody);
                }
            } else {
                LOGGER.error("Unknown error joining SolarThing team! Response: " + errorBody);
            }
        }
    }
    while (!Thread.currentThread().isInterrupted()) {
        LOGGER.debug("Going to do stuff now.");
        long now = System.currentTimeMillis();
        SimpleDate today = SimpleDate.fromDateMillis(now, zoneId);
        long dayStartTimeMillis = today.getDayStartDateMillis(zoneId);
        List<? extends PacketGroup> rawPacketGroups = null;
        try {
            rawPacketGroups = database.getStatusDatabase().query(new MillisQueryBuilder().startKey(dayStartTimeMillis).endKey(now).build());
            LOGGER.debug("Got packets");
        } catch (SolarThingDatabaseException e) {
            LOGGER.error("Couldn't get status packets", e);
        }
        if (rawPacketGroups != null) {
            List<FragmentedPacketGroup> packetGroups = PacketUtil.getPacketGroups(options.getSourceId(), options.getDefaultInstanceOptions(), rawPacketGroups);
            if (packetGroups != null) {
                FragmentedPacketGroup latestPacketGroup = packetGroups.get(packetGroups.size() - 1);
                if (latestPacketGroup.getDateMillis() < now - 5 * 60 * 1000) {
                    LOGGER.warn("The last packet is more than 5 minutes in the past! now=" + now + " packet date=" + latestPacketGroup.getDateMillis());
                    try {
                        LOGGER.debug("Packets: " + MAPPER.writeValueAsString(latestPacketGroup.getPackets()));
                    } catch (JsonProcessingException e) {
                        LOGGER.warn("Couldn't serialize for some reason", e);
                    }
                } else if (!handler.checkPackets(dayStartTimeMillis, packetGroups)) {
                    LOGGER.warn("Checking packets unsuccessful.");
                } else {
                    AddStatusParameters parameters = handler.getStatus(dayStartTimeMillis, packetGroups);
                    if (uploadStatus(service, parameters) && (options.isIncludeImport() || options.isIncludeExport())) {
                        // only upload output if status is successful
                        AddOutputParametersBuilder outputParametersBuilder = new AddOutputParametersBuilder(parameters.getDate());
                        PVOutputHandler.setImportedExported(outputParametersBuilder, packetGroups, AccumulationConfig.createDefault(dayStartTimeMillis), options.isIncludeImport(), options.isIncludeExport());
                        AddOutputParameters outputParameters = outputParametersBuilder.build();
                        uploadOutput(service, outputParameters);
                    }
                }
            } else {
                LOGGER.warn("Got " + rawPacketGroups.size() + " packets but, there must not have been any packets with the source: " + options.getSourceId());
            }
        }
        LOGGER.debug("Going to sleep now");
        try {
            // noinspection BusyWait
            Thread.sleep(5 * 60 * 1000);
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }
    return SolarThingConstants.EXIT_CODE_INTERRUPTED;
}