Examples with CacheDataPacket me.retrodaredevil.solarthing.type.cache.packets.CacheDataPacket used on opensource projects

Example 1 with CacheDataPacket

use of me.retrodaredevil.solarthing.type.cache.packets.CacheDataPacket in project solarthing by wildmountainfarms.

the class CacheHandler method queryOrCalculateCaches.

private <T extends CacheDataPacket> List<T> queryOrCalculateCaches(TypeReference<T> typeReference, String cacheName, String sourceId, long startPeriodNumber, long endPeriodNumber) {
    // the document IDs needed to return data
    List<String> documentIds = new ArrayList<>();
    // a map from a document ID to a period number
    Map<String, Long> documentIdPeriodNumberMap = new HashMap<>();
    for (long periodNumber = startPeriodNumber; periodNumber <= endPeriodNumber; periodNumber++) {
        Instant periodStart = getPeriodStartFromNumber(periodNumber);
        String documentId = CacheUtil.getDocumentId(periodStart, duration, sourceId, cacheName);
        documentIds.add(documentId);
        documentIdPeriodNumberMap.put(documentId, periodNumber);
    }
    BulkGetRequest request = BulkGetRequest.from(documentIds);
    final BulkGetResponse response;
    try {
        response = cacheDatabase.getDocumentsBulk(request);
    } catch (CouchDbException e) {
        throw new DatabaseException("CouchDB exception | message: " + e.getMessage(), e);
    }
    // map for documents that need to be updated. The value represents the revision that needs to be used to update it
    Map<String, String> documentIdRevisionMapForUpdate = new HashMap<>();
    // Map for period number -> cached data. This helps us make sure we only return a single piece of data for each period
    Map<Long, T> periodNumberPacketMap = new TreeMap<>();
    // Set for document IDs that we already have and do not need to be updated
    Set<String> doNotUpdateDocumentIdsSet = new HashSet<>();
    Long queryStartPeriodNumber = null;
    Long queryEndPeriodNumber = null;
    for (BulkGetResponse.Result result : response.getResults()) {
        if (result.hasConflicts()) {
            // replication going on with their databases.
            throw new UnexpectedResponseException("cache document with conflict! doc id: " + result.getDocumentId());
        }
        Long periodNumber = documentIdPeriodNumberMap.get(result.getDocumentId());
        if (periodNumber == null) {
            throw new IllegalStateException("Could not get period number for doc id: " + result.getDocumentId() + ". This should never happen.");
        }
        T value = null;
        if (!result.isError()) {
            JsonData jsonData = result.getJsonDataAssertNotConflicted();
            try {
                value = CouchDbJacksonUtil.readValue(mapper, jsonData, typeReference);
                if (value.getSourceId().equals(sourceId) && value.getCacheName().equals(cacheName)) {
                    periodNumberPacketMap.put(periodNumber, value);
                }
                doNotUpdateDocumentIdsSet.add(value.getDbId());
            } catch (JsonProcessingException ex) {
                // If we are in this catch block, one of two things has happened:
                // The JSON is invalid (unlikely), we updated how a given cache is serialized/deserialized, or we're dumb and never tested to see if the deserialization works.
                // If the JSON is actually invalid, getRevisionFromJsonData will throw an exception. Otherwise, we need to calculate the given cache again
                String revision = getRevisionFromJsonData(jsonData);
                documentIdRevisionMapForUpdate.put(result.getDocumentId(), revision);
            }
        }
        if (value == null) {
            if (queryStartPeriodNumber == null) {
                queryStartPeriodNumber = periodNumber;
                queryEndPeriodNumber = periodNumber;
            } else {
                queryStartPeriodNumber = Math.min(queryStartPeriodNumber, periodNumber);
                queryEndPeriodNumber = Math.max(queryEndPeriodNumber, periodNumber);
            }
        }
    }
    if (queryStartPeriodNumber != null) {
        List<CacheDataPacket> calculatedPackets = calculatePeriod(queryStartPeriodNumber, queryEndPeriodNumber);
        List<JsonData> calculatedPacketsJsonDataList = new ArrayList<>();
        int updateAttemptCount = 0;
        for (CacheDataPacket packet : calculatedPackets) {
            if (doNotUpdateDocumentIdsSet.contains(packet.getDbId())) {
                continue;
            }
            if (!sourceId.equals(packet.getSourceId()) || !cacheName.equals(packet.getCacheName())) {
                // This is the same with different cache names, we don't know which ones we need to update and which ones we cannot
                continue;
            }
            JsonData json;
            try {
                String revision = documentIdRevisionMapForUpdate.get(packet.getDbId());
                if (revision == null) {
                    json = new StringJsonData(mapper.writeValueAsString(packet));
                } else {
                    json = new StringJsonData(mapper.writeValueAsString(new DocumentRevisionWrapper(revision, packet)));
                    updateAttemptCount++;
                }
            } catch (JsonProcessingException e) {
                throw new RuntimeException("Should be able to serialize!", e);
            }
            calculatedPacketsJsonDataList.add(json);
        }
        final List<BulkDocumentResponse> postResponse;
        try {
            postResponse = cacheDatabase.postDocumentsBulk(new BulkPostRequest(calculatedPacketsJsonDataList));
        } catch (CouchDbException e) {
            throw new DatabaseException("Could not update cache", e);
        }
        int successCount = 0;
        int failCount = 0;
        for (BulkDocumentResponse documentResponse : postResponse) {
            if (documentResponse.isOk()) {
                successCount++;
            } else {
                failCount++;
                LOGGER.info("Error: " + documentResponse.getError() + " reason: " + documentResponse.getReason() + " on id: " + documentResponse.getId());
            }
        }
        LOGGER.debug("(Cache updating) Success: " + successCount + " fail: " + failCount + ". Tried to update: " + updateAttemptCount);
        int numberOfWantedType = 0;
        for (CacheDataPacket cacheDataPacket : calculatedPackets) {
            if (cacheDataPacket.getSourceId().equals(sourceId) && cacheDataPacket.getCacheName().equals(cacheName)) {
                @SuppressWarnings("unchecked") T packet = (T) cacheDataPacket;
                Long periodNumber = documentIdPeriodNumberMap.get(cacheDataPacket.getDbId());
                if (periodNumber == null) {
                    throw new NullPointerException("No period number for id: " + cacheDataPacket.getDbId());
                }
                periodNumberPacketMap.put(periodNumber, packet);
                numberOfWantedType++;
            }
        }
        LOGGER.debug("Calculated " + calculatedPackets.size() + " and " + numberOfWantedType + " were of type " + cacheName);
    } else {
        LOGGER.trace("Didn't have to get any data");
    }
    return new ArrayList<>(periodNumberPacketMap.values());
}

Example 2 with CacheDataPacket

use of me.retrodaredevil.solarthing.type.cache.packets.CacheDataPacket in project solarthing by wildmountainfarms.

the class CacheHandler method calculatePeriod.

private List<CacheDataPacket> calculatePeriod(long startPeriodNumber, long endPeriodNumber) {
    Instant firstPeriodStart = getPeriodStartFromNumber(startPeriodNumber);
    Instant lastPeriodEnd = getPeriodStartFromNumber(endPeriodNumber).plus(duration);
    Instant queryStart = firstPeriodStart.minus(INFO_DURATION);
    MillisQuery millisQuery = new MillisQueryBuilder().startKey(queryStart.toEpochMilli()).endKey(lastPeriodEnd.toEpochMilli()).inclusiveEnd(false).build();
    final List<? extends PacketGroup> packetGroups;
    try {
        packetGroups = database.getStatusDatabase().query(millisQuery);
    } catch (SolarThingDatabaseException e) {
        // The consumers of this API may be ok if there are holes in the data rather than getting no data at all, so maybe change this later?
        throw new DatabaseException("Couldn't query status packets for period. startPeriodNumber: " + startPeriodNumber + " endPeriodNumber: " + endPeriodNumber + " firstPeriodStart: " + firstPeriodStart, e);
    }
    List<CacheDataPacket> r = new ArrayList<>();
    Map<String, List<InstancePacketGroup>> sourceMap = PacketGroups.parsePackets(packetGroups, defaultInstanceOptions);
    for (Map.Entry<String, List<InstancePacketGroup>> entry : sourceMap.entrySet()) {
        String sourceId = entry.getKey();
        List<InstancePacketGroup> packets = entry.getValue();
        for (long periodNumber = startPeriodNumber; periodNumber <= endPeriodNumber; periodNumber++) {
            Instant periodStart = getPeriodStartFromNumber(periodNumber);
            for (CacheCreator creator : CACHE_CREATORS) {
                r.add(creator.createFrom(sourceId, packets, periodStart, duration));
            }
        }
    }
    return r;
}