Examples with BulkGetRequest me.retrodaredevil.couchdbjava.request.BulkGetRequest used on opensource projects

Example 1 with BulkGetRequest

use of me.retrodaredevil.couchdbjava.request.BulkGetRequest in project solarthing by wildmountainfarms.

the class CacheHandler method queryOrCalculateCaches.

private <T extends CacheDataPacket> List<T> queryOrCalculateCaches(TypeReference<T> typeReference, String cacheName, String sourceId, long startPeriodNumber, long endPeriodNumber) {
    // the document IDs needed to return data
    List<String> documentIds = new ArrayList<>();
    // a map from a document ID to a period number
    Map<String, Long> documentIdPeriodNumberMap = new HashMap<>();
    for (long periodNumber = startPeriodNumber; periodNumber <= endPeriodNumber; periodNumber++) {
        Instant periodStart = getPeriodStartFromNumber(periodNumber);
        String documentId = CacheUtil.getDocumentId(periodStart, duration, sourceId, cacheName);
        documentIds.add(documentId);
        documentIdPeriodNumberMap.put(documentId, periodNumber);
    }
    BulkGetRequest request = BulkGetRequest.from(documentIds);
    final BulkGetResponse response;
    try {
        response = cacheDatabase.getDocumentsBulk(request);
    } catch (CouchDbException e) {
        throw new DatabaseException("CouchDB exception | message: " + e.getMessage(), e);
    }
    // map for documents that need to be updated. The value represents the revision that needs to be used to update it
    Map<String, String> documentIdRevisionMapForUpdate = new HashMap<>();
    // Map for period number -> cached data. This helps us make sure we only return a single piece of data for each period
    Map<Long, T> periodNumberPacketMap = new TreeMap<>();
    // Set for document IDs that we already have and do not need to be updated
    Set<String> doNotUpdateDocumentIdsSet = new HashSet<>();
    Long queryStartPeriodNumber = null;
    Long queryEndPeriodNumber = null;
    for (BulkGetResponse.Result result : response.getResults()) {
        if (result.hasConflicts()) {
            // replication going on with their databases.
            throw new UnexpectedResponseException("cache document with conflict! doc id: " + result.getDocumentId());
        }
        Long periodNumber = documentIdPeriodNumberMap.get(result.getDocumentId());
        if (periodNumber == null) {
            throw new IllegalStateException("Could not get period number for doc id: " + result.getDocumentId() + ". This should never happen.");
        }
        T value = null;
        if (!result.isError()) {
            JsonData jsonData = result.getJsonDataAssertNotConflicted();
            try {
                value = CouchDbJacksonUtil.readValue(mapper, jsonData, typeReference);
                if (value.getSourceId().equals(sourceId) && value.getCacheName().equals(cacheName)) {
                    periodNumberPacketMap.put(periodNumber, value);
                }
                doNotUpdateDocumentIdsSet.add(value.getDbId());
            } catch (JsonProcessingException ex) {
                // If we are in this catch block, one of two things has happened:
                // The JSON is invalid (unlikely), we updated how a given cache is serialized/deserialized, or we're dumb and never tested to see if the deserialization works.
                // If the JSON is actually invalid, getRevisionFromJsonData will throw an exception. Otherwise, we need to calculate the given cache again
                String revision = getRevisionFromJsonData(jsonData);
                documentIdRevisionMapForUpdate.put(result.getDocumentId(), revision);
            }
        }
        if (value == null) {
            if (queryStartPeriodNumber == null) {
                queryStartPeriodNumber = periodNumber;
                queryEndPeriodNumber = periodNumber;
            } else {
                queryStartPeriodNumber = Math.min(queryStartPeriodNumber, periodNumber);
                queryEndPeriodNumber = Math.max(queryEndPeriodNumber, periodNumber);
            }
        }
    }
    if (queryStartPeriodNumber != null) {
        List<CacheDataPacket> calculatedPackets = calculatePeriod(queryStartPeriodNumber, queryEndPeriodNumber);
        List<JsonData> calculatedPacketsJsonDataList = new ArrayList<>();
        int updateAttemptCount = 0;
        for (CacheDataPacket packet : calculatedPackets) {
            if (doNotUpdateDocumentIdsSet.contains(packet.getDbId())) {
                continue;
            }
            if (!sourceId.equals(packet.getSourceId()) || !cacheName.equals(packet.getCacheName())) {
                // This is the same with different cache names, we don't know which ones we need to update and which ones we cannot
                continue;
            }
            JsonData json;
            try {
                String revision = documentIdRevisionMapForUpdate.get(packet.getDbId());
                if (revision == null) {
                    json = new StringJsonData(mapper.writeValueAsString(packet));
                } else {
                    json = new StringJsonData(mapper.writeValueAsString(new DocumentRevisionWrapper(revision, packet)));
                    updateAttemptCount++;
                }
            } catch (JsonProcessingException e) {
                throw new RuntimeException("Should be able to serialize!", e);
            }
            calculatedPacketsJsonDataList.add(json);
        }
        final List<BulkDocumentResponse> postResponse;
        try {
            postResponse = cacheDatabase.postDocumentsBulk(new BulkPostRequest(calculatedPacketsJsonDataList));
        } catch (CouchDbException e) {
            throw new DatabaseException("Could not update cache", e);
        }
        int successCount = 0;
        int failCount = 0;
        for (BulkDocumentResponse documentResponse : postResponse) {
            if (documentResponse.isOk()) {
                successCount++;
            } else {
                failCount++;
                LOGGER.info("Error: " + documentResponse.getError() + " reason: " + documentResponse.getReason() + " on id: " + documentResponse.getId());
            }
        }
        LOGGER.debug("(Cache updating) Success: " + successCount + " fail: " + failCount + ". Tried to update: " + updateAttemptCount);
        int numberOfWantedType = 0;
        for (CacheDataPacket cacheDataPacket : calculatedPackets) {
            if (cacheDataPacket.getSourceId().equals(sourceId) && cacheDataPacket.getCacheName().equals(cacheName)) {
                @SuppressWarnings("unchecked") T packet = (T) cacheDataPacket;
                Long periodNumber = documentIdPeriodNumberMap.get(cacheDataPacket.getDbId());
                if (periodNumber == null) {
                    throw new NullPointerException("No period number for id: " + cacheDataPacket.getDbId());
                }
                periodNumberPacketMap.put(periodNumber, packet);
                numberOfWantedType++;
            }
        }
        LOGGER.debug("Calculated " + calculatedPackets.size() + " and " + numberOfWantedType + " were of type " + cacheName);
    } else {
        LOGGER.trace("Didn't have to get any data");
    }
    return new ArrayList<>(periodNumberPacketMap.values());
}