Examples with TrackingKey org.commonjava.indy.folo.model.TrackingKey used on opensource projects

Example 6 with TrackingKey

use of org.commonjava.indy.folo.model.TrackingKey in project indy by Commonjava.

the class FoloRecordCassandra method transformDtxTrackingRecordToTrackingContent.

private TrackedContent transformDtxTrackingRecordToTrackingContent(TrackingKey trackingKey, List<DtxTrackingRecord> trackingRecords) {
    List<TrackedContentEntry> records = new ArrayList<>();
    for (DtxTrackingRecord record : trackingRecords) {
        records.add(DtxTrackingRecord.toTrackingContentEntry(record));
    }
    Set<TrackedContentEntry> uploads = records.stream().filter(record -> record.getEffect().toString().equals(UPLOADS)).collect(Collectors.toSet());
    // logger.warn("-- Processing {} uploads  from  tracking key {} " ,  uploads.size() ,  trackingKey);
    Set<TrackedContentEntry> downloads = records.stream().filter(record -> record.getEffect().toString().equals(DOWNLOADS)).collect(Collectors.toSet());
    // logger.warn("-- Processing {} downloads  from  tracking key {} " ,  downloads.size() ,  trackingKey);
    TrackedContent trackedContent = new TrackedContent(trackingKey, uploads, downloads);
    return trackedContent;
}

Example 7 with TrackingKey

use of org.commonjava.indy.folo.model.TrackingKey in project indy by Commonjava.

the class FoloRecordCassandra method getSealed.

// This may fail given a huge dataset (oom). Only used for test purpose !
@Override
public Set<TrackedContent> getSealed() {
    Set<TrackedContent> trackedContents = new HashSet<>();
    Set<TrackingKey> sealedTrackingKeys = getSealedTrackingKey();
    for (TrackingKey trackingKey : sealedTrackingKeys) {
        List<DtxTrackingRecord> dtxTrackingRecordsFromDb = getDtxTrackingRecordsFromDb(trackingKey);
        TrackedContent trackedContent = transformDtxTrackingRecordToTrackingContent(trackingKey, dtxTrackingRecordsFromDb);
        trackedContents.add(trackedContent);
    }
    return trackedContents;
}

Example 8 with TrackingKey

use of org.commonjava.indy.folo.model.TrackingKey in project indy by Commonjava.

the class FoloRecordCassandra method saveTrackedContentRecords.

private void saveTrackedContentRecords(TrackedContent record) {
    Set<TrackedContentEntry> downloads = record.getDownloads();
    Set<TrackedContentEntry> uploads = record.getUploads();
    TrackingKey key = record.getKey();
    for (TrackedContentEntry downloadEntry : downloads) {
        DtxTrackingRecord downloadRecord = DtxTrackingRecord.fromTrackedContentEntry(downloadEntry, true);
        trackingMapper.save(downloadRecord);
    }
    for (TrackedContentEntry uploadEntry : uploads) {
        DtxTrackingRecord uploadRecord = DtxTrackingRecord.fromTrackedContentEntry(uploadEntry, true);
        trackingMapper.save(uploadRecord);
    }
}

Example 9 with TrackingKey

use of org.commonjava.indy.folo.model.TrackingKey in project indy by Commonjava.

the class FoloISPN2CassandraMigrationAction method amendTrackingKeyAndEffect.

private void amendTrackingKeyAndEffect(TrackedContent item) {
    TrackingKey key = item.getKey();
    Set<TrackedContentEntry> uploads = item.getUploads();
    if (uploads != null) {
        uploads.forEach(up -> {
            if (up.getTrackingKey() == null) {
                up.setTrackingKey(key);
            }
            if (up.getEffect() == null) {
                up.setEffect(StoreEffect.UPLOAD);
            }
        });
    }
    Set<TrackedContentEntry> downloads = item.getDownloads();
    if (downloads != null) {
        downloads.forEach(down -> {
            if (down.getTrackingKey() == null) {
                down.setTrackingKey(key);
            }
            if (down.getEffect() == null) {
                down.setEffect(StoreEffect.DOWNLOAD);
            }
        });
    }
}

Example 10 with TrackingKey

use of org.commonjava.indy.folo.model.TrackingKey in project indy by Commonjava.

the class FoloISPN2CassandraMigrationAction method migrate.

public boolean migrate() {
    if (!STORAGE_CASSANDRA.equals(durableConfig.getFoloStorage())) {
        logger.info("Skip the migration if the storage is not cassandra. ");
        return true;
    }
    if (started) {
        logger.info("Migration is already started. ");
        return true;
    }
    logger.info("Migrate folo records from ISPN to cassandra start");
    AtomicInteger count = new AtomicInteger(0);
    AtomicInteger skippedCount = new AtomicInteger(0);
    Map<String, String> failed = new HashMap();
    // to hold completed keys
    Set<String> completed = new HashSet<>();
    try {
        started = true;
        Set<String> prevCompleted = loadPrevCompleted();
        Set<TrackingKey> keySet = cacheRecord.getSealedTrackingKey();
        logger.info("Get total records size: {}", keySet.size());
        keySet.forEach(key -> {
            if (!prevCompleted.contains(key.getId())) {
                migrateForKey(key, count, completed, failed);
            } else {
                int skipped = skippedCount.incrementAndGet();
                if (skipped % 100 == 0) {
                    logger.info("Skipped: {}", skipped);
                }
            }
        });
        logger.info("{}", count.get());
        logger.info("Migrate folo records from ISPN to cassandra done. Failed: {}, Skipped: {}", failed.size(), skippedCount.get());
    } catch (IOException e) {
        logger.error("Migration failed", e);
    } finally {
        started = false;
        dumpResult(completed, failed);
    }
    return true;
}