Examples with Unit com.instaclustr.esop.impl.AbstractTracker.Unit used on opensource projects

Example 1 with Unit

use of com.instaclustr.esop.impl.AbstractTracker.Unit in project esop by instaclustr.

the class AbstractTracker method submit.

public synchronized Session<UNIT> submit(final INTERACTOR interactor, final Operation<? extends REQUEST> operation, final Collection<ManifestEntry> entries, final String snapshotTag, final int concurrentConnections) {
    final Session<UNIT> currentSession = constructSession();
    currentSession.setSnapshotTag(snapshotTag);
    currentSession.setId(operation.id);
    if (entries.isEmpty()) {
        logger.info("0 files to process.");
        return currentSession;
    }
    // we have executor service per request in order to specify maximal
    // concurrent uploads, if we had one global executor, we could not "cap it".
    final ListeningExecutorService executorService = new FixedTasksExecutorSupplier().get(concurrentConnections);
    final Map<ListenableFuture<Void>, Unit> futures = new HashMap<>();
    for (final ManifestEntry entry : entries) {
        UNIT alreadySubmitted = null;
        final Iterator<UNIT> it = Collections.unmodifiableList(new ArrayList<>(units)).iterator();
        while (it.hasNext()) {
            UNIT unit = it.next();
            if (unit.getManifestEntry().objectKey.equals(entry.objectKey)) {
                alreadySubmitted = unit;
                break;
            }
        }
        if (alreadySubmitted == null) {
            final UNIT unit = constructUnitToSubmit(interactor, entry, operation.getShouldCancel(), snapshotTag, hashSpec);
            units.add(unit);
            futures.put(executorService.submit(unit), unit);
            submittedUnits.incrementAndGet();
            currentSession.addUnit(unit);
        } else {
            logger.info(String.format("Session %s skips as already submitted: %s", currentSession.getId(), alreadySubmitted.getManifestEntry().objectKey));
            currentSession.addUnit(alreadySubmitted);
        }
    }
    sessions.add(currentSession);
    submittedSessions.incrementAndGet();
    futures.forEach((key, value) -> key.addListener(() -> {
        synchronized (sessions) {
            // increment finished units across all sessions
            sessions.stream().filter(s -> s.getUnits().contains(value)).forEach(s -> {
                operationsService.operation(s.getId()).ifPresent(op -> {
                    s.finishedUnits.incrementAndGet();
                    logger.debug(String.format("Progress of operation %s: %s", op.id, s.getProgress()));
                    op.progress = s.getProgress();
                });
            });
            units.remove(value);
        }
    }, finisherExecutorService));
    currentSession.setExecutorService(executorService);
    return currentSession;
}