Search in sources :

Example 1 with Session

use of com.instaclustr.esop.impl.AbstractTracker.Session in project esop by instaclustr.

the class BaseBackupOperationCoordinator method coordinate.

@Override
public void coordinate(final Operation<BackupOperationRequest> operation) {
    final BackupOperationRequest request = operation.request;
    logger.info(request.toString());
    try {
        assert cassandraJMXService != null;
        assert backuperFactoryMap != null;
        assert bucketServiceFactoryMap != null;
        assert objectMapper != null;
        if (!request.skipBucketVerification) {
            try (final BucketService bucketService = bucketServiceFactoryMap.get(request.storageLocation.storageProvider).createBucketService(request)) {
                bucketService.checkBucket(request.storageLocation.bucket, request.createMissingBucket);
            }
        }
        final CassandraData cassandraData = CassandraData.parse(request.dataDirs.get(0));
        cassandraData.setDatabaseEntitiesFromRequest(request.entities);
        final List<String> tokens = new CassandraTokens(cassandraJMXService).act();
        logger.info("Tokens {}", tokens);
        if (!Snapshots.snapshotContainsTimestamp(operation.request.snapshotTag)) {
            if (operation.request.schemaVersion == null) {
                operation.request.schemaVersion = new CassandraSchemaVersion(cassandraJMXService).act();
            }
            operation.request.snapshotTag = resolveSnapshotTag(operation.request, System.currentTimeMillis());
        }
        logger.info("Taking snapshot with name {}", request.snapshotTag);
        new TakeSnapshotOperation(cassandraJMXService, new TakeSnapshotOperationRequest(request.entities, request.snapshotTag), cassandraVersionProvider).run0();
        Snapshots.hashSpec = hashSpec;
        final Snapshots snapshots = Snapshots.parse(request.dataDirs, request.snapshotTag);
        final Optional<Snapshot> snapshot = snapshots.get(request.snapshotTag);
        if (!snapshot.isPresent()) {
            throw new IllegalStateException(format("There is not any snapshot of tag %s", request.snapshotTag));
        }
        final Manifest manifest = Manifest.from(snapshot.get());
        manifest.setSchemaVersion(request.schemaVersion);
        manifest.setTokens(tokens);
        // manifest
        final Path localManifestPath = getLocalManifestPath(request.snapshotTag);
        Manifest.write(manifest, localManifestPath, objectMapper);
        manifest.setManifest(getManifestAsManifestEntry(localManifestPath));
        try (final Backuper backuper = backuperFactoryMap.get(request.storageLocation.storageProvider).createBackuper(request)) {
            final List<ManifestEntry> manifestEntries = manifest.getManifestEntries();
            Session<UploadUnit> uploadSession = null;
            try {
                uploadSession = uploadTracker.submit(backuper, operation, manifestEntries, request.snapshotTag, operation.request.concurrentConnections);
                uploadSession.waitUntilConsideredFinished();
                uploadTracker.cancelIfNecessary(uploadSession);
                final List<UploadUnit> failedUnits = uploadSession.getFailedUnits();
                if (!failedUnits.isEmpty()) {
                    final String message = failedUnits.stream().map(unit -> unit.getManifestEntry().objectKey.toString()).collect(Collectors.joining(","));
                    logger.error(message);
                    throw new IOException(format("Unable to upload some files successfully: %s", message));
                }
            } finally {
                uploadTracker.removeSession(uploadSession);
                uploadSession = null;
            }
            if (operation.request.uploadClusterTopology) {
                // here we will upload all topology because we do not know what restore might look like (what dc a restorer will restore against if any)
                final ClusterTopology topology = new CassandraClusterTopology(cassandraJMXService, null).act();
                ClusterTopology.upload(backuper, topology, objectMapper, operation.request.snapshotTag);
            }
        } finally {
            manifest.cleanup();
        }
    } catch (final Exception ex) {
        operation.addError(Error.from(ex));
    } finally {
        final ClearSnapshotOperation cso = new ClearSnapshotOperation(cassandraJMXService, new ClearSnapshotOperationRequest(request.snapshotTag));
        try {
            cso.run0();
        } catch (final Exception ex) {
            operation.addErrors(cso.errors);
        }
    }
}
Also used : CassandraData(com.instaclustr.esop.impl.CassandraData) ManifestEntry(com.instaclustr.esop.impl.ManifestEntry) Error(com.instaclustr.operations.Operation.Error) Manifest.getManifestAsManifestEntry(com.instaclustr.esop.impl.Manifest.getManifestAsManifestEntry) Provider(javax.inject.Provider) HashSpec(com.instaclustr.esop.impl.hash.HashSpec) OperationCoordinator(com.instaclustr.operations.OperationCoordinator) LoggerFactory(org.slf4j.LoggerFactory) Manifest.getLocalManifestPath(com.instaclustr.esop.impl.Manifest.getLocalManifestPath) BackuperFactory(com.instaclustr.esop.guice.BackuperFactory) Map(java.util.Map) Session(com.instaclustr.esop.impl.AbstractTracker.Session) CassandraClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology) Path(java.nio.file.Path) CassandraData(com.instaclustr.esop.impl.CassandraData) BucketServiceFactory(com.instaclustr.esop.guice.BucketServiceFactory) BucketService(com.instaclustr.esop.impl.BucketService) Backuper(com.instaclustr.esop.impl.backup.Backuper) CassandraSchemaVersion(com.instaclustr.esop.impl.interaction.CassandraSchemaVersion) Logger(org.slf4j.Logger) UploadTracker(com.instaclustr.esop.impl.backup.UploadTracker) Operation(com.instaclustr.operations.Operation) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) IOException(java.io.IOException) Manifest(com.instaclustr.esop.impl.Manifest) Snapshot(com.instaclustr.esop.impl.Snapshots.Snapshot) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ClearSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.ClearSnapshotOperation.ClearSnapshotOperationRequest) List(java.util.List) Snapshots(com.instaclustr.esop.impl.Snapshots) UploadUnit(com.instaclustr.esop.impl.backup.UploadTracker.UploadUnit) TakeSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.TakeSnapshotOperation.TakeSnapshotOperationRequest) CassandraVersion(com.instaclustr.cassandra.CassandraVersion) Optional(java.util.Optional) BackupOperationRequest(com.instaclustr.esop.impl.backup.BackupOperationRequest) CassandraTokens(com.instaclustr.esop.impl.interaction.CassandraTokens) ClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology.ClusterTopology) CassandraJMXService(jmx.org.apache.cassandra.service.CassandraJMXService) ClearSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.ClearSnapshotOperation.ClearSnapshotOperationRequest) TakeSnapshotOperationRequest(com.instaclustr.esop.impl.backup.coordination.TakeSnapshotOperation.TakeSnapshotOperationRequest) CassandraClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology) ClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology.ClusterTopology) CassandraSchemaVersion(com.instaclustr.esop.impl.interaction.CassandraSchemaVersion) UploadUnit(com.instaclustr.esop.impl.backup.UploadTracker.UploadUnit) CassandraClusterTopology(com.instaclustr.esop.topology.CassandraClusterTopology) CassandraTokens(com.instaclustr.esop.impl.interaction.CassandraTokens) Backuper(com.instaclustr.esop.impl.backup.Backuper) Snapshots(com.instaclustr.esop.impl.Snapshots) Manifest.getLocalManifestPath(com.instaclustr.esop.impl.Manifest.getLocalManifestPath) Path(java.nio.file.Path) BucketService(com.instaclustr.esop.impl.BucketService) IOException(java.io.IOException) Manifest(com.instaclustr.esop.impl.Manifest) IOException(java.io.IOException) Snapshot(com.instaclustr.esop.impl.Snapshots.Snapshot) BackupOperationRequest(com.instaclustr.esop.impl.backup.BackupOperationRequest) ManifestEntry(com.instaclustr.esop.impl.ManifestEntry) Manifest.getManifestAsManifestEntry(com.instaclustr.esop.impl.Manifest.getManifestAsManifestEntry)

Example 2 with Session

use of com.instaclustr.esop.impl.AbstractTracker.Session in project esop by instaclustr.

the class AbstractTracker method submit.

public synchronized Session<UNIT> submit(final INTERACTOR interactor, final Operation<? extends REQUEST> operation, final Collection<ManifestEntry> entries, final String snapshotTag, final int concurrentConnections) {
    final Session<UNIT> currentSession = constructSession();
    currentSession.setSnapshotTag(snapshotTag);
    currentSession.setId(operation.id);
    if (entries.isEmpty()) {
        logger.info("0 files to process.");
        return currentSession;
    }
    // we have executor service per request in order to specify maximal
    // concurrent uploads, if we had one global executor, we could not "cap it".
    final ListeningExecutorService executorService = new FixedTasksExecutorSupplier().get(concurrentConnections);
    final Map<ListenableFuture<Void>, Unit> futures = new HashMap<>();
    for (final ManifestEntry entry : entries) {
        UNIT alreadySubmitted = null;
        final Iterator<UNIT> it = Collections.unmodifiableList(new ArrayList<>(units)).iterator();
        while (it.hasNext()) {
            UNIT unit = it.next();
            if (unit.getManifestEntry().objectKey.equals(entry.objectKey)) {
                alreadySubmitted = unit;
                break;
            }
        }
        if (alreadySubmitted == null) {
            final UNIT unit = constructUnitToSubmit(interactor, entry, operation.getShouldCancel(), snapshotTag, hashSpec);
            units.add(unit);
            futures.put(executorService.submit(unit), unit);
            submittedUnits.incrementAndGet();
            currentSession.addUnit(unit);
        } else {
            logger.info(String.format("Session %s skips as already submitted: %s", currentSession.getId(), alreadySubmitted.getManifestEntry().objectKey));
            currentSession.addUnit(alreadySubmitted);
        }
    }
    sessions.add(currentSession);
    submittedSessions.incrementAndGet();
    futures.forEach((key, value) -> key.addListener(() -> {
        synchronized (sessions) {
            // increment finished units across all sessions
            sessions.stream().filter(s -> s.getUnits().contains(value)).forEach(s -> {
                operationsService.operation(s.getId()).ifPresent(op -> {
                    s.finishedUnits.incrementAndGet();
                    logger.debug(String.format("Progress of operation %s: %s", op.id, s.getProgress()));
                    op.progress = s.getProgress();
                });
            });
            units.remove(value);
        }
    }, finisherExecutorService));
    currentSession.setExecutorService(executorService);
    return currentSession;
}
Also used : ListenableFuture(com.google.common.util.concurrent.ListenableFuture) HashSpec(com.instaclustr.esop.impl.hash.HashSpec) OperationRequest(com.instaclustr.operations.OperationRequest) LoggerFactory(org.slf4j.LoggerFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MINUTES(java.util.concurrent.TimeUnit.MINUTES) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) FINISHED(com.instaclustr.esop.impl.AbstractTracker.Unit.State.FINISHED) HashSet(java.util.HashSet) Map(java.util.Map) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) JsonIgnore(com.fasterxml.jackson.annotation.JsonIgnore) Session(com.instaclustr.esop.impl.AbstractTracker.Session) FixedTasksExecutorSupplier(com.instaclustr.threading.Executors.FixedTasksExecutorSupplier) OperationsService(com.instaclustr.operations.OperationsService) Objects(com.google.common.base.Objects) Awaitility.await(org.awaitility.Awaitility.await) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Operation(com.instaclustr.operations.Operation) Collection(java.util.Collection) CANCELLED(com.instaclustr.esop.impl.AbstractTracker.Unit.State.CANCELLED) Set(java.util.Set) UUID(java.util.UUID) FAILED(com.instaclustr.esop.impl.AbstractTracker.Unit.State.FAILED) String.format(java.lang.String.format) AtomicLong(java.util.concurrent.atomic.AtomicLong) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) IGNORED(com.instaclustr.esop.impl.AbstractTracker.Unit.State.IGNORED) Optional(java.util.Optional) NOT_STARTED(com.instaclustr.esop.impl.AbstractTracker.Unit.State.NOT_STARTED) Unit(com.instaclustr.esop.impl.AbstractTracker.Unit) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) FixedTasksExecutorSupplier(com.instaclustr.threading.Executors.FixedTasksExecutorSupplier) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Unit(com.instaclustr.esop.impl.AbstractTracker.Unit)

Aggregations

Session (com.instaclustr.esop.impl.AbstractTracker.Session)2 HashSpec (com.instaclustr.esop.impl.hash.HashSpec)2 Operation (com.instaclustr.operations.Operation)2 String.format (java.lang.String.format)2 List (java.util.List)2 Map (java.util.Map)2 Optional (java.util.Optional)2 Logger (org.slf4j.Logger)2 LoggerFactory (org.slf4j.LoggerFactory)2 JsonIgnore (com.fasterxml.jackson.annotation.JsonIgnore)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 Objects (com.google.common.base.Objects)1 AbstractIdleService (com.google.common.util.concurrent.AbstractIdleService)1 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)1 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)1 CassandraVersion (com.instaclustr.cassandra.CassandraVersion)1 BackuperFactory (com.instaclustr.esop.guice.BackuperFactory)1 BucketServiceFactory (com.instaclustr.esop.guice.BucketServiceFactory)1 Unit (com.instaclustr.esop.impl.AbstractTracker.Unit)1 CANCELLED (com.instaclustr.esop.impl.AbstractTracker.Unit.State.CANCELLED)1