Examples with RecoveryManager org.apache.accumulo.manager.recovery.RecoveryManager used on opensource projects

Example 1 with RecoveryManager

use of org.apache.accumulo.manager.recovery.RecoveryManager in project accumulo by apache.

the class Manager method run.

@Override
public void run() {
    final ServerContext context = getContext();
    final String zroot = getZooKeeperRoot();
    // ACCUMULO-4424 Put up the Thrift servers before getting the lock as a sign of process health
    // when a hot-standby
    // 
    // Start the Manager's Client service
    clientHandler = new ManagerClientServiceHandler(this);
    // Ensure that calls before the manager gets the lock fail
    Iface haProxy = HighlyAvailableServiceWrapper.service(clientHandler, this);
    Iface rpcProxy = TraceUtil.wrapService(haProxy);
    final Processor<Iface> processor;
    if (context.getThriftServerType() == ThriftServerType.SASL) {
        Iface tcredsProxy = TCredentialsUpdatingWrapper.service(rpcProxy, clientHandler.getClass(), getConfiguration());
        processor = new Processor<>(tcredsProxy);
    } else {
        processor = new Processor<>(rpcProxy);
    }
    ServerAddress sa;
    try {
        sa = TServerUtils.startServer(context, getHostname(), Property.MANAGER_CLIENTPORT, processor, "Manager", "Manager Client Service Handler", null, Property.MANAGER_MINTHREADS, Property.MANAGER_MINTHREADS_TIMEOUT, Property.MANAGER_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
    } catch (UnknownHostException e) {
        throw new IllegalStateException("Unable to start server on host " + getHostname(), e);
    }
    clientService = sa.server;
    log.info("Started Manager client service at {}", sa.address);
    // block until we can obtain the ZK lock for the manager
    try {
        getManagerLock(ServiceLock.path(zroot + Constants.ZMANAGER_LOCK));
    } catch (KeeperException | InterruptedException e) {
        throw new IllegalStateException("Exception getting manager lock", e);
    }
    // upgrading.
    if (upgradeCoordinator.getStatus() != UpgradeCoordinator.UpgradeStatus.COMPLETE) {
        managerUpgrading.set(true);
    }
    try {
        MetricsUtil.initializeMetrics(getContext().getConfiguration(), this.applicationName, sa.getAddress());
        ManagerMetrics.init(getConfiguration(), this);
    } catch (Exception e1) {
        log.error("Error initializing metrics, metrics will not be emitted.", e1);
    }
    recoveryManager = new RecoveryManager(this, TIME_TO_CACHE_RECOVERY_WAL_EXISTENCE);
    context.getTableManager().addObserver(this);
    Thread statusThread = Threads.createThread("Status Thread", new StatusThread());
    statusThread.start();
    Threads.createThread("Migration Cleanup Thread", new MigrationCleanupThread()).start();
    tserverSet.startListeningForTabletServerChanges();
    try {
        blockForTservers();
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    ZooReaderWriter zReaderWriter = context.getZooReaderWriter();
    try {
        zReaderWriter.getChildren(zroot + Constants.ZRECOVERY, new Watcher() {

            @Override
            public void process(WatchedEvent event) {
                nextEvent.event("Noticed recovery changes %s", event.getType());
                try {
                    // watcher only fires once, add it back
                    zReaderWriter.getChildren(zroot + Constants.ZRECOVERY, this);
                } catch (Exception e) {
                    log.error("Failed to add log recovery watcher back", e);
                }
            }
        });
    } catch (KeeperException | InterruptedException e) {
        throw new IllegalStateException("Unable to read " + zroot + Constants.ZRECOVERY, e);
    }
    watchers.add(new TabletGroupWatcher(this, TabletStateStore.getStoreForLevel(DataLevel.USER, context, this), null) {

        @Override
        boolean canSuspendTablets() {
            // Always allow user data tablets to enter suspended state.
            return true;
        }
    });
    watchers.add(new TabletGroupWatcher(this, TabletStateStore.getStoreForLevel(DataLevel.METADATA, context, this), watchers.get(0)) {

        @Override
        boolean canSuspendTablets() {
            // setting.
            return getConfiguration().getBoolean(Property.MANAGER_METADATA_SUSPENDABLE);
        }
    });
    watchers.add(new TabletGroupWatcher(this, TabletStateStore.getStoreForLevel(DataLevel.ROOT, context), watchers.get(1)) {

        @Override
        boolean canSuspendTablets() {
            // Never allow root tablet to enter suspended state.
            return false;
        }
    });
    for (TabletGroupWatcher watcher : watchers) {
        watcher.start();
    }
    // Once we are sure the upgrade is complete, we can safely allow fate use.
    try {
        // wait for metadata upgrade running in background to complete
        if (null != upgradeMetadataFuture) {
            upgradeMetadataFuture.get();
        }
        // Everything is fully upgraded by this point.
        managerUpgrading.set(false);
    } catch (ExecutionException | InterruptedException e) {
        throw new IllegalStateException("Metadata upgrade failed", e);
    }
    try {
        final AgeOffStore<Manager> store = new AgeOffStore<>(new org.apache.accumulo.fate.ZooStore<>(getZooKeeperRoot() + Constants.ZFATE, context.getZooReaderWriter()), TimeUnit.HOURS.toMillis(8), System::currentTimeMillis);
        fate = new Fate<>(this, store, TraceRepo::toLogString);
        fate.startTransactionRunners(getConfiguration());
        context.getScheduledExecutor().scheduleWithFixedDelay(store::ageOff, 63000, 63000, TimeUnit.MILLISECONDS);
    } catch (KeeperException | InterruptedException e) {
        throw new IllegalStateException("Exception setting up FaTE cleanup thread", e);
    }
    initializeZkForReplication(zReaderWriter, zroot);
    // Make sure that we have a secret key (either a new one or an old one from ZK) before we start
    // the manager client service.
    Thread authenticationTokenKeyManagerThread = null;
    if (authenticationTokenKeyManager != null && keyDistributor != null) {
        log.info("Starting delegation-token key manager");
        try {
            keyDistributor.initialize();
        } catch (KeeperException | InterruptedException e) {
            throw new IllegalStateException("Exception setting up delegation-token key manager", e);
        }
        authenticationTokenKeyManagerThread = Threads.createThread("Delegation Token Key Manager", authenticationTokenKeyManager);
        authenticationTokenKeyManagerThread.start();
        boolean logged = false;
        while (!authenticationTokenKeyManager.isInitialized()) {
            // Print out a status message when we start waiting for the key manager to get initialized
            if (!logged) {
                log.info("Waiting for AuthenticationTokenKeyManager to be initialized");
                logged = true;
            }
            sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
        }
        // And log when we are initialized
        log.info("AuthenticationTokenSecretManager is initialized");
    }
    String address = sa.address.toString();
    log.info("Setting manager lock data to {}", address);
    try {
        managerLock.replaceLockData(address.getBytes());
    } catch (KeeperException | InterruptedException e) {
        throw new IllegalStateException("Exception updating manager lock", e);
    }
    while (!clientService.isServing()) {
        sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
    }
    // if the replication name is ever set, then start replication services
    final AtomicReference<TServer> replServer = new AtomicReference<>();
    context.getScheduledExecutor().scheduleWithFixedDelay(() -> {
        try {
            @SuppressWarnings("deprecation") Property p = Property.REPLICATION_NAME;
            if ((replServer.get() == null) && !getConfiguration().get(p).isEmpty()) {
                log.info("{} was set, starting repl services.", p.getKey());
                replServer.set(setupReplication());
            }
        } catch (UnknownHostException | KeeperException | InterruptedException e) {
            log.error("Error occurred starting replication services. ", e);
        }
    }, 0, 5000, TimeUnit.MILLISECONDS);
    // checking stored user hashes if any of them uses an outdated algorithm
    security.validateStoredUserCreditentials();
    // The manager is fully initialized. Clients are allowed to connect now.
    managerInitialized.set(true);
    while (clientService.isServing()) {
        sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    }
    log.info("Shutting down fate.");
    fate.shutdown();
    final long deadline = System.currentTimeMillis() + MAX_CLEANUP_WAIT_TIME;
    try {
        statusThread.join(remaining(deadline));
        if (null != replicationAssignerThread) {
            replicationAssignerThread.join(remaining(deadline));
        }
        if (null != replicationWorkThread) {
            replicationWorkThread.join(remaining(deadline));
        }
    } catch (InterruptedException e) {
        throw new IllegalStateException("Exception stopping replication workers", e);
    }
    var nullableReplServer = replServer.get();
    if (nullableReplServer != null) {
        nullableReplServer.stop();
    }
    // Signal that we want it to stop, and wait for it to do so.
    if (authenticationTokenKeyManager != null) {
        authenticationTokenKeyManager.gracefulStop();
        try {
            if (null != authenticationTokenKeyManagerThread) {
                authenticationTokenKeyManagerThread.join(remaining(deadline));
            }
        } catch (InterruptedException e) {
            throw new IllegalStateException("Exception waiting on delegation-token key manager", e);
        }
    }
    // don't stop
    for (TabletGroupWatcher watcher : watchers) {
        try {
            watcher.join(remaining(deadline));
        } catch (InterruptedException e) {
            throw new IllegalStateException("Exception waiting on watcher", e);
        }
    }
    log.info("exiting");
}